Tag new release 1.16.1

A call to assert(1) will always fail and cause Wget to crash. If such a
situation does arise, Wget should invoke abort() and provide a useful
error message to the user prior to exiting.

.gitignore

@@ -0,0 +1,93 @@
+# Project Root
+/.sc-start-sc_bindtextdomain
+/.sc-start-sc_prohibit_HAVE_MBRTOWC
+/.version
+/GNUmakefile
+/INSTALL
+/Makefile
+/Makefile.in
+/aclocal.m4
+/autom4te.cache/
+/config.cache
+/config.log
+/config.status
+/configure
+/lib/
+/maint.mk
+# Wildcard Ignores
+*~
+po/*.gmo*
+po/*.po*
+src/*.o
+tests/*.log
+tests/*.trs
+testenv/*.log
+testenv/*.trs
+# build-aux/
+build-aux/.gitignore
+build-aux/compile
+build-aux/config.guess
+build-aux/config.sub
+build-aux/depcomp
+build-aux/install-sh
+build-aux/mdate-sh
+build-aux/missing
+build-aux/snippet/
+build-aux/test-driver
+build-aux/texinfo.tex
+# doc/
+doc/Makefile
+doc/Makefile.in
+doc/sample.wgetrc.munged_for_texi_inclusion
+doc/stamp-vti
+doc/version.texi
+doc/wget.1
+doc/wget.info
+doc/wget.pod
+# m4/
+m4/.gitignore
+m4/gnulib-cache.m4
+# po/
+po/.gitignore
+po/.reference/
+po/LINGUAS
+po/Makefile
+po/Makefile.in
+po/Makevars
+po/POTFILES
+po/wget.pot
+po/stamp-po
+# src/
+src/.deps/
+src/Makefile
+src/Makefile.in
+src/build_info.c
+src/config.h
+src/config.h.in
+src/cscope.out
+src/css.c
+src/css_.c
+src/libunittest.a
+src/stamp-h1
+src/tags
+src/version.c
+src/wget
+# testenv/
+testenv/conf/__pycache__/
+testenv/exc/__pycache__/
+testenv/misc/__pycache__/
+testenv/server/__pycache__/
+testenv/server/http/__pycache__/
+testenv/test/__pycache__/
+testenv/Makefile
+testenv/Makefile.in
+# tests/
+tests/Makefile
+tests/Makefile.in
+tests/unit-tests
+# util/
+util/Makefile
+util/Makefile.in
+# others
+/tags
+/cscope.out

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "gnulib"]
+	path = gnulib
+	url = git://git.sv.gnu.org/gnulib.git

ChangeLog

@@ -1,3 +1,340 @@
+2014-12-08  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* cfg.mk (local-checks-to-skip): Add sc_cast_of_argument_to_free.
+
+2014-12-05  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: Fix message for when GnuTLS has not been found.
+
+2014-12-05  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* bootstrap.conf: Add modules fnmatch, memrchr, strcase, strptime,
+	strtoll, timegm
+	Remove module strcasecmp
+	Use module spawn-pipe instead of deprecated module pipe
+
+2014-11-25  Darshit Shah  <darnir@gmail.com>
+
+	* bootstrap.conf: Add flex as a prerequisite to building Wget from the
+	repository sources
+
+2014-11-22  Darshit Shah  <darnir@gmail.com>
+
+	* build-aux/build_info.pl: Include version.h in the final C file generated
+	by the script.
+
+2014-11-20  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Change option to --enable-assert. Assertions are disabled by
+	default.
+	* NEWS: Edit to reflect default option.
+	* README.checkout: Document --enable-assert for developers
+
+2014-11-20  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: Fix libpsl<0.6.0 detection
+
+2014-11-20  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* NEWS: remove form feeds
+
+2014-11-20  Darshit Shah  <darnir@gmail.com>
+
+	* .gitignore: po/stamp-po should be ignored
+
+2014-11-20  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Use pkg-config for libpsl detection
+	config status for libpsl is reflected by $with_libpsl
+
+2014-11-20  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Use autoconf macros where available
+
+2014-11-20  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Add the -Wextra flag when no CFLAGS are set
+
+2014-11-19  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: Check for random()
+
+2014-11-17  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* bootstrap.conf (gnulib_modules): Add module c-strcase
+
+2014-11-16  Darshit Shah  <darnir@gmail.com>
+
+	* po/stamp-po: Remove autogenerated file from checked  out sources
+
+2014-11-16  Darshit Shah  <darnir@gmail.com>
+
+	* gnulib: Update submodule
+	* bootstrap: Update script from latest gnulib
+
+2014-11-16  Darshit Shah  <darnir@gmail.com>
+
+	* .gitignore: Sort gitignore file based on directories
+
+2014-11-15  Darshit Shah  <darnir@gmail.com>
+	* configure.ac: Add AC_HEADER_ASSERT. Enables --disable-assert option
+
+2014-11-05  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: check for python3
+	* Makefile.am: only use python test suite if python3 found
+
+2014-11-05  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* contrib/tsocked-wget (TSOCKS_CONF_FILE): Remove empty new-line.
+
+2014-11-04  Darshit Shah  <darnir@gmail.com>
+
+	* README.checkout: Bootstrap script is ./bootstrap not ./bootstrap.sh
+
+2014-11-01  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Fix check for libpsl
+
+2014-11-01  Mike Frysinger <vapier@gentoo.org>
+
+	* configure.ac: Use pkg-config to check for zlib presence and
+	fallback to the previous code if that doesn't work.
+
+	* configure.ac: Use pkg-config to check for libuuid presence and
+	fallback to the previous code if that doesn't work.
+
+	* configure.ac: Use pkg-config to check for pcre presence and
+	fallback to the previous code if that doesn't work.
+
+	* configure.ac: Use pkg-config to check for openssl presence and
+	fallback to the previous code if that doesn't work.
+
+	* configure.ac: Use pkg-config to check for gnutls presence and
+	fallback to the previous code if that doesn't work.
+
+2014-07-28  Ángel González <keisial@gmail.com>
+	* contrib: Created contrib folder.
+	* contrib/tsocked-wget: Added wrapper for usage with socks proxy.
+
+2014-10-28  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: check for strlcpy()
+
+2014-10-28  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: add --enable-valgrind-tests
+
+2014-10-28  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* cfg.mk (local-checks-to-skip): Add sc_makefile_at_at_check.
+
+	* Makefile.am: Replace $FOO with @VAR@.
+
+2013-10-22  Ángel González <keisial@gmail.com>
+
+	* bootstrap.conf (gnulib_modules): Add module xstrndup.
+
+2014-09-25  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: removed WgetTest.pm.in
+	* .gitignore: removed WgetTest.pm
+
+2014-09-12  Darshit Shah  <darnir@gmail.com>
+
+	* bootstrap.conf: Add GNULib module mbiter
+
+2014-07-25  Darshit Shah  <darnir@gmail.com>
+
+	* .gitignore: Add a gitignore file for the project.
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Add testenv/Makefile to	AC_CONFIG_FILES.
+	* Makefile.am: Add testenv to SUBDIRS
+
+2014-07-22  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Fix broken code for detecting libpsl
+
+2014-07-21  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Fix check for Libpsl
+
+2014-06-28  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* cfg.mk (local-checks-to-skip): Remove some checks.
+
+2014-06-27  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* cfg.mk (VC_LIST_ALWAYS_EXCLUDE_REGEX): Ignore msdos/* and vms/* from
+	syntax-check.
+
+2014-06-22  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* configure.ac: Add check for uuid_create.
+
+2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* NEWS: Remove repeated word.
+
+	* po/POTFILES.in: Add lib/regcomp.c.
+
+	* configure.ac: Do not depend on always defined macros.
+	* vms/vms.h: Likewise.
+
+	* Makefile.am: Remove @VAR@ with $FOO.
+	* util/Makefile.am: Likewise.
+
+	* ABOUT-NLS: Remove terminating empty lines.
+	* MAILING-LIST: Likewise.
+	* util/rmold.pl: Likewise.
+
+	* util/trunc.c (main): Ensure bindtextdomain is called.
+
+2014-06-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* build-aux/build_info.pl: Remove trailing whitespaces.
+	* configure.ac: Likewise.
+
+2014-05-30  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Allow compilation without libpsl.
+	* README.checkout: Add libpsl as a dependency.
+
+2014-05-24  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* gnulib: update module.
+
+2014-02-24  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* gnulib: update module.
+	Reported by: Darshit Shah  <darnir@gmail.com>.
+
+2014-02-06  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* configure.ac: Update copyright years.
+
+2013-12-22  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* gnulib: add git submodule.
+
+2013-09-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* configure.ac: added a summary of build options
+	  fixed some indentations
+	  removed the unconditionally adding of libz with --with-ssl
+	  removed -lgcrypt and -lgpg-error for gnutls
+
+2013-07-23  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* configure.ac: Remove AM_CONDITIONAL HAVE_NETTLE.
+	Reported by: Darshit Shah  <darnir@gmail.com>.
+
+2013-07-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* configure.ac: check for libnettle when GNU TLS is used.
+
+2013-05-17 Bykov Aleksey <gnfalex@rambler.ru>
+
+	* bootstrap: Add `mkostemp'
+
+2012-10-07  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: Check for patchconf.
+
+2012-09-23  Merinov Nikolay  <kim.roader@gmail.com>
+
+	* m4/wget.m4 (WGET_FNMATCH): Add AC_LANG_SOURCE into
+	AC_COMPILE_IFELSE in order to silence autoconf 2.68 warning.
+
+2012-09-20  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap: Update from gnulib.
+
+2012-09-02  Nguyễn Thái Ngọc Duy  <pclouds@gmail.com> (tiny change)
+
+	* po/POTFILES.in: Add more files.
+
+2012-07-08  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap: Update from gnulib.
+	* bootstrap.conf (gnulib_extra_files): Remove $build_aux/missing.
+	* lib/Makefile.am: Delete file.
+
+2012-06-16  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Remove `closeout'.
+	Reported by: Micah Cowan  <micah@cowan.name>.
+
+2012-05-31  Ángel González <keisial@gmail.com>
+
+	* convert.c: fix segfault on wrong urls (bug 36570)
+
+2012-05-13  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Add `git-version-gen'.
+	* build-aux/bzr-version-gen: Remove file.
+	* configure.ac: Invoke `build-aux/git-version-gen' to get the dist
+	version.
+	* Makefile.am (EXTRA_DIST): Distribute build-aux/git-version-gen instead
+	of build-aux/bzr-version-gen.
+
+2012-04-11  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* bootstrap.conf (gnulib_modules): Include module `regex'.
+	* configure.ac: Check for PCRE library.
+
+2012-03-25 Ray Satiro <raysatiro@yahoo.com>
+
+	* configure.ac: Fix build under mingw when OpenSSL is used.
+
+2012-03-20  Ángel González <keisial@gmail.com>
+
+	* bootstrap.conf (gnulib_modules): Add modules `ftello',
+	`mkstemp' and `strtok_r'.
+
+2012-02-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Add module `closeout'.
+
+2012-01-09  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* configure.ac: Always try to use libz, even without SSL.
+
+2011-12-12  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* Makefile.am (EXTRA_DIST): Add build-aux/bzr-version-gen.
+	Reported by: Elan Ruusamäe <glen@pld-linux.org>.
+
+2011-12-11  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* util/trunc.c (main): Call `close' on the fd and check for errors.
+	Reported by: <dga@cs.cmu.edu>.
+
+2011-10-23  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Include module `vsnprintf'.
+
+2011-10-16  Steven Schubiger  <stsc@member.fsf.org>
+
+	* util/paramcheck.pl: Match 1 or more times where applicable.
+	(extract_entries): Return a copy instead of reference.
+
+2011-09-04  Alan Hourihane <alanh@fairlite.co.uk> (tiny change)
+
+	* configure.ac: Check for libz when gnutls is used.
+
+2011-08-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: Under mingw don't check for static OpenSSL
+	libraries if the shared version was already found.
+	Suggested by: Ray Satiro <raysatiro@yahoo.com>.
+
+2011-08-25  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: Check for `utime'.
+
 2011-08-11  Giuseppe Scrivano  <gscrivano@gnu.org>
 
 	* bootstrap.conf (gnulib_modules): Add `sigprocmask'.

MAILING-LIST

@@ -47,4 +47,3 @@ for submitting and discussing patches to GNU Wget.
 
    Messages from <wget-patches@sunsite.dk> are archived at
      `http://news.gmane.org/gmane.comp.web.wget.patches'.
-

Makefile.am

@@ -41,12 +41,12 @@ distuninstallcheck_listfiles = find . -type f | \
 ACLOCAL_AMFLAGS = -I m4
 
 # subdirectories in the distribution
-SUBDIRS = lib src doc po tests util
+SUBDIRS = lib src doc po tests util testenv
 
 EXTRA_DIST = ChangeLog.README MAILING-LIST \
              msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
              msdos/Makefile.WC ABOUT-NLS \
-             build-aux/build_info.pl .version
+             build-aux/build_info.pl build-aux/git-version-gen .version
 
 CLEANFILES = *~ *.bak $(DISTNAME).tar.gz
 

NEWS

@@ -1,12 +1,125 @@
 GNU Wget NEWS -- history of user-visible changes.
 
 Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Free Software
+Foundation, Inc.
 See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
-
-* Changes in Wget 1.13.2
+
+* Changes in Wget 16.1
+
+** Add --enable-assert configure option.
+
+** Use pkg-config to check for libraries presence.
+
+** Do not limit --secure-protocol=auto|pfs to TLSv1.0.
+
+** Add --secure-protocol=TLSv1_1|TLSv1_2 .
+
+** Full C89 source code compliance.
+
+** Select and use the most secure authentication scheme with HTTP connections.
+
+** Fix issues with turkish locales.
+
+** Handle 504 Gateway Timeout.
+
+** New option --crl-file to load Certificate Revocation Lists.
+
+** Add valgrind support to tests suite.
+
+** Fix an off-by-one problem in the progress bar (introduced in 1.16).
+
+* Changes in Wget 1.16
+
+** No longer create local symbolic links by default.  Closes CVE-2014-4877.
+
+** Use libpsl for verifying cookie domains.
+
+** Default progress bar output changed.
+
+** Introduce --show-progress to force display the progress bar.
+
+** Introduce --no-config.  The wgetrc files will not be read.
+
+** Introduce --start-pos to allow starting downloads from a specified position.
+
+** Fix a problem with ISA Server Proxy and keep-alive connections.
+
+* Changes in Wget 1.15
+
+** Add support for --method.
+
+** Add support for file names longer than MAX_FILE.
+
+** Support FTP listing for the FTP Server on Windows Server 2008 R2.
+
+** Fix a regression when -c and --content-disposition are used together.
+
+** Support shorthand URLs in an input file.
+
+** Fix -c with servers that don't specify a content-length.
+
+** Add support for MD5-SESS
+
+** Do not fail on non fatal GNU TLS alerts during handshake.
+
+** Add support for --https-only.  When used wget will follow only
+   HTTPS links in recursive mode.
+
+** Support Perfect-Forward Secrecy in --secure-protocol.
+
+** Fix a problem with some IRI links that are not followed when contained in a
+   HTML document.
+
+** Support some FTP servers that return an empty list with "LIST -a".
+
+** Specify Host with the HTTP CONNECT method.
+
+** Use the correct HTTP method on a redirection.
+
+* Changes in Wget 1.14
+
+** Add support for content-on-error.  It allows to store the HTTP
+   payload on 4xx or 5xx errors.
+
+** Add support for WARC files.
+
+** Fix a memory leak problem in the GNU TLS backend.
+
+** Autoreconf works again for distributed tarballs.
+
+** Print some diagnostic messages to stderr not to stdout.
+
+** Report stdout close errors.
+
+** Accept the --report-speed option.
+
+** Enable client certificates when GNU TLS is used.
+
+** Add support for TLS Server Name Indication.
+
+** Accept the arguments --accept-reject and --reject-regex.
+
+** The GNU TLS backend honors correctly the timeout value.
+
+** Add support for RFC 2617 Digest Access Authentication.
+
+* Changes in Wget 1.13.4
+
+** Now --version and --help work again.
+
+** Fix a build error on solaris 10 sparc.
+
+** Now --timestamping and --continue work well together.
+
+** Return a network failure when FTP downloads fail and --timestamping
+   is specified.
+
+** Fix a segfault on an incomplete STYLE tag.
+
+* Changes in Wget 1.13.3
 
 ** Support HTTP/1.1
 
@@ -71,7 +184,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
 ** Introduce `show_all_dns_entries' to print all IP addresses corresponding to
    a DNS name when it is resolved.
-
+
 * Changes in Wget 1.12
 
 ** Mailing list MOVED to bug-wget@gnu.org
@@ -125,7 +238,7 @@ forces the percent-encoding of all non-ASCII bytes
 now documented: save_headers, spider, and user_agent,
 auth_no_challenge, and keep_session_cookies. Also added documentation
 for the "lowercase" and "uppercase" values for --restrict-file-names, which had been present since Wget 1.11.
-
+
 * Changes in Wget 1.11.4
 
 ** Fixed an issue (apparently a regression) where -O would refuse to
@@ -139,13 +252,13 @@ locally with content.
 to a pointer-to-time_t.
 
 ** Translation updates for Catalan.
-
+
 * Changes in Wget 1.11.3
 
 ** Downgraded -N with -O to a warning, rather than an error.
 
 ** Translation updates
-
+
 * Changes in Wget 1.11.2
 
 ** Fixed a problem in authenticating over HTTPS through a proxy.
@@ -167,7 +280,7 @@ multiple times. This has been fixed in 1.11.2.
 for improved conformance with RFC 3986. However, this behavior presents
 problems for some FTP setups, and so they are now preserved again, for
 FTP URLs only.
-
+
 * Changes in Wget 1.11.1.
 
 ** Interrupted downloads no longer result in renaming the file
@@ -189,7 +302,7 @@ be surprising, and notes that they may change in the future.
 
 ** Documentation of --no-parents now explains how a trailing slash, or
 lack thereof, in the specified URL, will affect behavior.
-
+
 * Changes in Wget 1.11.
 
 ** Timestamping now uses the value from the most recent HTTP response,
@@ -235,7 +348,7 @@ available at http://hg.addictivecode.org/. Prior to this, the source
 code was hosted on Subversion (migrated from the original CVS); you can
 still get access to older tags and branches for Wget in the Subversion
 repository at http://addictivecode.org/svn/wget/.
-
+
 * Changes in Wget 1.10.
 
 ** Downloading files larger than 2GB, sometimes referred to as "large
@@ -346,9 +459,9 @@ http_password and proxy_password respectively.  The login and passwd
 .wgetrc commands have been deprecated.
 
 * `wget -b' now works correctly under Windows.
-
+
 * Wget 1.9.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.9.
 
 ** It is now possible to specify that POST method be used for HTTP
@@ -400,11 +513,11 @@ values "yes" and "no" along with the traditional "on" and "off".
 ** It is now possible to specify decimal values for timeouts, waiting
 periods, and download rate.  For instance, `--wait=0.5' now works as
 expected, as does `--dns-timeout=0.5' and even `--limit-rate=2.5k'.
-
+
 * Wget 1.8.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.8.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.8.
 
 ** A new progress indicator is now available and used by default.
@@ -464,9 +577,9 @@ no longer translated to `?', nor `%2B' to `+'.  Unsafe characters
 which are not reserved are still escaped, of course.
 
 ** No more than 20 successive redirections are allowed.
-
+
 * Wget 1.7.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.7.
 
 ** SSL (`https') pages now work if you compile Wget with SSL support;
@@ -508,7 +621,7 @@ large sites (thousands of documents).
 documentation.  (The last version that shipped with a man page was
 1.4.5).  To get this, you need to have pod2man from the Perl
 distribution installed on your system.
-
+
 * Changes in Wget 1.6
 
 ** Administrative changes.
@@ -559,13 +672,13 @@ passive_ftp is the only .wgetrc command which takes a lockable Boolean.
 bound to multiple IP addresses.
 
 ** wget now accepts (illegal per HTTP spec) relative URLs in HTTP redirects.
-
+
 * Wget 1.5.3 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.5.0
 
 ** Wget speaks many languages!
@@ -600,7 +713,7 @@ ASCII type transfer:
 
     wget "ftp://ftp.somewhere.com/welcome.msg;type=a"
 
-** `--help' and `--version' options have been redone to to conform to
+** `--help' and `--version' options have been redone to conform to
 standards set by other GNU utilities.
 
 ** Wget should now be compilable under MS Windows environment.  MS
@@ -611,11 +724,11 @@ download.
 
 ** The manual page, now hopelessly out of date, is no longer
 distributed with Wget.
-
+
 * Wget 1.4.5 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.4.4 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.4.3
 
 ** Wget is now a GNU utility.
@@ -631,7 +744,7 @@ distributed with Wget.
 ** Global wgetrc now goes to /usr/local/etc (i.e. $sysconfdir).
 
 ** Lots of bugfixes.
-
+
 * Changes in Wget 1.4.2
 
 ** New mirror site at ftp://sunsite.auc.dk/pub/infosystems/wget/,
@@ -649,7 +762,7 @@ files.
 ** --convert-links should work now.
 
 ** Minor bugfixes.
-
+
 * Changes in Wget 1.4.1
 
 ** Minor bugfixes.
@@ -657,7 +770,7 @@ files.
 ** Added -I (the opposite of -X).
 
 ** Dot tracing is now customizable; try wget --dot-style=binary
-
+
 * Changes in Wget 1.4.0
 
 ** Wget 1.4.0 [formerly known as Geturl] is an extensive rewrite of
@@ -738,7 +851,7 @@ turn on mirroring options).
 ** Lots of small features I can't remember. :-)
 
 ** A host of bugfixes.
-
+
 * Changes in Geturl 1.3
 
 ** Added FTP globbing support (ftp://fly.cc.fer.hr/*)
@@ -756,9 +869,9 @@ turn on mirroring options).
 ** Added support for robots.txt
 
 ** Fixed some minor bugs
-
+
 * Geturl 1.2 is a bugfix release with no user-visible changes.
-
+
 * Changes in Geturl 1.1
 
 ** REST supported in FTP
@@ -776,7 +889,7 @@ geturl -vo log http://fly.cc.fer.hr/
 ** <base href="xxx"> supported
 
 ** autoconf supported
-
+
 ----------------------------------------------------------------------
 Copyright information:
 

README.checkout

@@ -4,8 +4,8 @@ Compiling From Repository Sources
    does not contain automatically-generated files, even when these are
    normally present in the distribution tarballs. Therefore, to build GNU
    Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, libidn and libiconv
-   are not absolutely required):
+   more of the following (note that gettext, OpenSSL, libidn, libiconv
+   and libpsl are not absolutely required):
 
      * [20]autoconf (currently, GNU Wget requires version 2.61). This is
        needed to generate the configure script from configure.in. This is
@@ -68,10 +68,13 @@ Compiling From Repository Sources
 
      * [34]git is used to fetch gnulib files trough the bootstrap.sh script.
 
+     * [35]libpsl is required for using a public suffix list to check for valid
+       cookie domains.
+
    For those who might be confused as to what to do once they check out
    the source code, considering configure and Makefile do not yet exist at
-   that point, a shell script called bootstrap.sh has been provided. After
-   calling ./bootstrap.sh you're ready to build GNU Wget in the normal
+   that point, a shell script called bootstrap has been provided. After
+   calling ./bootstrap you're ready to build GNU Wget in the normal
    fashion, with ./configure and make.
 
    So, to sum up, after checking out the source code as described above,
@@ -80,14 +83,22 @@ Compiling From Repository Sources
  $  cd wget        # assumes you've cloned a repository to "./wget"
     2. Generate all the automatically-generated files required prior to
        configuring the package:
- $  ./bootstrap.sh
+ $  ./bootstrap
     3. Configure the package and compile it:
- $  ./configure [some_parameters]
+ $  ./configure --enable-assert [some_parameters]
  $  make
     4. Hack, compile, test, hack, compile, test...
  $  src/wget --version
  GNU Wget 1.12-devel (9cb2563197bc)
 
+ All developers are requested to enable the assertions on their development
+ builds to ensure a stable codebase. Assertions are added to state certain
+ assumptions about the code and its data which all developers should be mindful
+ of. To enable assertions, run the configure command with the --enable-assert
+ option, like this:
+
+ $ ./configure --enable-assert [other configure options]
+
  Copyright © 2008,2010 Free Software Foundation, Inc.
  License GPLv3+: GNU GPL version 3 or later
  <http://www.gnu.org/licenses/gpl.html>.
@@ -96,21 +107,21 @@ Compiling From Repository Sources
 
  Originally written by Hrvoje Niksic <hniksic@xemacs.org>.
 
-     * [35]Edit
-     * [36]Comments
-     * [37]Info
-     * [38]Attachments
+     * [36]Edit
+     * [37]Comments
+     * [38]Info
+     * [39]Attachments
      * More Actions:
        [Raw Text................] Do
 
-     * [39]MoinMoin Powered
-     * [40]Python Powered
-     * [41]GPL licensed
-     * [42]Valid HTML 4.01
+     * [40]MoinMoin Powered
+     * [41]Python Powered
+     * [42]GPL licensed
+     * [43]Valid HTML 4.01
      __________________________________________________________________
 
    All content © 2007 Free Software Foundation. For terms of use,
-   redistribution, and modification, please see the [43]WikiLicense page.
+   redistribution, and modification, please see the [44]WikiLicense page.
 
 References
 
@@ -129,12 +140,13 @@ References
   32. http://www.gnu.org/software/libidn/
   33. http://www.gnu.org/software/libiconv/
   34. http://git-scm.com/
-  35. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
-  36. http://wget.addictivecode.org/CompilingRepoSources
-  37. http://wget.addictivecode.org/CompilingRepoSources?action=info
-  38. http://wget.addictivecode.org/CompilingRepoSources?action=AttachFile
-  39. http://moinmo.in/
-  40. http://moinmo.in/Python
-  41. http://moinmo.in/GPL
-  42. http://validator.w3.org/check?uri=referer
-  43. http://wget.addictivecode.org/WikiLicense
+  35. https://github.com/rockdaboot/libpsl
+  36. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
+  37. http://wget.addictivecode.org/CompilingRepoSources
+  38. http://wget.addictivecode.org/CompilingRepoSources?action=info
+  39. http://wget.addictivecode.org/CompilingRepoSources?action=AttachFile
+  40. http://moinmo.in/
+  41. http://moinmo.in/Python
+  42. http://moinmo.in/GPL
+  43. http://validator.w3.org/check?uri=referer
+  44. http://wget.addictivecode.org/WikiLicense

bootstrap

@@ -1,10 +1,10 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2011-04-05.18; # UTC
+scriptversion=2013-12-05.23; # UTC
 
 # Bootstrap this package from checked-out sources.
 
-# Copyright (C) 2003-2011 Free Software Foundation, Inc.
+# Copyright (C) 2003-2014 Free Software Foundation, Inc.
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -36,12 +36,12 @@ nl='
 LC_ALL=C
 export LC_ALL
 
+# Ensure that CDPATH is not set.  Otherwise, the output from cd
+# would cause trouble in at least one use below.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
 local_gl_dir=gl
 
-# Temporary directory names.
-bt='._bootmp'
-bt_regex=`echo "$bt"| sed 's/\./[.]/g'`
-bt2=${bt}2
 me=$0
 
 usage() {
@@ -77,6 +77,33 @@ Running without arguments will suffice in most cases.
 EOF
 }
 
+# warnf_ FORMAT-STRING ARG1...
+warnf_ ()
+{
+  warnf_format_=$1
+  shift
+  nl='
+'
+  case $* in
+    *$nl*) me_=$(printf "$me"|tr "$nl|" '??')
+       printf "$warnf_format_" "$@" | sed "s|^|$me_: |" ;;
+    *) printf "$me: $warnf_format_" "$@" ;;
+  esac >&2
+}
+
+# warn_ WORD1...
+warn_ ()
+{
+  # If IFS does not start with ' ', set it and emit the warning in a subshell.
+  case $IFS in
+    ' '*) warnf_ '%s\n' "$*";;
+    *)    (IFS=' '; warn_ "$@");;
+  esac
+}
+
+# die WORD1...
+die() { warn_ "$@"; exit 1; }
+
 # Configuration.
 
 # Name of the Makefile.am
@@ -88,9 +115,12 @@ gnulib_modules=
 # Any gnulib files needed that are not in modules.
 gnulib_files=
 
-# A function to be called to edit gnulib.mk right after it's created.
+: ${AUTOPOINT=autopoint}
+: ${AUTORECONF=autoreconf}
+
+# A function to be called right after gnulib-tool is run.
 # Override it via your own definition in bootstrap.conf.
-gnulib_mk_hook() { :; }
+bootstrap_post_import_hook() { :; }
 
 # A function to be called after everything else in this script.
 # Override it via your own definition in bootstrap.conf.
@@ -105,24 +135,31 @@ po_download_command_format=\
 "rsync --delete --exclude '*.s1' -Lrtvz \
  'translationproject.org::tp/latest/%s/' '%s'"
 
+# Fallback for downloading .po files (if rsync fails).
+po_download_command_format2=\
+"wget --mirror -nd -q -np -A.po -P '%s' \
+ http://translationproject.org/latest/%s/"
+
+# Prefer a non-empty tarname (4th argument of AC_INIT if given), else
+# fall back to the package name (1st argument with munging)
 extract_package_name='
-  /^AC_INIT(/{
-     /.*,.*,.*, */{
-       s///
-       s/[][]//g
-       s/)$//
+  /^AC_INIT(\[*/{
+     s///
+     /^[^,]*,[^,]*,[^,]*,[ []*\([^][ ,)]\)/{
+       s//\1/
+       s/[],)].*//
        p
        q
      }
-     s/AC_INIT(\[*//
-     s/]*,.*//
+     s/[],)].*//
      s/^GNU //
      y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/
-     s/[^A-Za-z0-9_]/-/g
+     s/[^abcdefghijklmnopqrstuvwxyz0123456789_]/-/g
      p
   }
 '
-package=`sed -n "$extract_package_name" configure.ac` || exit
+package=$(sed -n "$extract_package_name" configure.ac) \
+  || die 'cannot find package name in configure.ac'
 gnulib_name=lib$package
 
 build_aux=build-aux
@@ -172,12 +209,28 @@ bootstrap_sync=false
 # Use git to update gnulib sources
 use_git=true
 
+check_exists() {
+  if test "$1" = "--verbose"; then
+    ($2 --version </dev/null) >/dev/null 2>&1
+    if test $? -ge 126; then
+      # If not found, run with diagnostics as one may be
+      # presented with env variables to set to find the right version
+      ($2 --version </dev/null)
+    fi
+  else
+    ($1 --version </dev/null) >/dev/null 2>&1
+  fi
+
+  test $? -lt 126
+}
+
 # find_tool ENVVAR NAMES...
 # -------------------------
 # Search for a required program.  Use the value of ENVVAR, if set,
-# otherwise find the first of the NAMES that can be run (i.e.,
-# supports --version).  If found, set ENVVAR to the program name,
-# die otherwise.
+# otherwise find the first of the NAMES that can be run.
+# If found, set ENVVAR to the program name, die otherwise.
+#
+# FIXME: code duplication, see also gnu-web-doc-update.
 find_tool ()
 {
   find_tool_envvar=$1
@@ -185,31 +238,21 @@ find_tool ()
   find_tool_names=$@
   eval "find_tool_res=\$$find_tool_envvar"
   if test x"$find_tool_res" = x; then
-    for i
-    do
-      if ($i --version </dev/null) >/dev/null 2>&1; then
-       find_tool_res=$i
-       break
+    for i; do
+      if check_exists $i; then
+        find_tool_res=$i
+        break
       fi
     done
-  else
-    find_tool_error_prefix="\$$find_tool_envvar: "
   fi
   if test x"$find_tool_res" = x; then
-    echo >&2 "$me: one of these is required: $find_tool_names"
-    exit 1
+    warn_ "one of these is required: $find_tool_names;"
+    die   "alternatively set $find_tool_envvar to a compatible tool"
   fi
-  ($find_tool_res --version </dev/null) >/dev/null 2>&1 || {
-    echo >&2 "$me: ${find_tool_error_prefix}cannot run $find_tool_res --version"
-    exit 1
-  }
   eval "$find_tool_envvar=\$find_tool_res"
   eval "export $find_tool_envvar"
 }
 
-# Find sha1sum, named gsha1sum on MacPorts, and shasum on MacOS 10.6.
-find_tool SHA1SUM sha1sum gsha1sum shasum
-
 # Override the default configuration, if necessary.
 # Make sure that bootstrap.conf is sourced from the current directory
 # if we were invoked as "sh bootstrap".
@@ -221,13 +264,12 @@ esac
 # Extra files from gnulib, which override files from other sources.
 test -z "${gnulib_extra_files}" && \
   gnulib_extra_files="
-        $build_aux/install-sh
-        $build_aux/missing
-        $build_aux/mdate-sh
-        $build_aux/texinfo.tex
-        $build_aux/depcomp
-        $build_aux/config.guess
-        $build_aux/config.sub
+        build-aux/install-sh
+        build-aux/mdate-sh
+        build-aux/texinfo.tex
+        build-aux/depcomp
+        build-aux/config.guess
+        build-aux/config.sub
         doc/INSTALL
 "
 
@@ -248,7 +290,7 @@ do
     usage
     exit;;
   --gnulib-srcdir=*)
-    GNULIB_SRCDIR=`expr "X$option" : 'X--gnulib-srcdir=\(.*\)'`;;
+    GNULIB_SRCDIR=${option#--gnulib-srcdir=};;
   --skip-po)
     SKIP_PO=t;;
   --force)
@@ -262,62 +304,56 @@ do
   --no-git)
     use_git=false;;
   *)
-    echo >&2 "$0: $option: unknown option"
-    exit 1;;
+    die "$option: unknown option";;
   esac
 done
 
-if $use_git || test -d "$GNULIB_SRCDIR"; then
-  :
-else
-  echo "$0: Error: --no-git requires --gnulib-srcdir" >&2
-  exit 1
-fi
+$use_git || test -d "$GNULIB_SRCDIR" \
+  || die "Error: --no-git requires --gnulib-srcdir"
 
 if test -n "$checkout_only_file" && test ! -r "$checkout_only_file"; then
-  echo "$0: Bootstrapping from a non-checked-out distribution is risky." >&2
-  exit 1
+  die "Bootstrapping from a non-checked-out distribution is risky."
 fi
 
-# Ensure that lines starting with ! sort last, per gitignore conventions
-# for whitelisting exceptions after a more generic blacklist pattern.
-sort_patterns() {
-  sort -u "$@" | sed '/^!/ {
-    H
-    d
-  }
-  $ {
-    P
-    x
-    s/^\n//
-  }'
+# Strip blank and comment lines to leave significant entries.
+gitignore_entries() {
+  sed '/^#/d; /^$/d' "$@"
 }
 
-# If $STR is not already on a line by itself in $FILE, insert it,
-# sorting the new contents of the file and replacing $FILE with the result.
-insert_sorted_if_absent() {
+# If $STR is not already on a line by itself in $FILE, insert it at the start.
+# Entries are inserted at the start of the ignore list to ensure existing
+# entries starting with ! are not overridden.  Such entries support
+# whitelisting exceptions after a more generic blacklist pattern.
+insert_if_absent() {
   file=$1
   str=$2
   test -f $file || touch $file
-  echo "$str" | sort_patterns - $file | cmp - $file > /dev/null \
-    || { echo "$str" | sort_patterns - $file > $file.bak \
-      && mv $file.bak $file; } \
-    || exit 1
+  test -r $file || die "Error: failed to read ignore file: $file"
+  duplicate_entries=$(gitignore_entries $file | sort | uniq -d)
+  if [ "$duplicate_entries" ] ; then
+    die "Error: Duplicate entries in $file: " $duplicate_entries
+  fi
+  linesold=$(gitignore_entries $file | wc -l)
+  linesnew=$( { echo "$str"; cat $file; } | gitignore_entries | sort -u | wc -l)
+  if [ $linesold != $linesnew ] ; then
+    { echo "$str" | cat - $file > $file.bak && mv $file.bak $file; } \
+      || die "insert_if_absent $file $str: failed"
+  fi
 }
 
 # Adjust $PATTERN for $VC_IGNORE_FILE and insert it with
-# insert_sorted_if_absent.
+# insert_if_absent.
 insert_vc_ignore() {
   vc_ignore_file="$1"
   pattern="$2"
   case $vc_ignore_file in
   *.gitignore)
-    # A .gitignore entry that does not start with `/' applies
-    # recursively to subdirectories, so prepend `/' to every
+    # A .gitignore entry that does not start with '/' applies
+    # recursively to subdirectories, so prepend '/' to every
     # .gitignore entry.
-    pattern=`echo "$pattern" | sed s,^,/,`;;
+    pattern=$(echo "$pattern" | sed s,^,/,);;
   esac
-  insert_sorted_if_absent "$vc_ignore_file" "$pattern"
+  insert_if_absent "$vc_ignore_file" "$pattern"
 }
 
 # Die if there is no AC_CONFIG_AUX_DIR($build_aux) line in configure.ac.
@@ -326,11 +362,8 @@ grep '^[	 ]*AC_CONFIG_AUX_DIR(\['"$build_aux"'\])' configure.ac \
     >/dev/null && found_aux_dir=yes
 grep '^[	 ]*AC_CONFIG_AUX_DIR('"$build_aux"')' configure.ac \
     >/dev/null && found_aux_dir=yes
-if test $found_aux_dir = no; then
-  echo "$0: expected line not found in configure.ac. Add the following:" >&2
-  echo "  AC_CONFIG_AUX_DIR([$build_aux])" >&2
-  exit 1
-fi
+test $found_aux_dir = yes \
+  || die "configure.ac lacks 'AC_CONFIG_AUX_DIR([$build_aux])'; add it"
 
 # If $build_aux doesn't exist, create it now, otherwise some bits
 # below will malfunction.  If creating it, also mark it as ignored.
@@ -385,7 +418,7 @@ sort_ver() { # sort -V is not generally available
 get_version() {
   app=$1
 
-  $app --version >/dev/null 2>&1 || return 1
+  $app --version >/dev/null 2>&1 || { $app --version; return 1; }
 
   $app --version 2>&1 |
   sed -n '# Move version to start of line.
@@ -419,20 +452,50 @@ check_versions() {
       $use_git || continue
     fi
     # Honor $APP variables ($TAR, $AUTOCONF, etc.)
-    appvar=`echo $app | tr '[a-z]-' '[A-Z]_'`
+    appvar=$(echo $app | LC_ALL=C tr '[a-z]-' '[A-Z]_')
     test "$appvar" = TAR && appvar=AMTAR
-    eval "app=\${$appvar-$app}"
-    inst_ver=$(get_version $app)
-    if [ ! "$inst_ver" ]; then
-      echo "$me: Error: '$app' not found" >&2
-      ret=1
-    elif [ ! "$req_ver" = "-" ]; then
-      latest_ver=$(sort_ver $req_ver $inst_ver | cut -d' ' -f2)
-      if [ ! "$latest_ver" = "$inst_ver" ]; then
-        echo "$me: Error: '$app' version == $inst_ver is too old" >&2
-        echo "       '$app' version >= $req_ver is required" >&2
+    case $appvar in
+        GZIP) ;; # Do not use $GZIP:  it contains gzip options.
+        *) eval "app=\${$appvar-$app}" ;;
+    esac
+
+    # Handle the still-experimental Automake-NG programs specially.
+    # They remain named as the mainstream Automake programs ("automake",
+    # and "aclocal") to avoid gratuitous incompatibilities with
+    # pre-existing usages (by, say, autoreconf, or custom autogen.sh
+    # scripts), but correctly identify themselves (as being part of
+    # "GNU automake-ng") when asked their version.
+    case $app in
+      automake-ng|aclocal-ng)
+        app=${app%-ng}
+        ($app --version | grep '(GNU automake-ng)') >/dev/null 2>&1 || {
+          warn_ "Error: '$app' not found or not from Automake-NG"
+          ret=1
+          continue
+        } ;;
+    esac
+    if [ "$req_ver" = "-" ]; then
+      # Merely require app to exist; not all prereq apps are well-behaved
+      # so we have to rely on $? rather than get_version.
+      if ! check_exists --verbose $app; then
+        warn_ "Error: '$app' not found"
         ret=1
       fi
+    else
+      # Require app to produce a new enough version string.
+      inst_ver=$(get_version $app)
+      if [ ! "$inst_ver" ]; then
+        warn_ "Error: '$app' not found"
+        ret=1
+      else
+        latest_ver=$(sort_ver $req_ver $inst_ver | cut -d' ' -f2)
+        if [ ! "$latest_ver" = "$inst_ver" ]; then
+          warnf_ '%s\n'                                        \
+              "Error: '$app' version == $inst_ver is too old"  \
+              "       '$app' version >= $req_ver is required"
+          ret=1
+        fi
+      fi
     fi
   done
 
@@ -447,6 +510,12 @@ print_versions() {
   # can't depend on column -t
 }
 
+# Find sha1sum, named gsha1sum on MacPorts, shasum on Mac OS X 10.6.
+# Also find the compatible sha1 utility on the BSDs
+if test x"$SKIP_PO" = x; then
+  find_tool SHA1SUM sha1sum gsha1sum shasum sha1
+fi
+
 use_libtool=0
 # We'd like to use grep -E, to see if any of LT_INIT,
 # AC_PROG_LIBTOOL, AM_PROG_LIBTOOL is used in configure.ac,
@@ -459,23 +528,54 @@ if test $use_libtool = 1; then
   find_tool LIBTOOLIZE glibtoolize libtoolize
 fi
 
+# gnulib-tool requires at least automake and autoconf.
+# If either is not listed, add it (with minimum version) as a prerequisite.
+case $buildreq in
+  *automake*) ;;
+  *) buildreq="automake 1.9
+$buildreq" ;;
+esac
+case $buildreq in
+  *autoconf*) ;;
+  *) buildreq="autoconf 2.59
+$buildreq" ;;
+esac
+
+# When we can deduce that gnulib-tool will require patch,
+# and when patch is not already listed as a prerequisite, add it, too.
+if test -d "$local_gl_dir" \
+    && ! find "$local_gl_dir" -name '*.diff' -exec false {} +; then
+  case $buildreq in
+    *patch*) ;;
+    *) buildreq="patch -
+$buildreq" ;;
+  esac
+fi
+
 if ! printf "$buildreq" | check_versions; then
   echo >&2
   if test -f README-prereq; then
-    echo "$0: See README-prereq for how to get the prerequisite programs" >&2
+    die "See README-prereq for how to get the prerequisite programs"
   else
-    echo "$0: Please install the prerequisite programs" >&2
+    die "Please install the prerequisite programs"
   fi
-  exit 1
+fi
+
+# Warn the user if autom4te appears to be broken; this causes known
+# issues with at least gettext 0.18.3.
+probe=$(echo 'm4_quote([hi])' | autom4te -l M4sugar -t 'm4_quote:$%' -)
+if test "x$probe" != xhi; then
+  warn_ "WARNING: your autom4te wrapper eats stdin;"
+  warn_ "if bootstrap fails, consider upgrading your autotools"
 fi
 
 echo "$0: Bootstrapping from checked-out $package sources..."
 
 # See if we can use gnulib's git-merge-changelog merge driver.
-if test -d .git && (git --version) >/dev/null 2>/dev/null ; then
+if $use_git && test -d .git && check_exists git; then
   if git config merge.merge-changelog.driver >/dev/null ; then
     :
-  elif (git-merge-changelog --version) >/dev/null 2>/dev/null ; then
+  elif check_exists git-merge-changelog; then
     echo "$0: initializing git-merge-changelog driver"
     git config merge.merge-changelog.name 'GNU-style ChangeLog merge driver'
     git config merge.merge-changelog.driver 'git-merge-changelog %O %A %B'
@@ -495,17 +595,21 @@ git_modules_config () {
   test -f .gitmodules && git config --file .gitmodules "$@"
 }
 
-gnulib_path=`git_modules_config submodule.gnulib.path`
-test -z "$gnulib_path" && gnulib_path=gnulib
+if $use_git; then
+  gnulib_path=$(git_modules_config submodule.gnulib.path)
+  test -z "$gnulib_path" && gnulib_path=gnulib
+fi
 
-# Get gnulib files.
+# Get gnulib files.  Populate $GNULIB_SRCDIR, possibly updating a
+# submodule, for use in the rest of the script.
 
 case ${GNULIB_SRCDIR--} in
 -)
+  # Note that $use_git is necessarily true in this case.
   if git_modules_config submodule.gnulib.url >/dev/null; then
     echo "$0: getting gnulib files..."
-    git submodule init || exit $?
-    git submodule update || exit $?
+    git submodule init -- "$gnulib_path" || exit $?
+    git submodule update -- "$gnulib_path" || exit $?
 
   elif [ ! -d "$gnulib_path" ]; then
     echo "$0: getting gnulib files..."
@@ -522,8 +626,8 @@ case ${GNULIB_SRCDIR--} in
   GNULIB_SRCDIR=$gnulib_path
   ;;
 *)
-  # Use GNULIB_SRCDIR as a reference.
-  if test -d "$GNULIB_SRCDIR"/.git && \
+  # Use GNULIB_SRCDIR directly or as a reference.
+  if $use_git && test -d "$GNULIB_SRCDIR"/.git && \
         git_modules_config submodule.gnulib.url >/dev/null; then
     echo "$0: getting gnulib files..."
     if git submodule -h|grep -- --reference > /dev/null; then
@@ -534,13 +638,14 @@ case ${GNULIB_SRCDIR--} in
       # This fallback allows at least git 1.5.5.
       if test -f "$gnulib_path"/gnulib-tool; then
         # Since file already exists, assume submodule init already complete.
-        git submodule update || exit $?
+        git submodule update -- "$gnulib_path" || exit $?
       else
         # Older git can't clone into an empty directory.
         rmdir "$gnulib_path" 2>/dev/null
         git clone --reference "$GNULIB_SRCDIR" \
           "$(git_modules_config submodule.gnulib.url)" "$gnulib_path" \
-          && git submodule init && git submodule update \
+          && git submodule init -- "$gnulib_path" \
+          && git submodule update -- "$gnulib_path" \
           || exit $?
       fi
     fi
@@ -549,18 +654,25 @@ case ${GNULIB_SRCDIR--} in
   ;;
 esac
 
+# $GNULIB_SRCDIR now points to the version of gnulib to use, and
+# we no longer need to use git or $gnulib_path below here.
+
 if $bootstrap_sync; then
   cmp -s "$0" "$GNULIB_SRCDIR/build-aux/bootstrap" || {
     echo "$0: updating bootstrap and restarting..."
+    case $(sh -c 'echo "$1"' -- a) in
+      a) ignored=--;;
+      *) ignored=ignored;;
+    esac
     exec sh -c \
       'cp "$1" "$2" && shift && exec "${CONFIG_SHELL-/bin/sh}" "$@"' \
-      -- "$GNULIB_SRCDIR/build-aux/bootstrap" \
+      $ignored "$GNULIB_SRCDIR/build-aux/bootstrap" \
       "$0" "$@" --no-bootstrap-sync
   }
 fi
 
 gnulib_tool=$GNULIB_SRCDIR/gnulib-tool
-<$gnulib_tool || exit
+<$gnulib_tool || exit $?
 
 # Get translations.
 
@@ -568,7 +680,10 @@ download_po_files() {
   subdir=$1
   domain=$2
   echo "$me: getting translations into $subdir for $domain..."
-  cmd=`printf "$po_download_command_format" "$domain" "$subdir"`
+  cmd=$(printf "$po_download_command_format" "$domain" "$subdir")
+  eval "$cmd" && return
+  # Fallback to HTTP.
+  cmd=$(printf "$po_download_command_format2" "$subdir" "$domain")
   eval "$cmd"
 }
 
@@ -591,7 +706,7 @@ update_po_files() {
     && ls "$ref_po_dir"/*.po 2>/dev/null |
       sed 's|.*/||; s|\.po$||' > "$po_dir/LINGUAS" || return
 
-  langs=`cd $ref_po_dir && echo *.po|sed 's/\.po//g'`
+  langs=$(cd $ref_po_dir && echo *.po | sed 's/\.po//g')
   test "$langs" = '*' && langs=x
   for po in $langs; do
     case $po in x) continue;; esac
@@ -599,11 +714,10 @@ update_po_files() {
     cksum_file="$ref_po_dir/$po.s1"
     if ! test -f "$cksum_file" ||
         ! test -f "$po_dir/$po.po" ||
-        ! $SHA1SUM -c --status "$cksum_file" \
-            < "$new_po" > /dev/null; then
+        ! $SHA1SUM -c "$cksum_file" < "$new_po" > /dev/null 2>&1; then
       echo "$me: updated $po_dir/$po.po..."
       cp "$new_po" "$po_dir/$po.po" \
-          && $SHA1SUM < "$new_po" > "$cksum_file"
+          && $SHA1SUM < "$new_po" > "$cksum_file" || return
     fi
   done
 }
@@ -628,18 +742,18 @@ symlink_to_dir()
 
     # If the destination directory doesn't exist, create it.
     # This is required at least for "lib/uniwidth/cjk.h".
-    dst_dir=`dirname "$dst"`
+    dst_dir=$(dirname "$dst")
     if ! test -d "$dst_dir"; then
       mkdir -p "$dst_dir"
 
       # If we've just created a directory like lib/uniwidth,
       # tell version control system(s) it's ignorable.
       # FIXME: for now, this does only one level
-      parent=`dirname "$dst_dir"`
+      parent=$(dirname "$dst_dir")
       for dot_ig in x $vc_ignore; do
         test $dot_ig = x && continue
         ig=$parent/$dot_ig
-        insert_vc_ignore $ig `echo "$dst_dir"|sed 's,.*/,,'`
+        insert_vc_ignore $ig "${dst_dir##*/}"
       done
     fi
 
@@ -656,21 +770,28 @@ symlink_to_dir()
         cp -fp "$src" "$dst"
       }
     else
+      # Leave any existing symlink alone, if it already points to the source,
+      # so that broken build tools that care about symlink times
+      # aren't confused into doing unnecessary builds.  Conversely, if the
+      # existing symlink's time stamp is older than the source, make it afresh,
+      # so that broken tools aren't confused into skipping needed builds.  See
+      # <http://lists.gnu.org/archive/html/bug-gnulib/2011-05/msg00326.html>.
       test -h "$dst" &&
-      src_ls=`ls -diL "$src" 2>/dev/null` && set $src_ls && src_i=$1 &&
-      dst_ls=`ls -diL "$dst" 2>/dev/null` && set $dst_ls && dst_i=$1 &&
-      test "$src_i" = "$dst_i" || {
+      src_ls=$(ls -diL "$src" 2>/dev/null) && set $src_ls && src_i=$1 &&
+      dst_ls=$(ls -diL "$dst" 2>/dev/null) && set $dst_ls && dst_i=$1 &&
+      test "$src_i" = "$dst_i" &&
+      both_ls=$(ls -dt "$src" "$dst") &&
+      test "X$both_ls" = "X$dst$nl$src" || {
         dot_dots=
         case $src in
         /*) ;;
         *)
           case /$dst/ in
           *//* | */../* | */./* | /*/*/*/*/*/)
-             echo >&2 "$me: invalid symlink calculation: $src -> $dst"
-             exit 1;;
-          /*/*/*/*/)	dot_dots=../../../;;
-          /*/*/*/)	dot_dots=../../;;
-          /*/*/)	dot_dots=../;;
+             die "invalid symlink calculation: $src -> $dst";;
+          /*/*/*/*/)    dot_dots=../../../;;
+          /*/*/*/)      dot_dots=../../;;
+          /*/*/)        dot_dots=../;;
           esac;;
         esac
 
@@ -681,164 +802,94 @@ symlink_to_dir()
   }
 }
 
-cp_mark_as_generated()
-{
-  cp_src=$1
-  cp_dst=$2
-
-  if cmp -s "$cp_src" "$GNULIB_SRCDIR/$cp_dst"; then
-    symlink_to_dir "$GNULIB_SRCDIR" "$cp_dst"
-  elif cmp -s "$cp_src" "$local_gl_dir/$cp_dst"; then
-    symlink_to_dir $local_gl_dir "$cp_dst"
-  else
-    case $cp_dst in
-      *.[ch])             c1='/* '; c2=' */';;
-      *.texi)             c1='@c '; c2=     ;;
-      *.m4|*/Make*|Make*) c1='# ' ; c2=     ;;
-      *)                  c1=     ; c2=     ;;
-    esac
-
-    # If the destination directory doesn't exist, create it.
-    # This is required at least for "lib/uniwidth/cjk.h".
-    dst_dir=`dirname "$cp_dst"`
-    test -d "$dst_dir" || mkdir -p "$dst_dir"
-
-    if test -z "$c1"; then
-      cmp -s "$cp_src" "$cp_dst" || {
-        # Copy the file first to get proper permissions if it
-        # doesn't already exist.  Then overwrite the copy.
-        echo "$me: cp -f $cp_src $cp_dst" &&
-        rm -f "$cp_dst" &&
-        cp "$cp_src" "$cp_dst-t" &&
-        sed "s!$bt_regex/!!g" "$cp_src" > "$cp_dst-t" &&
-        mv -f "$cp_dst-t" "$cp_dst"
-      }
-    else
-      # Copy the file first to get proper permissions if it
-      # doesn't already exist.  Then overwrite the copy.
-      cp "$cp_src" "$cp_dst-t" &&
-      (
-        echo "$c1-*- buffer-read-only: t -*- vi: set ro:$c2" &&
-        echo "${c1}DO NOT EDIT! GENERATED AUTOMATICALLY!$c2" &&
-        sed "s!$bt_regex/!!g" "$cp_src"
-      ) > $cp_dst-t &&
-      if cmp -s "$cp_dst-t" "$cp_dst"; then
-        rm -f "$cp_dst-t"
-      else
-        echo "$me: cp $cp_src $cp_dst # with edits" &&
-        mv -f "$cp_dst-t" "$cp_dst"
-      fi
-    fi
-  fi
-}
-
 version_controlled_file() {
-  dir=$1
+  parent=$1
   file=$2
-  found=no
-  if test -d CVS; then
-    grep -F "/$file/" $dir/CVS/Entries 2>/dev/null |
-             grep '^/[^/]*/[0-9]' > /dev/null && found=yes
-  elif test -d .git; then
-    git rm -n "$dir/$file" > /dev/null 2>&1 && found=yes
+  if test -d .git; then
+    git rm -n "$file" > /dev/null 2>&1
   elif test -d .svn; then
-    svn log -r HEAD "$dir/$file" > /dev/null 2>&1 && found=yes
+    svn log -r HEAD "$file" > /dev/null 2>&1
+  elif test -d CVS; then
+    grep -F "/${file##*/}/" "$parent/CVS/Entries" 2>/dev/null |
+             grep '^/[^/]*/[0-9]' > /dev/null
   else
-    echo "$me: no version control for $dir/$file?" >&2
+    warn_ "no version control for $file?"
+    false
   fi
-  test $found = yes
 }
 
-slurp() {
-  for dir in . `(cd $1 && find * -type d -print)`; do
-    copied=
-    sep=
-    for file in `ls -a $1/$dir`; do
-      case $file in
-      .|..) continue;;
-      # FIXME: should all file names starting with "." be ignored?
-      .*) continue;;
-      esac
-      test -d $1/$dir/$file && continue
-      for excluded_file in $excluded_files; do
-        test "$dir/$file" = "$excluded_file" && continue 2
+# NOTE: we have to be careful to run both autopoint and libtoolize
+# before gnulib-tool, since gnulib-tool is likely to provide newer
+# versions of files "installed" by these two programs.
+# Then, *after* gnulib-tool (see below), we have to be careful to
+# run autoreconf in such a way that it does not run either of these
+# two just-pre-run programs.
+
+# Import from gettext.
+with_gettext=yes
+grep '^[	 ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \
+    with_gettext=no
+
+if test $with_gettext = yes || test $use_libtool = 1; then
+
+  tempbase=.bootstrap$$
+  trap "rm -f $tempbase.0 $tempbase.1" 1 2 13 15
+
+  > $tempbase.0 > $tempbase.1 &&
+  find . ! -type d -print | sort > $tempbase.0 || exit
+
+  if test $with_gettext = yes; then
+    # Released autopoint has the tendency to install macros that have been
+    # obsoleted in current gnulib, so run this before gnulib-tool.
+    echo "$0: $AUTOPOINT --force"
+    $AUTOPOINT --force || exit
+  fi
+
+  # Autoreconf runs aclocal before libtoolize, which causes spurious
+  # warnings if the initial aclocal is confused by the libtoolized
+  # (or worse out-of-date) macro directory.
+  # libtoolize 1.9b added the --install option; but we support back
+  # to libtoolize 1.5.22, where the install action was default.
+  if test $use_libtool = 1; then
+    install=
+    case $($LIBTOOLIZE --help) in
+      *--install*) install=--install ;;
+    esac
+    echo "running: $LIBTOOLIZE $install --copy"
+    $LIBTOOLIZE $install --copy
+  fi
+
+  find . ! -type d -print | sort >$tempbase.1
+  old_IFS=$IFS
+  IFS=$nl
+  for file in $(comm -13 $tempbase.0 $tempbase.1); do
+    IFS=$old_IFS
+    parent=${file%/*}
+    version_controlled_file "$parent" "$file" || {
+      for dot_ig in x $vc_ignore; do
+        test $dot_ig = x && continue
+        ig=$parent/$dot_ig
+        insert_vc_ignore "$ig" "${file##*/}"
       done
-      if test $file = Makefile.am && test "X$gnulib_mk" != XMakefile.am; then
-        copied=$copied${sep}$gnulib_mk; sep=$nl
-        remove_intl='/^[^#].*\/intl/s/^/#/;'"s!$bt_regex/!!g"
-        sed "$remove_intl" $1/$dir/$file |
-        cmp - $dir/$gnulib_mk > /dev/null || {
-          echo "$me: Copying $1/$dir/$file to $dir/$gnulib_mk ..." &&
-          rm -f $dir/$gnulib_mk &&
-          sed "$remove_intl" $1/$dir/$file >$dir/$gnulib_mk &&
-          gnulib_mk_hook $dir/$gnulib_mk
-        }
-      elif { test "${2+set}" = set && test -r $2/$dir/$file; } ||
-           version_controlled_file $dir $file; then
-        echo "$me: $dir/$file overrides $1/$dir/$file"
-      else
-        copied=$copied$sep$file; sep=$nl
-        if test $file = gettext.m4; then
-          echo "$me: patching m4/gettext.m4 to remove need for intl/* ..."
-          rm -f $dir/$file
-          sed '
-            /^AC_DEFUN(\[AM_INTL_SUBDIR],/,/^]/c\
-              AC_DEFUN([AM_INTL_SUBDIR], [])
-            /^AC_DEFUN(\[gt_INTL_SUBDIR_CORE],/,/^]/c\
-              AC_DEFUN([gt_INTL_SUBDIR_CORE], [])
-            $a\
-              AC_DEFUN([gl_LOCK_EARLY], [])
-          ' $1/$dir/$file >$dir/$file
-        else
-          cp_mark_as_generated $1/$dir/$file $dir/$file
-        fi
-      fi || exit
-    done
-
-    for dot_ig in x $vc_ignore; do
-      test $dot_ig = x && continue
-      ig=$dir/$dot_ig
-      if test -n "$copied"; then
-        insert_vc_ignore $ig "$copied"
-        # If an ignored file name ends with .in.h, then also add
-        # the name with just ".h".  Many gnulib headers are generated,
-        # e.g., stdint.in.h -> stdint.h, dirent.in.h ->..., etc.
-        # Likewise for .gperf -> .h, .y -> .c, and .sin -> .sed
-        f=`echo "$copied" |
-          sed '
-            s/\.in\.h$/.h/
-            s/\.sin$/.sed/
-            s/\.y$/.c/
-            s/\.gperf$/.h/
-          '
-        `
-        insert_vc_ignore $ig "$f"
-
-        # For files like sys_stat.in.h and sys_time.in.h, record as
-        # ignorable the directory we might eventually create: sys/.
-        f=`echo "$copied"|sed 's/sys_.*\.in\.h$/sys/'`
-        insert_vc_ignore $ig "$f"
-      fi
-    done
+    }
   done
-}
+  IFS=$old_IFS
 
-
-# Create boot temporary directories to import from gnulib and gettext.
-rm -fr $bt $bt2 &&
-mkdir $bt $bt2 || exit
+  rm -f $tempbase.0 $tempbase.1
+  trap - 1 2 13 15
+fi
 
 # Import from gnulib.
 
 gnulib_tool_options="\
  --import\
  --no-changelog\
- --aux-dir $bt/$build_aux\
- --doc-base $bt/$doc_base\
+ --aux-dir $build_aux\
+ --doc-base $doc_base\
  --lib $gnulib_name\
- --m4-base $bt/$m4_base/\
- --source-base $bt/$source_base/\
- --tests-base $bt/$tests_base\
+ --m4-base $m4_base/\
+ --source-base $source_base/\
+ --tests-base $tests_base\
  --local-dir $local_gl_dir\
  $gnulib_tool_option_extras\
 "
@@ -850,25 +901,14 @@ if test $use_libtool = 1; then
 fi
 echo "$0: $gnulib_tool $gnulib_tool_options --import ..."
 $gnulib_tool $gnulib_tool_options --import $gnulib_modules &&
-slurp $bt || exit
 
 for file in $gnulib_files; do
-  symlink_to_dir "$GNULIB_SRCDIR" $file || exit
+  symlink_to_dir "$GNULIB_SRCDIR" $file \
+    || die "failed to symlink $file"
 done
 
-
-# Import from gettext.
-with_gettext=yes
-grep '^[	 ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \
-    with_gettext=no
-
-if test $with_gettext = yes; then
-  echo "$0: (cd $bt2; ${AUTOPOINT-autopoint}) ..."
-  cp configure.ac $bt2 &&
-  (cd $bt2 && ${AUTOPOINT-autopoint} && rm configure.ac) &&
-  slurp $bt2 $bt || exit
-fi
-rm -fr $bt $bt2 || exit
+bootstrap_post_import_hook \
+  || die "bootstrap_post_import_hook failed"
 
 # Remove any dangling symlink matching "*.m4" or "*.[ch]" in some
 # gnulib-populated directories.  Such .m4 files would cause aclocal to fail.
@@ -882,37 +922,32 @@ find "$m4_base" "$source_base" \
   -depth \( -name '*.m4' -o -name '*.[ch]' \) \
   -type l -xtype l -delete > /dev/null 2>&1
 
-# Reconfigure, getting other files.
+# Invoke autoreconf with --force --install to ensure upgrades of tools
+# such as ylwrap.
+AUTORECONFFLAGS="--verbose --install --force -I $m4_base $ACLOCAL_FLAGS"
 
-# Skip autoheader if it's not needed.
-grep -E '^[	 ]*AC_CONFIG_HEADERS?\>' configure.ac >/dev/null ||
-  AUTOHEADER=true
-
-for command in \
-  libtool \
-  "${ACLOCAL-aclocal} --force -I '$m4_base' $ACLOCAL_FLAGS" \
-  "${AUTOCONF-autoconf} --force" \
-  "${AUTOHEADER-autoheader} --force" \
-  "${AUTOMAKE-automake} --add-missing --copy --force-missing"
-do
-  if test "$command" = libtool; then
-    test $use_libtool = 0 \
-      && continue
-    command="${LIBTOOLIZE-libtoolize} -c -f"
-  fi
-  echo "$0: $command ..."
-  eval "$command" || exit
-done
+# Some systems (RHEL 5) are using ancient autotools, for which the
+# --no-recursive option had not been invented.  Detect that lack and
+# omit the option when it's not supported.  FIXME in 2017: remove this
+# hack when RHEL 5 autotools are updated, or when they become irrelevant.
+case $($AUTORECONF --help) in
+  *--no-recursive*) AUTORECONFFLAGS="$AUTORECONFFLAGS --no-recursive";;
+esac
 
+# Tell autoreconf not to invoke autopoint or libtoolize; they were run above.
+echo "running: AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS"
+AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS \
+  || die "autoreconf failed"
 
 # Get some extra files from gnulib, overriding existing files.
 for file in $gnulib_extra_files; do
   case $file in
   */INSTALL) dst=INSTALL;;
-  build-aux/*) dst=$build_aux/`expr "$file" : 'build-aux/\(.*\)'`;;
+  build-aux/*) dst=$build_aux/${file#build-aux/};;
   *) dst=$file;;
   esac
-  symlink_to_dir "$GNULIB_SRCDIR" $file $dst || exit
+  symlink_to_dir "$GNULIB_SRCDIR" $file $dst \
+    || die "failed to symlink $file"
 done
 
 if test $with_gettext = yes; then
@@ -928,7 +963,19 @@ if test $with_gettext = yes; then
       a\
           '"$XGETTEXT_OPTIONS"' $${end_of_xgettext_options+}
     }
-  ' po/Makevars.template >po/Makevars || exit 1
+  ' po/Makevars.template >po/Makevars \
+    || die 'cannot generate po/Makevars'
+
+  # If the 'gettext' module is in use, grab the latest Makefile.in.in.
+  # If only the 'gettext-h' module is in use, assume autopoint already
+  # put the correct version of this file into place.
+  case $gnulib_modules in
+  *gettext-h*) ;;
+  *gettext*)
+    cp $GNULIB_SRCDIR/build-aux/po/Makefile.in.in po/Makefile.in.in \
+      || die "cannot create po/Makefile.in.in"
+    ;;
+  esac
 
   if test -d runtime-po; then
     # Similarly for runtime-po/Makevars, but not quite the same.
@@ -942,7 +989,8 @@ if test $with_gettext = yes; then
         a\
             '"$XGETTEXT_OPTIONS_RUNTIME"' $${end_of_xgettext_options+}
       }
-    ' po/Makevars.template >runtime-po/Makevars || exit 1
+    ' po/Makevars.template >runtime-po/Makevars \
+    || die 'cannot generate runtime-po/Makevars'
 
     # Copy identical files from po to runtime-po.
     (cd po && cp -p Makefile.in.in *-quot *.header *.sed *.sin ../runtime-po)

bootstrap.conf

@@ -1,5 +1,5 @@
 # bootstrap.conf - Bootstrap configuration.
-# Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
 # Inc.
 #
 # This file is part of GNU Wget.
@@ -28,31 +28,41 @@ gnulib_modules="
 accept
 alloca
 announce-gen
+base32
 bind
 c-ctype
+c-strcase
 clock-time
 close
 connect
 fcntl
+fnmatch
 futimens
+ftello
 getaddrinfo
 getopt-gnu
 getpass-gnu
 getpeername
 getsockname
+git-version-gen
 gnupload
 ioctl
 iconv
 iconv-h
 listen
 maintainer-makefile
+mbiter
 mbtowc
+memrchr
 mkdir
+mkstemp
+mkostemp
 crypto/md5
-pipe
+crypto/sha1
 quote
 quotearg
 recv
+regex
 select
 send
 setsockopt
@@ -60,18 +70,25 @@ sigprocmask
 sigpipe
 snprintf
 socket
+spawn-pipe
 stdbool
-strcasestr
+strcase
 strerror_r-posix
+strptime
+strtok_r
+strtoll
+timegm
+tmpdir
 unlocked-io
 update-copyright
 vasprintf
+vsnprintf
 write
+xstrndup
 "
 
 gnulib_extra_files="
 	$build_aux/install-sh
-	$build_aux/missing
 	$build_aux/mdate-sh
 	$build_aux/texinfo.tex
 	$build_aux/depcomp
@@ -93,4 +110,5 @@ perl       5.5
 rsync      -
 tar        -
 xz         -
+flex       -
 "

build-aux/build_info.pl

@@ -100,7 +100,7 @@ EOC
 EOC
             }
                 print $fh <<EOC;
-#else 
+#else
   "-$name",
 #endif
 
@@ -137,3 +137,4 @@ __DATA__
 
 #include "wget.h"
 #include <stdio.h>
+#include "version.h"

build-aux/bzr-version-gen

@@ -1,57 +0,0 @@
-#!/bin/sh
-
-scriptversion=2011-08-11.08; # UTC
-
-# Copyright (C) 2010, 2011 Free Software Foundation, Inc.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# Additional permission under GNU GPL version 3 section 7
-
-
-# Written by Giuseppe Scrivano.
-
-if test -f .tarball-version
-then
-    cat .tarball-version | tr -d '\n'
-    exit 0
-fi
-
-DIRTY=""
-
-test -n "`bzr diff | tr -d '\n'`" && DIRTY="-dirty"
-
-REVNO=`bzr revno`
-
-TAG=`bzr tags -r $REVNO | cut -d' ' -f1`
-if test -z "$TAG"
-then
-    TAG=`bzr tags --sort=time -r ..$REVNO | tail -n1 | cut -d' ' -f1`
-
-    # No tags yet
-    test -z "$TAG" && TAG="unknown"
-
-    TAG=$TAG-$REVNO
-fi
-
-
-printf "%s%s" "$TAG" "$DIRTY"
-
-# Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:

cfg.mk

@@ -4,7 +4,6 @@ local-checks-to-skip = \
     sc_const_long_option \
     sc_error_message_uppercase \
     sc_file_system \
-    sc_makefile_TAB_only_indentation \
     sc_immutable_NEWS \
     sc_copyright_check \
     sc_makefile_path_separator_check \
@@ -13,17 +12,18 @@ local-checks-to-skip = \
     sc_two_space_separator_in_usage \
     sc_useless_cpp_parens \
     \
-    sc_space_tab \
     sc_prohibit_atoi_atof \
     sc_prohibit_strcmp \
-    sc_makefile_check \
     sc_cast_of_alloca_return_value \
     sc_cast_of_x_alloc_return_value \
     sc_prohibit_S_IS_definition \
     \
     sc_program_name \
-
+    sc_makefile_at_at_check \
+    sc_cast_of_argument_to_free \
 
 #SHELL=bash -x
 show-vc-list-except:
 	@$(VC_LIST_EXCEPT)
+
+VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)$$

configure.ac

@@ -1,6 +1,6 @@
 dnl Template file for GNU Autoconf
-dnl Copyright (C) 1995, 1996, 1997, 2001, 2007, 2008, 2009, 2010, 2011
-dnl Free Software Foundation, Inc.
+dnl Copyright (C) 1995, 1996, 1997, 2001, 2007, 2008, 2009, 2010, 2011, 2012,
+dnl 2013, 2014 Free Software Foundation, Inc.
 
 dnl This program is free software; you can redistribute it and/or modify
 dnl it under the terms of the GNU General Public License as published by
@@ -31,7 +31,7 @@ dnl Process this file with autoconf to produce a configure script.
 dnl
 
 AC_INIT([wget],
-  [m4_esyscmd([build-aux/bzr-version-gen])],
+        m4_esyscmd([build-aux/git-version-gen .tarball-version]),
 	[bug-wget@gnu.org])
 AC_PREREQ(2.61)
 
@@ -57,35 +57,101 @@ AC_CANONICAL_HOST
 AC_DEFINE_UNQUOTED([OS_TYPE], "$host_os",
                    [Define to be the name of the operating system.])
 
+
 dnl
-dnl Process features.
+dnl Process External Libraries
 dnl
 
-AC_ARG_WITH(ssl,
-[[  --without-ssl           disable SSL autodetection
-  --with-ssl={gnutls,openssl} specify the SSL backend.  GNU TLS is the default.]])
+dnl Libpsl: Public Suffix List checking
+AC_ARG_WITH([libpsl],
+  [AS_HELP_STRING([--without-libpsl], [disable support for libpsl cookie checking.])])
 
-AC_ARG_ENABLE(opie,
-[  --disable-opie          disable support for opie or s/key FTP login],
-ENABLE_OPIE=$enableval, ENABLE_OPIE=yes)
-test x"${ENABLE_OPIE}" = xyes && AC_DEFINE([ENABLE_OPIE], 1,
-   [Define if you want the Opie support for FTP compiled in.])
+dnl SSL: Configure SSL backend to use
+AC_ARG_WITH([ssl],
+  [AS_HELP_STRING([--with-ssl={gnutls,openssl}], [specify SSL backend. GNU TLS is the default.])])
 
-AC_ARG_ENABLE(digest,
-[  --disable-digest        disable support for HTTP digest authorization],
-ENABLE_DIGEST=$enableval, ENABLE_DIGEST=yes)
-test x"${ENABLE_DIGEST}" = xyes && AC_DEFINE([ENABLE_DIGEST], 1,
-   [Define if you want the HTTP Digest Authorization compiled in.])
 
-AC_ARG_ENABLE(ntlm,
-[  --disable-ntlm          disable support for NTLM authorization],
-[ENABLE_NTLM=$enableval], [ENABLE_NTLM=auto])
+dnl Zlib: Configure use iof zlib for compression
+AC_ARG_WITH([zlib],
+  [AS_HELP_STRING([--without-zlib], [disable zlib.])])
 
-AC_ARG_ENABLE(debug,
-[  --disable-debug         disable support for debugging output],
-ENABLE_DEBUG=$enableval, ENABLE_DEBUG=yes)
-test x"${ENABLE_DEBUG}" = xyes && AC_DEFINE([ENABLE_DEBUG], 1,
-   [Define if you want the debug output support compiled in.])
+
+dnl
+dnl Process features
+dnl
+
+dnl Opie: Support for opie s/key FTP logins
+AC_ARG_ENABLE([opie],
+  [AS_HELP_STRING([--disable-opie], [disable support for opie or s/key FTP login])],
+  [ENABLE_OPIE=$enableval],
+  [ENABLE_OPIE=yes])
+
+AS_IF([test "x$ENABLE_OPIE" = xyes],
+  [AC_DEFINE([ENABLE_OPIE], [1], [Define if you want Opie support for FTP compiled in.])],
+  []
+)
+
+
+dnl Digest: Support for HTTP Digest Authentication
+AC_ARG_ENABLE([digest],
+  [AS_HELP_STRING([--disable-digest], [disable support for HTTP digest authorization])],
+  [ENABLE_DIGEST=$enableval],
+  [ENABLE_DIGEST=yes])
+
+AS_IF([test "x$ENABLE_DIGEST" = xyes],
+  [AC_DEFINE([ENABLE_DIGEST], [1], [Define if you want the HTTP Digest Authorization compiled in.])],
+  []
+)
+
+
+dnl NTLM: Support for HTTP NTLM Authentication
+AC_ARG_ENABLE([ntlm],
+  [AS_HELP_STRING([--disable-ntlm], [disable support for NTLM authorization])],
+  [ENABLE_NTLM=$enableval],
+  [ENABLE_NTLM=auto]
+)
+
+
+dnl Debug: Support for printing debugging output
+AC_ARG_ENABLE([debug],
+  [AS_HELP_STRING([--disable-debug], [disable support for debugging output])],
+  [ENABLE_DEBUG=$enableval],
+  [ENABLE_DEBUG=yes])
+
+AS_IF([test "x$ENABLE_DEBUG" = xyes],
+  [AC_DEFINE([ENABLE_DEBUG], [1], [Define if you want the debug output support compiled in.])],
+  []
+)
+
+dnl Valgrind-tests: Should test suite be run under valgrind?
+AC_ARG_ENABLE(valgrind-tests,
+  [AS_HELP_STRING([--enable-valgrind-tests], [enable using Valgrind for tests])],
+  [ENABLE_VALGRIND=$enableval],
+  [ENABLE_VALGRIND=no])
+
+AS_IF([test "x$ENABLE_VALGRIND" != xno], [
+  AC_CHECK_PROG(HAVE_VALGRIND, valgrind, yes, no)
+  AS_IF([test "x$HAVE_VALGRIND" = xyes], [
+    VALGRIND_TESTS="1"
+    AC_SUBST(VALGRIND_TESTS)
+    VALGRIND_INFO="Test suite will be run under Valgrind"
+  ], [
+    VALGRIND_INFO="Valgrind not found"
+  ])
+], [
+  VALGRIND_INFO="Valgrind testing not enabled"
+])
+
+AC_ARG_ENABLE(assert,
+  [AS_HELP_STRING([--enable-assert], [enable assertions in code base])],
+  [ENABLE_ASSERTION=$enableval],
+  [ENABLE_ASSERTION=no]
+)
+
+AS_IF([test "x$ENABLE_ASSERTION" != xyes], [
+  CFLAGS="-DNDEBUG $CFLAGS"
+  ],
+  [])
 
 dnl
 dnl Find the compiler
@@ -101,6 +167,12 @@ AC_AIX
 
 gl_EARLY
 
+dnl
+dnl Find python3
+dnl
+AM_PATH_PYTHON([3.0],,[:])
+AM_CONDITIONAL([HAVE_PYTHON3], [test "$PYTHON" != :])
+
 dnl
 dnl Gettext
 dnl
@@ -118,9 +190,11 @@ dnl   if compiler is gcc, then
 dnl     use -O2 and some warning flags
 dnl   else
 dnl     use os-specific flags or -O
+dnl
+dnl For these tests, the system considers clang to be gcc
 if test -n "$auto_cflags"; then
   if test -n "$GCC"; then
-    CFLAGS="$CFLAGS -O2 -Wall"
+    CFLAGS="$CFLAGS -O2 -Wall -Wextra"
   else
     case "$host_os" in
       *hpux*)  CFLAGS="$CFLAGS +O3"                      ;;
@@ -139,8 +213,7 @@ AC_C_VOLATILE
 
 dnl Check for basic headers, even though we expect them to exist and
 dnl #include them unconditionally in the code.  Their detection is
-dnl still needed because test programs used by Autoconf macros check
-dnl for STDC_HEADERS, HAVE_SYS_TYPES_H, etc. before using them.
+dnl still needed because test programs used by Autoconf macros check.
 dnl Without the checks they will fail to be included in test programs,
 dnl which will subsequently fail.
 AC_HEADER_STDC
@@ -181,9 +254,7 @@ AC_CHECK_TYPES([uint32_t, uintptr_t, intptr_t, int64_t])
 AC_CHECK_TYPES(sig_atomic_t, [], [], [
 #include <stdio.h>
 #include <sys/types.h>
-#if HAVE_INTTYPES_H
-# include <inttypes.h>
-#endif
+#include <inttypes.h>
 #include <signal.h>
 ])
 
@@ -195,9 +266,9 @@ dnl Checks for library functions.
 dnl
 AC_FUNC_MMAP
 AC_FUNC_FSEEKO
-AC_CHECK_FUNCS(strptime timegm vsnprintf vasprintf drand48)
+AC_CHECK_FUNCS(strptime timegm vsnprintf vasprintf drand48 pathconf)
 AC_CHECK_FUNCS(strtoll usleep ftello sigblock sigsetjmp memrchr wcwidth mbtowc)
-AC_CHECK_FUNCS(sleep symlink)
+AC_CHECK_FUNCS(sleep symlink utime strlcpy random)
 
 if test x"$ENABLE_OPIE" = xyes; then
   AC_LIBOBJ([ftp-opie])
@@ -234,10 +305,43 @@ dnl
 dnl Checks for libraries.
 dnl
 
-AS_IF([test x"$with_ssl" = xopenssl], [
-    dnl some versions of openssl use zlib compression
-    AC_CHECK_LIB(z, compress)
+PKG_PROG_PKG_CONFIG
 
+AS_IF([test "x$with_libpsl" != xno], [
+  PKG_CHECK_MODULES([LIBPSL], libpsl, [
+    with_libpsl=yes
+    # correct $LIBPSL_LIBS (in libpsl <= 0.6.0)
+    AS_IF([test "x$LIBPSL_LIBS" = "x-llibpsl "], [LIBPSL_LIBS="-lpsl"])
+    LIBS="$LIBPSL_LIBS $LIBS"
+    CFLAGS="$LIBPSL_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])
+  ], [
+    AC_SEARCH_LIBS(psl_builtin, psl,
+      [with_libpsl=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])],
+      [with_libpsl=no;  AC_MSG_WARN(*** libpsl was not found. Fallback to builtin cookie checking.)])
+  ])
+])
+
+AS_IF([test x"$with_zlib" != xno], [
+  with_zlib=yes
+  PKG_CHECK_MODULES([ZLIB], zlib, [
+    LIBS="$ZLIB_LIBS $LIBS"
+    CFLAGS="$ZLIB_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBZ], [1], [Define if using zlib.])
+  ], [
+    AC_CHECK_LIB(z, compress)
+  ])
+])
+
+AS_IF([test x"$with_ssl" = xopenssl], [
+  PKG_CHECK_MODULES([OPENSSL], [openssl], [
+    AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+    AC_LIBOBJ([openssl])
+    LIBS="$OPENSSL_LIBS $LIBS"
+    CFLAGS="$OPENSSL_CFLAGS -DHAVE_LIBSSL $CFLAGS"
+    LIBSSL=" " # ntlm check below wants this
+    AC_DEFINE([HAVE_LIBSSL], [1], [Define if using openssl.])
+  ], [
     dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
     dnl doesn't record its dependency on libdl, so we need to make sure
     dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
@@ -246,17 +350,23 @@ AS_IF([test x"$with_ssl" = xopenssl], [
       AC_CHECK_LIB(dl, shl_load)
     ])
 
+    ssl_found=no
     case $host_os in
       *mingw32* )
         dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
-        
+
         AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
         if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
         then
-          AC_CHECK_LIB(ssl32, SSL_connect,,
-                    AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
-          AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
-          
+          AC_CHECK_LIB(ssl32, SSL_connect, [
+            ssl_found=yes
+            AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
+            AC_LIBOBJ([openssl])
+            LIBS="${LIBS} -lssl32"
+            AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
+          ],
+          AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
+
         else
           LIBS+=' -lgdi32'
           dnl fallback and test static libs
@@ -265,72 +375,97 @@ AS_IF([test x"$with_ssl" = xopenssl], [
       ;;
     esac
 
-    dnl Now actually check for -lssl
-    AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
-  #include <openssl/ssl.h>
-  #include <openssl/x509.h>
-  #include <openssl/err.h>
-  #include <openssl/rand.h>
-  #include <openssl/des.h>
-  #include <openssl/md4.h>
-  #include <openssl/md5.h>
-    ], [SSL_library_init ()])
-    if test x"$LIBSSL" != x
-    then
-      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
-      AC_LIBOBJ([openssl])
-      LIBS="$LIBSSL $LIBS"
-    elif test x"$with_ssl" != x
-    then
-      AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
-    fi
-
-    AC_LIBOBJ([openssl])
+    AS_IF([test x$ssl_found != xyes], [
+      dnl Now actually check for -lssl if it wasn't already found
+      AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+#include <openssl/err.h>
+#include <openssl/rand.h>
+#include <openssl/des.h>
+#include <openssl/md4.h>
+#include <openssl/md5.h>
+      ], [SSL_library_init ()])
+      if test x"$LIBSSL" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+        AC_LIBOBJ([openssl])
+        LIBS="$LIBSSL $LIBS"
+      elif test x"$with_ssl" != x
+      then
+        AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
+      fi
+    ])
+  ])
 ], [
-  # --with-ssl is not gnutls: check if it's no
+  # --with-ssl is not openssl: check if it's no
   AS_IF([test x"$with_ssl" != xno], [
-    dnl Now actually check for -lssl
+    dnl default is -lgnutls
+    with_ssl=gnutls
 
-    AC_CHECK_LIB(z, compress)
-    AC_CHECK_LIB(gpg-error, gpg_err_init)
-    AC_CHECK_LIB(gcrypt, gcry_control)
-
-  dnl Now actually check for -lssl
-  AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
+    dnl Now actually check for -lgnutls
+    PKG_CHECK_MODULES([GNUTLS], [gnutls], [
+      AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+      AC_LIBOBJ([gnutls])
+      LIBS="$GNUTLS_LIBS $LIBS"
+      CFLAGS="$GNUTLS_CFLAGS -DHAVE_LIBGNUTLS $CFLAGS"
+      AC_DEFINE([HAVE_LIBGNUTLS], [1], [Define if using gnutls.])
+    ], [
+      AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
 #include <gnutls/gnutls.h>
-  ], [gnutls_global_init()])
-  if test x"$LIBGNUTLS" != x
-  then
-    AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
-    AC_LIBOBJ([gnutls])
-    LIBS="$LIBGNUTLS $LIBS"
-  else
-    AC_MSG_ERROR([--with-ssl was given, but GNUTLS is not available.])
-  fi
+      ], [gnutls_global_init()])
+      if test x"$LIBGNUTLS" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+        AC_LIBOBJ([gnutls])
+        LIBS="$LIBGNUTLS $LIBS"
+      else
+        AC_MSG_ERROR([GnuTLS has not been found. Use --with-ssl=openssl if you explicitly want OpenSSL.])
+      fi
+    ])
 
-  AC_CHECK_FUNCS(gnutls_priority_set_direct)
-  ]) # endif: --with-ssl == no?
+    AC_CHECK_FUNCS(gnutls_priority_set_direct)
+  ]) # endif: --with-ssl != no?
 ]) # endif: --with-ssl == openssl?
 
-
 dnl Enable NTLM if requested and if SSL is available.
-if test x"$LIBSSL" != x
+if test x"$LIBSSL" != x || test "$ac_cv_lib_ssl32_SSL_connect" = yes
 then
   if test x"$ENABLE_NTLM" != xno
   then
+    ENABLE_NTLM=yes
     AC_DEFINE([ENABLE_NTLM], 1,
      [Define if you want the NTLM authorization support compiled in.])
     AC_LIBOBJ([http-ntlm])
   fi
 else
-  dnl If SSL is unavailable and the user explicitly requested NTLM,
-  dnl abort.
-  if test x"$ENABLE_NTLM" = xyes
+  AC_CHECK_LIB(nettle, nettle_md4_init, [HAVE_NETTLE=yes], [HAVE_NETTLE=no; AC_MSG_WARN(*** libnettle was not found. You will not be able to use NTLM)])
+
+  if test x"$HAVE_NETTLE" = xyes
   then
-    AC_MSG_ERROR([NTLM authorization requested and OpenSSL not found; aborting])
+    AC_SUBST(NETTLE_LIBS, "-lnettle")
+    AC_DEFINE([HAVE_NETTLE], [1], [Use libnettle])
+    if test x"$ENABLE_NTLM" != xno
+    then
+      ENABLE_NTLM=yes
+      AC_DEFINE([ENABLE_NTLM], 1,
+       [Define if you want the NTLM authorization support compiled in.])
+      AC_LIBOBJ([http-ntlm])
+      LIBS="$NETTLE_LIBS $LIBS"
+    fi
+  else
+    dnl If SSL is unavailable and the user explicitly requested NTLM,
+    dnl abort.
+    if test x"$ENABLE_NTLM" = xyes
+    then
+      AC_MSG_ERROR([NTLM authorization requested and SSL not enabled; aborting])
+    fi
   fi
 fi
 
+
 dnl **********************************************************************
 dnl Checks for IPv6
 dnl **********************************************************************
@@ -504,6 +639,59 @@ if test "X$iri" != "Xno"; then
   fi
 fi
 
+dnl
+dnl Check for UUID
+dnl
+
+AC_ARG_WITH(libuuid, AC_HELP_STRING([--without-libuuid],
+                                    [Generate UUIDs for WARC files via libuuid]))
+
+AS_IF([test "X$with_libuuid" != "Xno"],[
+  PKG_CHECK_MODULES([UUID], uuid, [
+    LIBS="$UUID_LIBS $LIBS"
+    CFLAGS="$UUID_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBUUID], [1], [Define if using libuuid.])
+  ], [
+    AC_CHECK_HEADER(uuid.h,
+                    AC_CHECK_FUNC(uuid, uuid_create,
+                      [AC_DEFINE([HAVE_UUID_CREATE], 1,
+                                 [Define if uuid_create is available.])
+                      ])
+    )
+
+    AC_CHECK_HEADER(uuid/uuid.h,
+                    AC_CHECK_LIB(uuid, uuid_generate,
+                      [LIBS="${LIBS} -luuid"
+                       AC_DEFINE([HAVE_LIBUUID], 1,
+                                 [Define if libuuid is available.])
+                      ])
+    )
+  ])
+])
+
+dnl
+dnl Check for PCRE
+dnl
+
+AC_ARG_ENABLE(pcre, AC_HELP_STRING([--disable-pcre],
+                                   [Disable PCRE style regular expressions]))
+
+AS_IF([test "X$enable_pcre" != "Xno"],[
+  PKG_CHECK_MODULES([PCRE], libpcre, [
+    LIBS="$PCRE_LIBS $LIBS"
+    CFLAGS="$PCRE_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBPCRE], [1], [Define if using libpcre.])
+  ], [
+    AC_CHECK_HEADER(pcre.h,
+                    AC_CHECK_LIB(pcre, pcre_compile,
+                      [LIBS="${LIBS} -lpcre"
+                       AC_DEFINE([HAVE_LIBPCRE], 1,
+                                 [Define if libpcre is available.])
+                      ])
+    )
+  ])
+])
+
 
 dnl Needed by src/Makefile.am
 AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
@@ -513,7 +701,27 @@ dnl
 dnl Create output
 dnl
 AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
-                 po/Makefile.in tests/Makefile tests/WgetTest.pm
-                 lib/Makefile])
+                 po/Makefile.in tests/Makefile
+                 lib/Makefile testenv/Makefile])
 AC_CONFIG_HEADERS([src/config.h])
 AC_OUTPUT
+
+AC_MSG_NOTICE([Summary of build options:
+
+  Version:           $PACKAGE_VERSION
+  Host OS:           $host_os
+  Install prefix:    $prefix
+  Compiler:          $CC
+  CFlags:            $CFLAGS $CPPFLAGS
+  LDFlags:           $LDFLAGS
+  Libs:              $LIBS
+  SSL:               $with_ssl
+  Zlib:              $with_zlib
+  PSL:               $with_libpsl
+  Digest:            $ENABLE_DIGEST
+  NTLM:              $ENABLE_NTLM
+  OPIE:              $ENABLE_OPIE
+  Debugging:         $ENABLE_DEBUG
+  Assertions:        $ENABLE_ASSERTION
+  Valgrind:          $VALGRIND_INFO
+])

contrib/tsocked-wget

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Script that executes wget using a socks proxy if the environment variable
+# socks_proxy is set.
+#
+# The socks_proxy variable shall have one of the forms:
+#   socks://username:password@host:port
+#   socks4://username:password@host:port
+#   socks5://username:password@host:port
+# with username, password and port fields being optional
+#
+# As socksification applies to the whole process, domains defined in the
+# no_proxy setting are *not* excluded.
+#
+
+# Requisites: bash (version 3.2.3 or later), tsocks and wget
+
+####################################
+#  Copyright (C) 2014 Ángel González
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+#
+
+set -eu
+
+# Binary to be executed. Use an absolute path if installing this script as 'wget'
+WGET=wget
+
+if [ -z "${socks_proxy:-}" ]; then
+exec "$WGET" "$@"
+fi
+
+CONFIG=""
+
+if [[ "${socks_proxy}" =~ ^socks[45]?:// ]]; then
+ if [[ "${socks_proxy:5:1}" != ":" ]]; then
+  CONFIG+="server_type = ${socks_proxy:5:1}"
+  socks_proxy="${socks_proxy:9}"
+ else
+  socks_proxy="${socks_proxy:8}"
+ fi
+elif [[ "${socks_proxy}" =~ ^[[:alnum:]]*:// ]]; then
+ echo "Bad value specified for socks_proxy: $socks_proxy" >&2
+ exit 2
+fi
+
+if [[ "${socks_proxy}" =~ ^([^@:]*)(:([^@]*))?@ ]]; then
+ unset TSOCKS_USERNAME
+ CONFIG+="
+   default_user = ${BASH_REMATCH[1]}"
+
+ if [ ! -z "${BASH_REMATCH[3]}" ]; then
+  unset TSOCKS_PASSWORD
+  CONFIG+="
+   default_pass = ${BASH_REMATCH[3]}"
+ fi
+ socks_proxy="${socks_proxy:${#BASH_REMATCH[0]}}"
+fi
+
+
+# Get rid of trailing slashes
+if [[ "${socks_proxy}" =~ ^([^/]*)/ ]]; then
+ socks_proxy="${socks_proxy:0:${#BASH_REMATCH[1]}}"
+fi
+
+if [[ "${socks_proxy}" =~ :([0-9]+)$ ]]; then
+  CONFIG+="
+   server_port = ${BASH_REMATCH[1]}"
+  socks_proxy=${socks_proxy:0:${#socks_proxy} - ${#BASH_REMATCH[0]}}
+fi
+
+CONFIG+="
+   server = ${socks_proxy}"
+
+TSOCKS_CONF_FILE=<(echo "$CONFIG") exec tsocks "$WGET" --no-proxy "$@"

doc/ChangeLog

@@ -1,3 +1,159 @@
+2014-11-06  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi: added description for --crl-file
+
+2014-10-28  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am: Replace $FOO with @VAR@.
+
+2014-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi (symbolic links): Update documentation of retr-symlinks to
+	reflect the new default. Add warning about potential security issues with
+	--retr-symlinks=yes.
+
+2014-10-16  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi (Download Options): update --secure-protocol description
+
+2014-08-03  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Download Options): Fix texinfo warning.
+
+2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am: Remove @VAR@ with $FOO.
+
+	* fdl.texi: Remove terminating empty lines.
+
+2014-06-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* wget.texi: Remove trailing whitespaces.
+
+	* fdl.texi: Copy from gnulib.
+
+2014-05-24  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi: Document noscroll parameter to progress=bar
+
+2014-05-01  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi: Add documentation for --show-progress
+
+2014-02-10  Yousong Zhou  <yszhou4tech@gmail.com>
+
+	* wget.texi: Add documentation for --start-pos.
+
+2013-12-29  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* wget.texi: Update to GFDL 1.3.
+
+2013-12-22  Mike Frysinger  <vapier@gentoo.org>
+
+	* sample.wgetrc: add links to the manual.
+
+2013-10-06  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi: add/explain quoting of wildcard patterns
+
+2013-09-04  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* sample.wgetrc: added "secureprotocol" example
+	* wget.texi (HTTPS (SSL/TLS) Options): Document PFS.
+
+2013-08-23  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* sample.wgetrc: added "httpsonly" example
+
+2013-08-22  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi: added description for --https-only
+
+2013-08-13  Hrvoje Niksic  <hniksic@xemacs.org>
+
+	* wget.texi (Download Options): Fix misspelling.
+
+2013-07-09  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Download Options): Add documentation for --backups.
+	(Wgetrc Commands): Add documentation for backups.
+	Reported by: Tomas Hozza <thozza@redhat.com>.
+
+2013-07-11  Tomas Hozza  <thozza@redhat.com>
+
+	* wget.texi: Document --regex-type and --preserve-permissions.
+
+2013-06-17  Dave Reisner  <dreisner@archlinux.org> (tiny change)
+
+	* texi2pod.pl: Fix formatting error that causes build to fail with
+	Perl 5.18
+
+2013-06-17  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi (POST): Explain the new redirection rules.
+	* wget.texi (Other HTTP Methods): Same.
+	* wget.texi (body-data): Fix typo in description.
+
+2013-05-10  Darshit Shah <darnir@gmail.com>  (tiny change)
+
+	* wget.texi (No of tries): Fix typo to make it clear that --tries
+	option sets number of tries not retries.
+	Reported by: Hauke Hoffman <haukebjoernhoffman@googlemail.com>
+
+2013-04-28  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Types of Files...): Document --accept-regex,
+	--reject-regex.
+	Reported by: Tomas Hozza <thozza@redhat.com>.
+
+2013-04-14  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Download Options): Document "mega" dot style.
+
+2013-04-05  Darshit Shah <darnir@gmail.com>
+
+	* doc/wget.texi: Fix ambiguous wording in --post-data section. Make it
+	clear that wget does not check for the format of the post-data.
+	* doc/wget.texi: Add documentation for --method, --body-data and
+	--body-file.
+
+2012-10-08  Stefano Lattarini <stefano.lattarini@gmail.com>  (tiny change)
+
+	docs: fix errors and warnings with Texinfo 5
+	Or rather, with the development version 4.13.90, which will
+	eventually become Texinfo 5.0.
+	* wget.texi: Use '@item' instead of '@itemx' in several places,
+	as Texinfo 5 refuses to process an '@itemx' that is not preceded
+	by an '@item'.
+
+2012-08-28  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+        * doc/wget.texi: remove -nv from --report-speed
+
+2012-08-04  mancha <mancha@mac.hush.com> (tiny change)
+
+	* wget.texi: Export ENVIRONMENT to the man page.
+
+2012-06-09  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Logging and Input File Options): Document "--report-speed".
+	(HTTPS (SSL/TLS) Options): Document WARC.
+
+	* texi2pod.pl: Revert change from 2011-08-06.
+
+2012-05-13  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Types of Files): Document --accept-regex and
+	--reject-regex.
+
+2011-10-02  Henrik Holst <henrik.holst@millistream.com> (tiny change)
+
+	* wget.texi (HTTP Options): Document option --content-on-error.
+
+2011-09-27  Karl Berry <karl@freefriends.org> (tiny change)
+
+	* wget.texi: Make dir entry consistent with others.
+
 2011-08-18  Giuseppe Scrivano  <gscrivano@gnu.org>
 
 	* texi2pod.pl: Don't assume the perl executable is under /usr/bin/.

doc/fdl.texi

@@ -5,8 +5,7 @@
 @c hence no sectioning command or @node.
 
 @display
-Copyright @copyright{} 2000, 2001, 2002, 2007, 2008, 2009, 2010, 2011
-Free Software Foundation, Inc.
+Copyright @copyright{} 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
 @uref{http://fsf.org/}
 
 Everyone is permitted to copy and distribute verbatim copies
@@ -93,16 +92,16 @@ An image format is not Transparent if used for any substantial amount
 of text.  A copy that is not ``Transparent'' is called ``Opaque''.
 
 Examples of suitable formats for Transparent copies include plain
-@sc{ascii} without markup, Texinfo input format, La@TeX{} input
-format, @acronym{SGML} or @acronym{XML} using a publicly available
-@acronym{DTD}, and standard-conforming simple @acronym{HTML},
-PostScript or @acronym{PDF} designed for human modification.  Examples
-of transparent image formats include @acronym{PNG}, @acronym{XCF} and
-@acronym{JPG}.  Opaque formats include proprietary formats that can be
-read and edited only by proprietary word processors, @acronym{SGML} or
-@acronym{XML} for which the @acronym{DTD} and/or processing tools are
-not generally available, and the machine-generated @acronym{HTML},
-PostScript or @acronym{PDF} produced by some word processors for
+ASCII without markup, Texinfo input format, La@TeX{} input
+format, SGML or XML using a publicly available
+DTD, and standard-conforming simple HTML,
+PostScript or PDF designed for human modification.  Examples
+of transparent image formats include PNG, XCF and
+JPG@.  Opaque formats include proprietary formats that can be
+read and edited only by proprietary word processors, SGML or
+XML for which the DTD and/or processing tools are
+not generally available, and the machine-generated HTML,
+PostScript or PDF produced by some word processors for
 output purposes only.
 
 The ``Title Page'' means, for a printed book, the title page itself,
@@ -482,7 +481,7 @@ license notices just after the title page:
 @end smallexample
 
 If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts,
-replace the ``with@dots{}Texts.'' line with this:
+replace the ``with@dots{}Texts.''@: line with this:
 
 @smallexample
 @group
@@ -504,4 +503,3 @@ to permit their use in free software.
 @c Local Variables:
 @c ispell-local-pdict: "ispell-dict"
 @c End:
-

doc/sample.wgetrc

@@ -5,7 +5,10 @@
 ## You can use this file to change the default behaviour of wget or to
 ## avoid having to type many many command-line options. This file does
 ## not contain a comprehensive list of commands -- look at the manual
-## to find out what you can put into this file.
+## to find out what you can put into this file. You can find this here:
+##   $ info wget.info 'Startup File'
+## Or online here:
+##   https://www.gnu.org/software/wget/manual/wget.html#Startup-File
 ##
 ## Wget initialization file can reside in /usr/local/etc/wgetrc
 ## (global, for all users) or $HOME/.wgetrc (for a single user).
@@ -123,3 +126,9 @@
 
 # Force the default remote server encoding
 #remoteencoding = UTF-8
+
+# Turn on to prevent following non-HTTPS links when in recursive mode
+#httpsonly = off
+
+# Tune HTTPS security (auto, SSLv2, SSLv3, TLSv1, PFS)
+#secureprotocol = auto

doc/texi2pod.pl

@@ -1,6 +1,7 @@
 #! /usr/bin/env perl
 
-#   Copyright (C) 1999, 2000, 2001, 2003, 2010 Free Software Foundation, Inc.
+#   Copyright (C) 1999, 2000, 2001, 2003, 2007, 2009, 2010, 2011 Free
+#   Software Foundation, Inc.
 
 # This file is part of GCC.
 
@@ -15,14 +16,15 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with GCC; see the file COPYING.  If not, write to
-# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
-# Boston MA 02110-1301, USA.
+# along with GCC.  If not, see <http://www.gnu.org/licenses/>.
 
 # This does trivial (and I mean _trivial_) conversion of Texinfo
 # markup to Perl POD format.  It's intended to be used to extract
 # something suitable for a manpage from a Texinfo document.
 
+use warnings;
+BEGIN { eval { require warnings; } and warnings->import; }
+
 $output = 0;
 $skipping = 0;
 %sects = ();
@@ -36,7 +38,6 @@ $shift = "";
 $fnno = 1;
 $inf = "";
 $ibase = "";
-@ipath = ();
 
 while ($_ = shift) {
     if (/^-D(.*)$/) {
@@ -52,13 +53,6 @@ while ($_ = shift) {
 	die "flags may only contain letters, digits, hyphens, dashes and underscores\n"
 	    unless $flag =~ /^[a-zA-Z0-9_-]+$/;
 	$defs{$flag} = $value;
-    } elsif (/^-I(.*)$/) {
-	if ($1 ne "") {
-	    $flag = $1;
-	} else {
-	    $flag = shift;
-	}
-        push (@ipath, $flag);
     } elsif (/^-/) {
 	usage();
     } else {
@@ -162,8 +156,6 @@ while(<$inf>) {
 	} elsif ($ended =~ /^(?:itemize|enumerate|[fv]?table)$/) {
 	    $_ = "\n=back\n";
 	    $ic = pop @icstack;
-	} elsif ($ended eq "multitable") {
-	    $_ = "\n=back\n";
 	} else {
 	    die "unknown command \@end $ended at line $.\n";
 	}
@@ -213,18 +205,14 @@ while(<$inf>) {
 
     # Now the ones that have to be replaced by special escapes
     # (which will be turned back into text by unmunge())
-    # Replace @@ before @{ and @} in order to parse @samp{@@} correctly.
     s/&/&amp;/g;
     s/\@\@/&at;/g;
     s/\@\{/&lbrace;/g;
     s/\@\}/&rbrace;/g;
-    s/\@`\{(.)\}/&$1grave;/g;
 
-    # Inside a verbatim block, handle @var, @samp and @url specially.
+    # Inside a verbatim block, handle @var specially.
     if ($shift ne "") {
 	s/\@var\{([^\}]*)\}/<$1>/g;
-	s/\@samp\{([^\}]*)\}/"$1"/g;
-	s/\@url\{([^\}]*)\}/<$1>/g;
     }
 
     # POD doesn't interpret E<> inside a verbatim block.
@@ -243,23 +231,17 @@ while(<$inf>) {
 	$inf = gensym();
 	$file = postprocess($1);
 
-	# Try cwd and $ibase, then explicit -I paths.
-	$done = 0;
-	foreach $path ("", $ibase, @ipath) {
-	    $mypath = $file;
-	    $mypath = $path . "/" . $mypath if ($path ne "");
-	    open($inf, "<" . $mypath) and ($done = 1, last);
-	}
-	die "cannot find $file" if !$done;
+	# Try cwd and $ibase.
+	open($inf, "<" . $file)
+	    or open($inf, "<" . $ibase . "/" . $file)
+		or die "cannot open $file or $ibase/$file: $!\n";
 	next;
     };
 
-    /^\@(?:section|unnumbered|unnumberedsec|center|heading)\s+(.+)$/
+    /^\@(?:section|unnumbered|unnumberedsec|center)\s+(.+)$/
 	and $_ = "\n=head2 $1\n";
     /^\@subsection\s+(.+)$/
 	and $_ = "\n=head3 $1\n";
-    /^\@subsubsection\s+(.+)$/
-	and $_ = "\n=head4 $1\n";
 
     # Block command handlers:
     /^\@itemize(?:\s+(\@[a-z]+|\*|-))?/ and do {
@@ -268,7 +250,7 @@ while(<$inf>) {
 	if (defined $1) {
 	    $ic = $1;
 	} else {
-	    $ic = '*';
+	    $ic = '@bullet';
 	}
 	$_ = "\n=over 4\n";
 	$endw = "itemize";
@@ -286,12 +268,6 @@ while(<$inf>) {
 	$endw = "enumerate";
     };
 
-    /^\@multitable\s.*/ and do {
-	push @endwstack, $endw;
-	$endw = "multitable";
-	$_ = "\n=over 4\n";
-    };
-
     /^\@([fv]?table)\s+(\@[a-z]+)/ and do {
 	push @endwstack, $endw;
 	push @icstack, $ic;
@@ -301,7 +277,6 @@ while(<$inf>) {
 	$ic =~ s/\@(?:code|kbd)/C/;
 	$ic =~ s/\@(?:dfn|var|emph|cite|i)/I/;
 	$ic =~ s/\@(?:file)/F/;
-	$ic =~ s/\@(?:asis)//;
 	$_ = "\n=over 4\n";
     };
 
@@ -312,29 +287,14 @@ while(<$inf>) {
 	$_ = "";	# need a paragraph break
     };
 
-    /^\@item\s+(.*\S)\s*$/ and $endw eq "multitable" and do {
-	@columns = ();
-	for $column (split (/\s*\@tab\s*/, $1)) {
-	    # @strong{...} is used a @headitem work-alike
-	    $column =~ s/^\@strong{(.*)}$/$1/;
-	    push @columns, $column;
-	}
-	$_ = "\n=item ".join (" : ", @columns)."\n";
-    };
-
     /^\@itemx?\s*(.+)?$/ and do {
 	if (defined $1) {
-            if ($ic) {
-		if ($endw eq "enumerate") {
-		    $_ = "\n=item $ic $1\n";
-		    $ic =~ s/(\d+)/$1 + 1/eg;
-		} else {
-		    # Entity escapes prevent munging by the <>
-		    # processing below.
-		    $_ = "\n=item $ic\&LT;$1\&GT;\n";
-		}
+            my $thing = $1;
+            if ($ic =~ /\@asis/) {
+                $_ = "\n=item C<$thing>\n";
             } else {
-                $_ = "\n=item $1\n";
+                # Entity escapes prevent munging by the <> processing below.
+                $_ = "\n=item $ic\&LT;$thing\&GT;\n";
             }
 	} else {
 	    $_ = "\n=item $ic\n";
@@ -355,11 +315,12 @@ die "No filename or title\n" unless defined $fn && defined $tl;
 $sects{NAME} = "$fn \- $tl\n";
 $sects{FOOTNOTES} .= "=back\n" if exists $sects{FOOTNOTES};
 
-for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT FILES
-	      BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) {
+for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT EXITSTATUS
+           FILES BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) {
     if(exists $sects{$sect}) {
 	$head = $sect;
 	$head =~ s/SEEALSO/SEE ALSO/;
+	$head =~ s/EXITSTATUS/EXIT STATUS/;
 	print "=head1 $head\n\n";
 	print scalar unmunge ($sects{$sect});
 	print "\n";
@@ -391,13 +352,11 @@ sub postprocess
     s/\@r\{([^\}]*)\}/R<$1>/g;
     s/\@(?:dfn|var|emph|cite|i)\{([^\}]*)\}/I<$1>/g;
     s/\@(?:code|kbd)\{([^\}]*)\}/C<$1>/g;
-    s/\@(?:samp|strong|key|option|env|command|b)\{([^\}]*)\}/B<$1>/g;
+    s/\@(?:gccoptlist|samp|strong|key|option|env|command|b)\{([^\}]*)\}/B<$1>/g;
     s/\@sc\{([^\}]*)\}/\U$1/g;
-    s/\@acronym\{([^\}]*)\}/\U$1/g;
     s/\@file\{([^\}]*)\}/F<$1>/g;
     s/\@w\{([^\}]*)\}/S<$1>/g;
     s/\@(?:dmn|math)\{([^\}]*)\}/$1/g;
-    s/\@\///g;
 
     # keep references of the form @ref{...}, print them bold
     s/\@(?:ref)\{([^\}]*)\}/B<$1>/g;
@@ -419,9 +378,6 @@ sub postprocess
     s/\@gol//g;
     s/\@\*\s*\n?//g;
 
-    # Anchors are thrown away
-    s/\@anchor\{(?:[^\}]*)\}//g;
-
     # @uref can take one, two, or three arguments, with different
     # semantics each time.  @url and @email are just like @uref with
     # one argument, for our purposes.
@@ -429,10 +385,6 @@ sub postprocess
     s/\@uref\{([^\},]*),([^\},]*)\}/$2 (C<$1>)/g;
     s/\@uref\{([^\},]*),([^\},]*),([^\},]*)\}/$3/g;
 
-    # Handle gccoptlist here, so it can contain the above formatting
-    # commands.
-    s/\@gccoptlist\{([^\}]*)\}/B<$1>/g;
-
     # Un-escape <> at this point.
     s/&LT;/</g;
     s/&GT;/>/g;
@@ -466,7 +418,6 @@ sub unmunge
     # Replace escaped symbols with their equivalents.
     local $_ = $_[0];
 
-    s/&(.)grave;/E<$1grave>/g;
     s/&lt;/E<lt>/g;
     s/&gt;/E<gt>/g;
     s/&lbrace;/\{/g;

doc/wget.texi

@@ -20,9 +20,9 @@
 @set Wget Wget
 @c man title Wget The non-interactive network downloader.
 
-@dircategory Network Applications
+@dircategory Network applications
 @direntry
-* Wget: (wget).         The non-interactive network downloader.
+* Wget: (wget).                 Non-interactive network downloader.
 @end direntry
 
 @copying
@@ -47,11 +47,11 @@ notice identical to this one except for the removal of this paragraph
 (this paragraph not being relevant to the printed manual).
 @end ignore
 Permission is granted to copy, distribute and/or modify this document
-under the terms of the GNU Free Documentation License, Version 1.2 or
+under the terms of the GNU Free Documentation License, Version 1.3 or
 any later version published by the Free Software Foundation; with no
-Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.  A
-copy of the license is included in the section entitled ``GNU Free
-Documentation License''.
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the section entitled
+``GNU Free Documentation License''.
 @c man end
 @end copying
 
@@ -479,6 +479,9 @@ Turn off verbose without being completely quiet (use @samp{-q} for
 that), which means that error messages and basic information still get
 printed.
 
+@item --report-speed=@var{type}
+Output bandwidth as @var{type}.  The only accepted value is @samp{bits}.
+
 @cindex input-file
 @item -i @var{file}
 @itemx --input-file=@var{file}
@@ -547,10 +550,10 @@ IPs.
 
 @cindex retries
 @cindex tries
-@cindex number of retries
+@cindex number of tries
 @item -t @var{number}
 @itemx --tries=@var{number}
-Set number of retries to @var{number}.  Specify 0 or @samp{inf} for
+Set number of tries to @var{number}. Specify 0 or @samp{inf} for
 infinite retrying.  The default is to retry 20 times, with the exception
 of fatal errors like ``connection refused'' or ``not found'' (404),
 which are not retried.
@@ -627,6 +630,13 @@ Note that when @samp{-nc} is specified, files with the suffixes
 @samp{.html} or @samp{.htm} will be loaded from the local disk and
 parsed as if they had been retrieved from the Web.
 
+@cindex backing up files
+@item --backups=@var{backups}
+Before (over)writing a file, back up an existing file by adding a
+@samp{.1} suffix (@samp{_1} on VMS) to the file name.  Such backup
+files are rotated to @samp{.2}, @samp{.3}, and so on, up to
+@var{backups} (and lost beyond that).
+
 @cindex continue retrieval
 @cindex incomplete downloads
 @cindex resume download
@@ -691,6 +701,22 @@ Another instance where you'll get a garbled file if you try to use
 Note that @samp{-c} only works with @sc{ftp} servers and with @sc{http}
 servers that support the @code{Range} header.
 
+@cindex offset
+@cindex continue retrieval
+@cindex incomplete downloads
+@cindex resume download
+@cindex start position
+@item --start-pos=@var{OFFSET}
+Start downloading at zero-based position @var{OFFSET}.  Offset may be expressed
+in bytes, kilobytes with the `k' suffix, or megabytes with the `m' suffix, etc.
+
+@samp{--start-pos} has higher precedence over @samp{--continue}.  When
+@samp{--start-pos} and @samp{--continue} are both specified, wget will emit a
+warning then proceed as if @samp{--continue} was absent.
+
+Server support for continued download is required, otherwise @samp{--start-pos}
+cannot help.  See @samp{-c} for details.
+
 @cindex progress indicator
 @cindex dot style
 @item --progress=@var{type}
@@ -706,21 +732,53 @@ Use @samp{--progress=dot} to switch to the ``dot'' display.  It traces
 the retrieval by printing dots on the screen, each dot representing a
 fixed amount of downloaded data.
 
-When using the dotted retrieval, you may also set the @dfn{style} by
+The progress @var{type} can also take one or more parameters.  The parameters
+vary based on the @var{type} selected.  Parameters to @var{type} are passed by
+appending them to the type sperated by a colon (:) like this:
+@samp{--progress=@var{type}:@var{parameter1}:@var{parameter2}}.
+
+When using the dotted retrieval, you may set the @dfn{style} by
 specifying the type as @samp{dot:@var{style}}.  Different styles assign
 different meaning to one dot.  With the @code{default} style each dot
 represents 1K, there are ten dots in a cluster and 50 dots in a line.
 The @code{binary} style has a more ``computer''-like orientation---8K
 dots, 16-dots clusters and 48 dots per line (which makes for 384K
-lines).  The @code{mega} style is suitable for downloading very large
+lines).  The @code{mega} style is suitable for downloading large
 files---each dot represents 64K retrieved, there are eight dots in a
 cluster, and 48 dots on each line (so each line contains 3M).
+If @code{mega} is not enough then you can use the @code{giga}
+style---each dot represents 1M retrieved, there are eight dots in a
+cluster, and 32 dots on each line (so each line contains 32M).
+
+With @samp{--progress=bar}, there are currently two possible parameters,
+@var{force} and @var{noscroll}.
+
+When the output is not a TTY, the progress bar always falls back to ``dot'',
+even if @samp{--progress=bar} was passed to Wget during invokation. This
+behaviour can be overridden and the ``bar'' output forced by using the ``force''
+parameter as @samp{--progress=bar:force}.
+
+By default, the @samp{bar} style progress bar scroll the name of the file from
+left to right for the file being downloaded if the filename exceeds the maximum
+length allotted for its display.  In certain cases, such as with
+@samp{--progress=bar:force}, one may not want the scrolling filename in the
+progress bar.  By passing the ``noscroll'' parameter, Wget can be forced to
+display as much of the filename as possible without scrolling through it.
 
 Note that you can set the default style using the @code{progress}
 command in @file{.wgetrc}.  That setting may be overridden from the
-command line.  The exception is that, when the output is not a TTY, the
-``dot'' progress will be favored over ``bar''.  To force the bar output,
-use @samp{--progress=bar:force}.
+command line.  For example, to force the bar output without scrolling,
+use @samp{--progress=bar:force:noscroll}.
+
+@item --show-progress
+Force wget to display the progress bar in any verbosity.
+
+By default, wget only displays the progress bar in verbose mode.  One may
+however want wget to display the progress bar on screen in conjunction with
+any other verbosity modes like @samp{--no-verbose} or @samp{--quiet}.  This
+is often a desired a property when invoking wget to download several small/large
+files.  In such a case, wget could simply be invoked with this parameter to get
+a much cleaner output on the screen.
 
 @item -N
 @itemx --timestamping
@@ -729,7 +787,7 @@ Turn on time-stamping.  @xref{Time-Stamping}, for details.
 @item --no-use-server-timestamps
 Don't set the local file's timestamp by the one on the server.
 
-By default, when a file is downloaded, it's timestamps are set to
+By default, when a file is downloaded, its timestamps are set to
 match those from the remote file. This allows the use of
 @samp{--timestamping} on subsequent invocations of wget. However, it
 is sometimes useful to base the local file's timestamp on when it was
@@ -872,12 +930,13 @@ recommendation to block many unrelated users from a web site due to the
 actions of one.
 
 @cindex proxy
-@itemx --no-proxy
+@item --no-proxy
 Don't use proxies, even if the appropriate @code{*_proxy} environment
 variable is defined.
 
 @c man end
-For more information about the use of proxies with Wget, @xref{Proxies}.
+@xref{Proxies}, for more information about the use of proxies with
+Wget.
 @c man begin OPTIONS
 
 @cindex quota
@@ -973,7 +1032,7 @@ are outside the range of @sc{ascii} characters (that is, greater than
 whose encoding does not match the one used locally.
 
 @cindex IPv6
-@itemx -4
+@item -4
 @itemx --inet4-only
 @itemx -6
 @itemx --inet6-only
@@ -1219,8 +1278,8 @@ really important, do not leave them lying in those files either---edit
 the files and delete them after Wget has started the download.
 
 @iftex
-For more information about security issues with Wget, @xref{Security
-Considerations}.
+@xref{Security Considerations}, for more information about security
+issues with Wget.
 @end iftex
 
 @cindex Keep-Alive, turning off
@@ -1454,6 +1513,11 @@ like everything else. Wget does not currently support
 @code{application/x-www-form-urlencoded}. Only one of
 @samp{--post-data} and @samp{--post-file} should be specified.
 
+Please note that wget does not require the content to be of the form
+@code{key1=value1&key2=value2}, and neither does it test for it. Wget will
+simply transmit whatever data is provided to it. Most servers however expect
+the POST data to be in the above format when processing HTML Forms.
+
 Please be aware that Wget needs to know the size of the POST data in
 advance.  Therefore the argument to @code{--post-file} must be a regular
 file; specifying a FIFO or something like @file{/dev/stdin} won't work.
@@ -1464,14 +1528,15 @@ use chunked unless it knows it's talking to an HTTP/1.1 server.  And it
 can't know that until it receives a response, which in turn requires the
 request to have been completed -- a chicken-and-egg problem.
 
-Note: if Wget is redirected after the POST request is completed, it
-will not send the POST data to the redirected URL.  This is because
-URLs that process POST often respond with a redirection to a regular
-page, which does not desire or accept POST.  It is not completely
-clear that this behavior is optimal; if it doesn't work out, it might
-be changed in the future.
+Note: As of version 1.15 if Wget is redirected after the POST request is
+completed, its behaviour will depend on the response code returned by the
+server.  In case of a 301 Moved Permanently, 302 Moved Temporarily or
+307 Temporary Redirect, Wget will, in accordance with RFC2616, continue
+to send a POST request.
+In case a server wants the client to change the Request method upon
+redirection, it should send a 303 See Other response code.
 
-This example shows how to log to a server using POST and then proceed to
+This example shows how to log in to a server using POST and then proceed to
 download the desired pages, presumably only accessible to authorized
 users:
 
@@ -1494,6 +1559,37 @@ them (and neither will browsers) and the @file{cookies.txt} file will
 be empty.  In that case use @samp{--keep-session-cookies} along with
 @samp{--save-cookies} to force saving of session cookies.
 
+@cindex Other HTTP Methods
+@item --method=@var{HTTP-Method}
+For the purpose of RESTful scripting, Wget allows sending of other HTTP Methods
+without the need to explicitly set them using @samp{--header=Header-Line}.
+Wget will use whatever string is passed to it after @samp{--method} as the HTTP
+Method to the server.
+
+@item --body-data=@var{Data-String}
+@itemx --body-file=@var{Data-File}
+Must be set when additional data needs to be sent to the server along with the
+Method specified using @samp{--method}.  @samp{--body-data} sends @var{string} as
+data, whereas @samp{--body-file} sends the contents of @var{file}.  Other than that,
+they work in exactly the same way.
+
+Currently, @samp{--body-file} is @emph{not} for transmitting files as a whole.
+Wget does not currently support @code{multipart/form-data} for transmitting data;
+only @code{application/x-www-form-urlencoded}. In the future, this may be changed
+so that wget sends the @samp{--body-file} as a complete file instead of sending its
+contents to the server. Please be aware that Wget needs to know the contents of
+BODY Data in advance, and hence the argument to @samp{--body-file} should be a
+regular file. See @samp{--post-file} for a more detailed explanation.
+Only one of @samp{--body-data} and @samp{--body-file} should be specified.
+
+If Wget is redirected after the request is completed, Wget will
+suspend the current method and send a GET request till the redirection
+is completed.  This is true for all redirection response codes except
+307 Temporary Redirect which is used to explicitly specify that the
+request method should @emph{not} change.  Another exception is when
+the method is set to @code{POST}, in which case the redirection rules
+specified under @samp{--post-data} are followed.
+
 @cindex Content-Disposition
 @item --content-disposition
 
@@ -1506,6 +1602,12 @@ This option is useful for some file-downloading CGI programs that use
 @code{Content-Disposition} headers to describe what the name of a
 downloaded file should be.
 
+@cindex Content On Error
+@item --content-on-error
+
+If this is set to on, wget will not skip the content when the server responds
+with a http status code that indicates error.
+
 @cindex Trust server names
 @item --trust-server-names
 
@@ -1539,16 +1641,24 @@ without SSL support, none of these options are available.
 @cindex SSL protocol, choose
 @item --secure-protocol=@var{protocol}
 Choose the secure protocol to be used.  Legal values are @samp{auto},
-@samp{SSLv2}, @samp{SSLv3}, and @samp{TLSv1}.  If @samp{auto} is used,
-the SSL library is given the liberty of choosing the appropriate
-protocol automatically, which is achieved by sending an SSLv2 greeting
-and announcing support for SSLv3 and TLSv1.  This is the default.
+@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1}, @samp{TLSv1_2}
+and @samp{PFS}.  If @samp{auto} is used, the SSL library is given the
+liberty of choosing the appropriate protocol automatically, which is
+achieved by sending a TLSv1 greeting. This is the default.
 
-Specifying @samp{SSLv2}, @samp{SSLv3}, or @samp{TLSv1} forces the use
-of the corresponding protocol.  This is useful when talking to old and
-buggy SSL server implementations that make it hard for OpenSSL to
-choose the correct protocol version.  Fortunately, such servers are
-quite rare.
+Specifying @samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1} or
+@samp{TLSv1_2} forces the use of the corresponding protocol.  This is
+useful when talking to old and buggy SSL server implementations that
+make it hard for the underlying SSL library to choose the correct
+protocol version.  Fortunately, such servers are quite rare.
+
+Specifying @samp{PFS} enforces the use of the so-called Perfect Forward
+Security cipher suites. In short, PFS adds security by creating a one-time
+key for each SSL connection. It has a bit more CPU impact on client and server.
+We use known to be secure ciphers (e.g. no MD4) and the TLS protocol.
+
+@item --https-only
+When in recursive mode, only HTTPS links are followed.
 
 @cindex SSL certificate, check
 @item --no-check-certificate
@@ -1615,6 +1725,11 @@ it allows Wget to fetch certificates on demand.
 Without this option Wget looks for CA certificates at the
 system-specified locations, chosen at OpenSSL installation time.
 
+@cindex SSL CRL, certificate revocation list
+@item --crl-file=@var{file}
+Specifies a CRL file in @var{file}.  This is needed for certificates
+that have been revocated by the CAs.
+
 @cindex entropy, specifying source of
 @cindex randomness, specifying source of
 @item --random-file=@var{file}
@@ -1652,6 +1767,36 @@ not used), EGD is never contacted.  EGD is not needed on modern Unix
 systems that support @file{/dev/random}.
 @end table
 
+@cindex WARC
+@table @samp
+@item --warc-file=@var{file}
+Use @var{file} as the destination WARC file.
+
+@item --warc-header=@var{string}
+Use @var{string} into as the warcinfo record.
+
+@item --warc-max-size=@var{size}
+Set the maximum size of the WARC files to @var{size}.
+
+@item --warc-cdx
+Write CDX index files.
+
+@item --warc-dedup=@var{file}
+Do not store records listed in this CDX file.
+
+@item --no-warc-compression
+Do not compress WARC files with GZIP.
+
+@item --no-warc-digests
+Do not calculate SHA1 digests.
+
+@item --no-warc-keep-log
+Do not store the log file in a WARC record.
+
+@item --warc-tempdir=@var{dir}
+Specify the location for temporary files created by the WARC writer.
+@end table
+
 @node FTP Options, Recursive Retrieval Options, HTTPS (SSL/TLS) Options, Invoking
 @section FTP Options
 
@@ -1675,8 +1820,8 @@ really important, do not leave them lying in those files either---edit
 the files and delete them after Wget has started the download.
 
 @iftex
-For more information about security issues with Wget, @xref{Security
-Considerations}.
+@xref{Security Considerations}, for more information about security
+issues with Wget.
 @end iftex
 
 @cindex .listing files, removing
@@ -1737,19 +1882,24 @@ in some rare firewall configurations, active FTP actually works when
 passive FTP doesn't.  If you suspect this to be the case, use this
 option, or set @code{passive_ftp=off} in your init file.
 
+@cindex file permissions
+@item --preserve-permissions
+Preserve remote file permissions instead of permissions set by umask.
+
 @cindex symbolic links, retrieving
 @item --retr-symlinks
-Usually, when retrieving @sc{ftp} directories recursively and a symbolic
-link is encountered, the linked-to file is not downloaded.  Instead, a
-matching symbolic link is created on the local filesystem.  The
-pointed-to file will not be downloaded unless this recursive retrieval
-would have encountered it separately and downloaded it anyway.
+By default, when retrieving @sc{ftp} directories recursively and a symbolic link
+is encountered, the symbolic link is traversed and the pointed-to files are
+retrieved.  Currently, Wget does not traverse symbolic links to directories to
+download them recursively, though this feature may be added in the future.
 
-When @samp{--retr-symlinks} is specified, however, symbolic links are
-traversed and the pointed-to files are retrieved.  At this time, this
-option does not cause Wget to traverse symlinks to directories and
-recurse through them, but in the future it should be enhanced to do
-this.
+When @samp{--retr-symlinks=no} is specified, the linked-to file is not
+downloaded.  Instead, a matching symbolic link is created on the local
+filesystem.  The pointed-to file will not be retrieved unless this recursive
+retrieval would have encountered it separately and downloaded it anyway.  This
+option poses a security risk where a malicious FTP Server may cause Wget to
+write to files outside of the intended directories through a specially crafted
+@sc{.listing} file.
 
 Note that when retrieving a file (not a directory) because it was
 specified on the command-line, rather than because it was recursed to,
@@ -1973,6 +2123,17 @@ accept or reject (@pxref{Types of Files}). Note that if
 any of the wildcard characters, @samp{*}, @samp{?}, @samp{[} or
 @samp{]}, appear in an element of @var{acclist} or @var{rejlist},
 it will be treated as a pattern, rather than a suffix.
+In this case, you have to enclose the pattern into quotes to prevent
+your shell from expanding it, like in @samp{-A "*.mp3"} or @samp{-A '*.mp3'}.
+
+@item --accept-regex @var{urlregex}
+@itemx --reject-regex @var{urlregex}
+Specify a regular expression to accept or reject the complete URL.
+
+@item --regex-type @var{regextype}
+Specify the regular expression type.  Possible types are @samp{posix} or
+@samp{pcre}.  Note that to be able to use @samp{pcre} type, wget has to be
+compiled with libpcre support.
 
 @item -D @var{domain-list}
 @itemx --domains=@var{domain-list}
@@ -2021,8 +2182,10 @@ dedicated @samp{--page-requisites} option.
 Ignore case when matching files and directories.  This influences the
 behavior of -R, -A, -I, and -X options, as well as globbing
 implemented when downloading from FTP sites.  For example, with this
-option, @samp{-A *.txt} will match @samp{file1.txt}, but also
+option, @samp{-A "*.txt"} will match @samp{file1.txt}, but also
 @samp{file2.TXT}, @samp{file3.TxT}, and so on.
+The quotes in the example are to prevent the shell from expanding the
+pattern.
 
 @item -H
 @itemx --span-hosts
@@ -2278,6 +2441,8 @@ in @file{.wgetrc}.
 @item -A @var{acclist}
 @itemx --accept @var{acclist}
 @itemx accept = @var{acclist}
+@itemx --accept-regex @var{urlregex}
+@itemx accept-regex = @var{urlregex}
 The argument to @samp{--accept} option is a list of file suffixes or
 patterns that Wget will download during recursive retrieval.  A suffix
 is the ending part of a file, and consists of ``normal'' letters,
@@ -2294,6 +2459,9 @@ a description of how pattern matching works.
 Of course, any number of suffixes and patterns can be combined into a
 comma-separated list, and given as an argument to @samp{-A}.
 
+The argument to @samp{--accept-regex} option is a regular expression which
+is matched against the complete URL.
+
 @cindex reject wildcards
 @cindex reject suffixes
 @cindex wildcards, reject
@@ -2301,6 +2469,8 @@ comma-separated list, and given as an argument to @samp{-A}.
 @item -R @var{rejlist}
 @itemx --reject @var{rejlist}
 @itemx reject = @var{rejlist}
+@itemx --reject-regex @var{urlregex}
+@itemx reject-regex = @var{urlregex}
 The @samp{--reject} option works the same way as @samp{--accept}, only
 its logic is the reverse; Wget will download all files @emph{except} the
 ones matching the suffixes (or patterns) in the list.
@@ -2312,6 +2482,9 @@ Analogously, to download all files except the ones beginning with
 expansion by the shell.
 @end table
 
+The argument to @samp{--accept-regex} option is a regular expression which
+is matched against the complete URL.
+
 @noindent
 The @samp{-A} and @samp{-R} options may be combined to achieve even
 better fine-tuning of which files to retrieve.  E.g. @samp{wget -A
@@ -2780,9 +2953,11 @@ enables it).
 Enable/disable saving pre-converted files with the suffix
 @samp{.orig}---the same as @samp{-K} (which enables it).
 
-@c @item backups = @var{number}
-@c #### Document me!
-@c
+@item backups = @var{number}
+Use up to @var{number} backups for a file.  Backups are rotated by
+adding an incremental counter that starts at @samp{1}.  The default is
+@samp{0}.
+
 @item base = @var{string}
 Consider relative @sc{url}s in input files (specified via the
 @samp{input} command or the @samp{--input-file}/@samp{-i} option,
@@ -3044,7 +3219,7 @@ display properly---the same as @samp{-p}.
 Change setting of passive @sc{ftp}, equivalent to the
 @samp{--passive-ftp} option.
 
-@itemx password = @var{string}
+@item password = @var{string}
 Specify password @var{string} for both @sc{ftp} and @sc{http} file retrieval. 
 This command can be overridden using the @samp{ftp_password} and 
 @samp{http_password} command for @sc{ftp} and @sc{http} respectively.
@@ -3526,34 +3701,36 @@ internal networks from the rest of Internet.  In order to obtain
 information from the Web, their users connect and retrieve remote data
 using an authorized proxy.
 
+@c man begin ENVIRONMENT
 Wget supports proxies for both @sc{http} and @sc{ftp} retrievals.  The
 standard way to specify proxy location, which Wget recognizes, is using
 the following environment variables:
 
-@table @code
+@table @env
 @item http_proxy
 @itemx https_proxy
-If set, the @code{http_proxy} and @code{https_proxy} variables should
+If set, the @env{http_proxy} and @env{https_proxy} variables should
 contain the @sc{url}s of the proxies for @sc{http} and @sc{https}
 connections respectively.
 
 @item ftp_proxy
 This variable should contain the @sc{url} of the proxy for @sc{ftp}
-connections.  It is quite common that @code{http_proxy} and
-@code{ftp_proxy} are set to the same @sc{url}.
+connections.  It is quite common that @env{http_proxy} and
+@env{ftp_proxy} are set to the same @sc{url}.
 
 @item no_proxy
 This variable should contain a comma-separated list of domain extensions
 proxy should @emph{not} be used for.  For instance, if the value of
-@code{no_proxy} is @samp{.mit.edu}, proxy will not be used to retrieve
+@env{no_proxy} is @samp{.mit.edu}, proxy will not be used to retrieve
 documents from MIT.
 @end table
+@c man end
 
 In addition to the environment variables, proxy location and settings
 may be specified from within Wget itself.
 
 @table @samp
-@itemx --no-proxy
+@item --no-proxy
 @itemx proxy = on/off
 This option and the corresponding command may be used to suppress the
 use of proxy, even if the appropriate environment variables are set.
@@ -4265,7 +4442,7 @@ subscribers of the Wget mailing list.
 @appendix Copying this manual
   
 @menu
-* GNU Free Documentation License::  Licnse for copying this manual.
+* GNU Free Documentation License::  License for copying this manual.
 @end menu
 
 @node GNU Free Documentation License,  , Copying this manual, Copying this manual

lib/Makefile.am

@@ -1,18 +0,0 @@
-# GNU Wget
-
-# Copyright (C) 2010, 2011 Free Software Foundation, Inc.
-
-## This program is free software: you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published by
-## the Free Software Foundation, either version 3 of the License, or
-## (at your option) any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-include gnulib.mk

m4/wget.m4

@@ -51,8 +51,8 @@ dnl /usr/local/include (!), which GCC uses before /usr/include.
 
 AC_DEFUN([WGET_FNMATCH], [
   AC_MSG_CHECKING([for working fnmatch.h])
-  AC_COMPILE_IFELSE([#include <fnmatch.h>
-                    ], [
+  AC_COMPILE_IFELSE([AC_LANG_SOURCE([#include <fnmatch.h>
+                    ])], [
     AC_MSG_RESULT(yes)
     AC_DEFINE([HAVE_WORKING_FNMATCH_H], 1,
               [Define if fnmatch.h can be included.])

msdos/ChangeLog

@@ -1,3 +1,9 @@
+2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.DJ: Remove terminating empty lines.
+	* Makefile.WC: Likewise.
+	* msdos.c: Likewise.
+
 2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
 
 	* Makefile.WC: Added compilation of new file msdos.c.

msdos/Makefile.DJ

@@ -87,4 +87,3 @@ depend: version.c
         sed -e 's/^\([a-zA-Z0-9_-]*\.o:\)/$$(OBJ_DIR)\/\1/' > depend.dj
 
 -include depend.dj
-

msdos/Makefile.WC

@@ -65,4 +65,3 @@ version.c: ..\msdos\Makefile.WC
 clean: .SYMBOLIC
 	- rm $(OBJ_DIR)\*.obj wget.exe wget.map version.c css.c
 	- rmdir $(OBJ_DIR)
-

msdos/msdos.c

@@ -105,4 +105,3 @@ getpass (const char *prompt)
 
   return strdup (getpassbuf);
 }
-

po/POTFILES.in

@@ -5,8 +5,13 @@
 
 # Package source files
 lib/error.c
+lib/gai_strerror.c
 lib/getopt.c
 lib/quotearg.c
+lib/regcomp.c
+lib/spawn-pipe.c
+lib/w32spawn.h
+lib/wait-process.c
 lib/xalloc-die.c
 src/connect.c
 src/convert.c
@@ -32,3 +37,4 @@ src/retr.c
 src/spider.c
 src/url.c
 src/utils.c
+src/warc.c

po/stamp-po

@@ -1 +0,0 @@
-timestamp

src/DESCRIP_MODS.MMS

@@ -56,4 +56,3 @@ MODS_OBJS_LIB_SRC_SSL =
 .ENDIF                          # CDEFS_SSL [else]
 
 MODS_OBJS_LIB_SRC = $(MODS_OBJS_LIB_SRC_NOSSL) $(MODS_OBJS_LIB_SRC_SSL)
-

src/DESCRIP_SRC.MMS

@@ -32,4 +32,3 @@ CFLAGS_SPEC = /prefix_library_entries = \
 # Object library modules.
 
 .INCLUDE DESCRIP_MODS.MMS
-

src/Makefile.am

@@ -42,18 +42,18 @@ LIBS     = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME)
 EXTRA_DIST = css.l css.c css_.c build_info.c.in
 
 bin_PROGRAMS = wget
-wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c    		  \
-	       css_.c css-url.c \
-	       ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c \
-	       http.c init.c log.c main.c netrc.c progress.c ptimer.c     \
-	       recur.c res.c retr.c spider.c url.c	          	  \
-	       utils.c exits.c build_info.c $(IRI_OBJ)			  \
-	       css-url.h css-tokens.h connect.h convert.h cookies.h       \
-	       ftp.h hash.h host.h html-parse.h html-url.h      \
-	       http.h http-ntlm.h init.h log.h mswindows.h netrc.h        \
-	       options.h progress.h ptimer.h recur.h res.h retr.h         \
-	       spider.h ssl.h sysdep.h url.h utils.h wget.h iri.h 	  \
-	       exits.h gettext.h
+wget_SOURCES = connect.c convert.c cookies.c ftp.c	\
+		css_.c css-url.c	\
+		ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c	\
+		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
+		recur.c res.c retr.c spider.c url.c warc.c	\
+		utils.c exits.c build_info.c $(IRI_OBJ)	\
+		css-url.h css-tokens.h connect.h convert.h cookies.h	\
+		ftp.h hash.h host.h html-parse.h html-url.h	\
+		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
+		options.h progress.h ptimer.h recur.h res.h retr.h	\
+		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
+		exits.h version.h
 nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
 LDADD = $(LIBOBJS) ../lib/libgnu.a
@@ -74,6 +74,7 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	echo '/* version.c */' > $@
 	echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@
 	echo '' >> $@
+	echo '#include "version.h"' >> $@
 	echo 'const char *version_string = "@VERSION@";' >> $@
 	echo 'const char *compilation_string = "'$(COMPILE)'";' \
 	    | $(ESCAPEQUOTE) >> $@

src/build_info.c.in

@@ -2,12 +2,13 @@ digest          defined ENABLE_DIGEST
 https           defined HAVE_SSL
 ipv6            defined ENABLE_IPV6
 iri             defined ENABLE_IRI
-large-file      SIZEOF_OFF_T >= 8
+large-file      SIZEOF_OFF_T >= 8 || defined WINDOWS
 
 nls             defined ENABLE_NLS
 ntlm            defined ENABLE_NTLM
 opie            defined ENABLE_OPIE
+psl             defined HAVE_LIBPSL
 
 ssl choice:
-    openssl     defined HAVE_LIBSSL
+    openssl     defined HAVE_LIBSSL || defined HAVE_LIBSSL32
     gnutls      defined HAVE_LIBGNUTLS

src/connect.c

@@ -53,18 +53,13 @@ as that of the covered work.  */
 
 #include <errno.h>
 #include <string.h>
-#ifdef HAVE_SYS_TIME_H
-# include <sys/time.h>
-#endif
+#include <sys/time.h>
 #include "utils.h"
 #include "host.h"
 #include "connect.h"
 #include "hash.h"
 
-/* Apparently needed for Interix: */
-#ifdef HAVE_STDINT_H
-# include <stdint.h>
-#endif
+#include <stdint.h>
 
 /* Define sockaddr_storage where unavailable (presumably on IPv4-only
    hosts).  */
@@ -173,7 +168,7 @@ sockaddr_size (const struct sockaddr *sa)
       abort ();
     }
 }
-
+
 /* Resolve the bind address specified via --bind-address and store it
    to SA.  The resolved value is stored in a static variable and
    reused after the first invocation of this function.
@@ -218,7 +213,7 @@ resolve_bind_address (struct sockaddr *sa)
   should_bind = true;
   return true;
 }
-
+
 struct cwt_context {
   int fd;
   const struct sockaddr *addr;
@@ -255,7 +250,7 @@ connect_with_timeout (int fd, const struct sockaddr *addr, socklen_t addrlen,
     errno = ETIMEDOUT;
   return ctx.result;
 }
-
+
 /* Connect via TCP to the specified address and port.
 
    If PRINT is non-NULL, it is the host name to print that we're
@@ -275,7 +270,7 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
       const char *txt_addr = print_address (ip);
       if (0 != strcmp (print, txt_addr))
         {
-				  char *str = NULL, *name;
+          char *str = NULL, *name;
 
           if (opt.enable_iri && (name = idn_decode ((char *) print)) != NULL)
             {
@@ -289,11 +284,17 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
           logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "),
                      str ? str : escnonprint_uri (print), txt_addr, port);
 
-					if (str)
-					  xfree (str);
+          xfree (str);
         }
       else
-        logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port);
+        {
+           if (ip->family == AF_INET)
+               logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port);
+#ifdef ENABLE_IPV6
+           else if (ip->family == AF_INET6)
+               logprintf (LOG_VERBOSE, _("Connecting to [%s]:%d... "), txt_addr, port);
+#endif
+        }
     }
 
   /* Store the sockaddr info to SA.  */
@@ -427,7 +428,7 @@ connect_to_host (const char *host, int port)
 
   return -1;
 }
-
+
 /* Create a socket, bind it to local interface BIND_ADDRESS on port
    *PORT, set up a listen backlog, and return the resulting socket, or
    -1 in case of error.
@@ -554,6 +555,7 @@ socket_ip_address (int sock, ip_address *ip, int endpoint)
   if (ret < 0)
     return false;
 
+  memset(ip, 0, sizeof(ip_address));
   ip->family = sockaddr->sa_family;
   switch (sockaddr->sa_family)
     {
@@ -581,6 +583,36 @@ socket_ip_address (int sock, ip_address *ip, int endpoint)
     }
 }
 
+/* Get the socket family of connection on FD and store
+   Return family type on success, -1 otherwise.
+
+   If ENDPOINT is ENDPOINT_LOCAL, it returns the sock family of the local
+   (client) side of the socket.  Else if ENDPOINT is ENDPOINT_PEER, it
+   returns the sock family of the remote (peer's) side of the socket.  */
+
+int
+socket_family (int sock, int endpoint)
+{
+  struct sockaddr_storage storage;
+  struct sockaddr *sockaddr = (struct sockaddr *) &storage;
+  socklen_t addrlen = sizeof (storage);
+  int ret;
+
+  memset (sockaddr, 0, addrlen);
+
+  if (endpoint == ENDPOINT_LOCAL)
+    ret = getsockname (sock, sockaddr, &addrlen);
+  else if (endpoint == ENDPOINT_PEER)
+    ret = getpeername (sock, sockaddr, &addrlen);
+  else
+    abort ();
+
+  if (ret < 0)
+    return -1;
+
+  return sockaddr->sa_family;
+}
+
 /* Return true if the error from the connect code can be considered
    retryable.  Wget normally retries after errors, but the exception
    are the "unsupported protocol" type errors (possible on IPv4/IPv6
@@ -713,7 +745,7 @@ wget uses blocking sockets so we must convert them back to blocking
        or EOF/error. */
     return false;
 }
-
+
 /* Basic socket operations, mostly EINTR wrappers.  */
 
 static int
@@ -761,7 +793,7 @@ sock_close (int fd)
 #undef read
 #undef write
 #undef close
-
+
 /* Reading and writing from the network.  We build around the socket
    (file descriptor) API, but support "extended" operations for things
    that are not mere file descriptors under the hood, such as SSL
@@ -812,7 +844,7 @@ void *
 fd_transport_context (int fd)
 {
   struct transport_info *info = hash_table_get (transport_map, (void *)(intptr_t) fd);
-  return info->ctx;
+  return info ? info->ctx : NULL;
 }
 
 /* When fd_read/fd_write are called multiple times in a loop, they should

src/connect.h

@@ -32,7 +32,7 @@ as that of the covered work.  */
 #ifndef CONNECT_H
 #define CONNECT_H
 
-#include "host.h"		/* for definition of ip_address */
+#include "host.h"       /* for definition of ip_address */
 
 /* Function declarations */
 
@@ -51,6 +51,7 @@ enum {
   ENDPOINT_PEER
 };
 bool socket_ip_address (int, ip_address *, int);
+int  socket_family (int sock, int endpoint);
 
 bool retryable_socket_connect_error (int);
 

src/convert.c

@@ -1,5 +1,5 @@
 /* Conversion of links to local files.
-   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
+   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014
    Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -58,7 +58,7 @@ struct hash_table *downloaded_css_set;
 static void convert_links (const char *, struct urlpos *);
 
 
-void
+static void
 convert_links_in_hashtable (struct hash_table *downloaded_set,
                             int is_css,
                             int *file_count)
@@ -124,6 +124,9 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
           set_uri_encoding (pi, opt.locale, true);
 
           u = url_parse (cur_url->url->url, NULL, pi, true);
+          if (!u)
+              continue;
+
           local_name = hash_table_get (dl_url_file_map, u->url);
 
           /* Decide on the conversion type.  */
@@ -462,7 +465,7 @@ write_backup_file (const char *file, downloaded_file_t downloaded_file_return)
     converted_files = make_string_hash_table (0);
 
   /* We can get called twice on the same URL thanks to the
-     convert_all_links() call in main().  If we write the .orig file
+     convert_all_links() call in main.  If we write the .orig file
      each time in such a case, it'll end up containing the first-pass
      conversion, not the original file.  So, see if we've already been
      called on this file. */
@@ -623,6 +626,7 @@ local_quote_string (const char *file, bool no_html_quote)
   /* Allocate space assuming the worst-case scenario, each character
      having to be quoted.  */
   to = newname = (char *)alloca (3 * strlen (file) + 1);
+  newname[0] = '\0';
   for (from = file; *from; from++)
     switch (*from)
       {
@@ -657,7 +661,7 @@ local_quote_string (const char *file, bool no_html_quote)
 
   return no_html_quote ? strdup (newname) : html_quote_string (newname);
 }
-
+
 /* Book-keeping code for dl_file_url_map, dl_url_file_map,
    downloaded_html_list, and downloaded_html_set.  Other code calls
    these functions to let us know that a file has been downloaded.  */
@@ -832,7 +836,7 @@ register_download (const char *url, const char *file)
   hash_table_put (dl_url_file_map, xstrdup (url), xstrdup (file));
 }
 
-/* Register that FROM has been redirected to TO.  This assumes that TO
+/* Register that FROM has been redirected to "TO".  This assumes that TO
    is successfully downloaded and already registered using
    register_download() above.  */
 
@@ -870,7 +874,7 @@ register_delete_file (const char *file)
 /* Register that FILE is an HTML file that has been downloaded. */
 
 void
-register_html (const char *url, const char *file)
+register_html (const char *file)
 {
   if (!downloaded_html_set)
     downloaded_html_set = make_string_hash_table (0);
@@ -880,7 +884,7 @@ register_html (const char *url, const char *file)
 /* Register that FILE is a CSS file that has been downloaded. */
 
 void
-register_css (const char *url, const char *file)
+register_css (const char *file)
 {
   if (!downloaded_css_set)
     downloaded_css_set = make_string_hash_table (0);
@@ -912,7 +916,7 @@ convert_cleanup (void)
   if (converted_files)
     string_set_free (converted_files);
 }
-
+
 /* Book-keeping code for downloaded files that enables extension
    hacks.  */
 
@@ -1008,7 +1012,7 @@ downloaded_files_free (void)
       downloaded_files_hash = NULL;
     }
 }
-
+
 /* The function returns the pointer to the malloc-ed quoted version of
    string s.  It will recognize and quote numeric and special graphic
    entities, as per RFC1866:
@@ -1084,4 +1088,3 @@ html_quote_string (const char *s)
 /*
  * vim: et ts=2 sw=2
  */
-

src/convert.h

@@ -31,18 +31,18 @@ as that of the covered work.  */
 #ifndef CONVERT_H
 #define CONVERT_H
 
-struct hash_table;		/* forward decl */
+struct hash_table;              /* forward decl */
 extern struct hash_table *dl_url_file_map;
 extern struct hash_table *downloaded_html_set;
 extern struct hash_table *downloaded_css_set;
 
 enum convert_options {
-  CO_NOCONVERT = 0,		/* don't convert this URL */
-  CO_CONVERT_TO_RELATIVE,	/* convert to relative, e.g. to
+  CO_NOCONVERT = 0,             /* don't convert this URL */
+  CO_CONVERT_TO_RELATIVE,       /* convert to relative, e.g. to
                                    "../../otherdir/foo.gif" */
-  CO_CONVERT_TO_COMPLETE,	/* convert to absolute, e.g. to
-				   "http://orighost/somedir/bar.jpg". */
-  CO_NULLIFY_BASE		/* change to empty string. */
+  CO_CONVERT_TO_COMPLETE,       /* convert to absolute, e.g. to
+                                   "http://orighost/somedir/bar.jpg". */
+  CO_NULLIFY_BASE               /* change to empty string. */
 };
 
 struct url;
@@ -51,36 +51,36 @@ struct url;
    position in an HTML document, etc.  */
 
 struct urlpos {
-  struct url *url;		/* the URL of the link, after it has
-				   been merged with the base */
-  char *local_name;		/* local file to which it was saved
-				   (used by convert_links) */
+  struct url *url;              /* the URL of the link, after it has
+                                   been merged with the base */
+  char *local_name;             /* local file to which it was saved
+                                   (used by convert_links) */
 
   /* reserved for special links such as <base href="..."> which are
      used when converting links, but ignored when downloading.  */
-  unsigned int ignore_when_downloading	:1;
+  unsigned int ignore_when_downloading  :1;
 
   /* Information about the original link: */
 
-  unsigned int link_relative_p	:1; /* the link was relative */
-  unsigned int link_complete_p	:1; /* the link was complete (had host name) */
-  unsigned int link_base_p	:1; /* the url came from <base href=...> */
-  unsigned int link_inline_p	:1; /* needed to render the page */
-  unsigned int link_css_p	:1; /* the url came from CSS */
-  unsigned int link_expect_html	:1; /* expected to contain HTML */
-  unsigned int link_expect_css	:1; /* expected to contain CSS */
+  unsigned int link_relative_p  :1; /* the link was relative */
+  unsigned int link_complete_p  :1; /* the link was complete (had host name) */
+  unsigned int link_base_p  :1;     /* the url came from <base href=...> */
+  unsigned int link_inline_p    :1; /* needed to render the page */
+  unsigned int link_css_p   :1;     /* the url came from CSS */
+  unsigned int link_expect_html :1; /* expected to contain HTML */
+  unsigned int link_expect_css  :1; /* expected to contain CSS */
 
-  unsigned int link_refresh_p	:1; /* link was received from
-				       <meta http-equiv=refresh content=...> */
-  int refresh_timeout;		/* for reconstructing the refresh. */
+  unsigned int link_refresh_p   :1; /* link was received from
+                                       <meta http-equiv=refresh content=...> */
+  int refresh_timeout;              /* for reconstructing the refresh. */
 
   /* Conversion requirements: */
-  enum convert_options convert;	/* is conversion required? */
+  enum convert_options convert;     /* is conversion required? */
 
   /* URL's position in the buffer. */
   int pos, size;
 
-  struct urlpos *next;		/* next list element */
+  struct urlpos *next;              /* next list element */
 };
 
 /* downloaded_file() takes a parameter of this type and returns this type. */
@@ -101,8 +101,8 @@ downloaded_file_t downloaded_file (downloaded_file_t, const char *);
 
 void register_download (const char *, const char *);
 void register_redirection (const char *, const char *);
-void register_html (const char *, const char *);
-void register_css (const char *, const char *);
+void register_html (const char *);
+void register_css (const char *);
 void register_delete_file (const char *);
 void convert_all_links (void);
 void convert_cleanup (void);

src/cookies.c

@@ -51,11 +51,16 @@ as that of the covered work.  */
 #include <assert.h>
 #include <errno.h>
 #include <time.h>
+#ifdef HAVE_LIBPSL
+# include <libpsl.h>
+#endif
 #include "utils.h"
 #include "hash.h"
 #include "cookies.h"
 #include "http.h"               /* for http_atotm */
-
+#include "c-strcase.h"
+
+
 /* Declarations of `struct cookie' and the most basic functions. */
 
 /* Cookie jar serves as cookie storage and a means of retrieving
@@ -95,7 +100,7 @@ struct cookie {
   int port;                     /* port number */
   char *path;                   /* path prefix of the cookie */
 
-  unsigned discard_requested :1; /* whether cookie was created to
+  unsigned discard_requested :1;/* whether cookie was created to
                                    request discarding another
                                    cookie. */
 
@@ -148,13 +153,13 @@ cookie_expired_p (const struct cookie *c)
 static void
 delete_cookie (struct cookie *cookie)
 {
-  xfree_null (cookie->domain);
-  xfree_null (cookie->path);
-  xfree_null (cookie->attr);
-  xfree_null (cookie->value);
+  xfree (cookie->domain);
+  xfree (cookie->path);
+  xfree (cookie->attr);
+  xfree (cookie->value);
   xfree (cookie);
 }
-
+
 /* Functions for storing cookies.
 
    All cookies can be reached beginning with jar->chains.  The key in
@@ -299,7 +304,12 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)
 
           res = hash_table_get_pair (jar->chains, victim->domain,
                                      &chain_key, NULL);
-          assert (res != 0);
+
+          if (res == 0)
+            {
+              logprintf (LOG_VERBOSE, _("Unable to get cookie for %s\n"),
+                         victim->domain);
+            }
           if (!victim->next)
             {
               /* VICTIM was the only cookie in the chain.  Destroy the
@@ -314,7 +324,7 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)
       DEBUGP (("Discarded old cookie.\n"));
     }
 }
-
+
 /* Functions for parsing the `Set-Cookie' header, and creating new
    cookies from the wire.  */
 
@@ -346,7 +356,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
   struct cookie *cookie = cookie_new ();
   param_token name, value;
 
-  if (!extract_param (&ptr, &name, &value, ';'))
+  if (!extract_param (&ptr, &name, &value, ';', NULL))
     goto error;
   if (!value.b)
     goto error;
@@ -360,13 +370,13 @@ parse_set_cookie (const char *set_cookie, bool silent)
   cookie->attr = strdupdelim (name.b, name.e);
   cookie->value = strdupdelim (value.b, value.e);
 
-  while (extract_param (&ptr, &name, &value, ';'))
+  while (extract_param (&ptr, &name, &value, ';', NULL))
     {
       if (TOKEN_IS (name, "domain"))
         {
           if (!TOKEN_NON_EMPTY (value))
             goto error;
-          xfree_null (cookie->domain);
+          xfree (cookie->domain);
           /* Strictly speaking, we should set cookie->domain_exact if the
              domain doesn't begin with a dot.  But many sites set the
              domain to "foo.com" and expect "subhost.foo.com" to get the
@@ -379,7 +389,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
         {
           if (!TOKEN_NON_EMPTY (value))
             goto error;
-          xfree_null (cookie->path);
+          xfree (cookie->path);
           cookie->path = strdupdelim (value.b, value.e);
         }
       else if (TOKEN_IS (name, "expires"))
@@ -391,6 +401,9 @@ parse_set_cookie (const char *set_cookie, bool silent)
             goto error;
           BOUNDED_TO_ALLOCA (value.b, value.e, value_copy);
 
+          /* Check if expiration spec is valid.
+             If not, assume default (cookie doesn't expire, but valid only for
+             this session.) */
           expires = http_atotm (value_copy);
           if (expires != (time_t) -1)
             {
@@ -402,10 +415,6 @@ parse_set_cookie (const char *set_cookie, bool silent)
               if (cookie->expiry_time < cookies_now)
                 cookie->discard_requested = 1;
             }
-          else
-            /* Error in expiration spec.  Assume default (cookie doesn't
-               expire, but valid only for this session.)  */
-            ;
         }
       else if (TOKEN_IS (name, "max-age"))
         {
@@ -433,9 +442,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
           /* ignore value completely */
           cookie->secure = 1;
         }
-      else
-        /* Ignore unrecognized attribute. */
-        ;
+      /* else: Ignore unrecognized attribute. */
     }
   if (*ptr)
     /* extract_param has encountered a syntax error */
@@ -456,16 +463,16 @@ parse_set_cookie (const char *set_cookie, bool silent)
 
 #undef TOKEN_IS
 #undef TOKEN_NON_EMPTY
-
+
 /* Sanity checks.  These are important, otherwise it is possible for
    mailcious attackers to destroy important cookie information and/or
    violate your privacy.  */
 
 
 #define REQUIRE_DIGITS(p) do {                  \
-  if (!c_isdigit (*p))                            \
+  if (!c_isdigit (*p))                          \
     return false;                               \
-  for (++p; c_isdigit (*p); p++)                  \
+  for (++p; c_isdigit (*p); p++)                \
     ;                                           \
 } while (0)
 
@@ -501,19 +508,61 @@ numeric_address_p (const char *addr)
 /* Check whether COOKIE_DOMAIN is an appropriate domain for HOST.
    Originally I tried to make the check compliant with rfc2109, but
    the sites deviated too often, so I had to fall back to "tail
-   matching", as defined by the original Netscape's cookie spec.  */
+   matching", as defined by the original Netscape's cookie spec.
+
+   Wget now uses libpsl to check domain names against a public suffix
+   list to see if they are valid. However, since we don't provide a
+   psl on our own, if libpsl is compiled without a public suffix list,
+   fall back to using the original "tail matching" heuristic. Also if
+   libpsl is unable to convert the domain to lowercase, which means that
+   it doesnt have any runtime conversion support, we again fall back to
+   "tail matching" since libpsl states the results are unpredictable with
+   upper case strings.
+   */
 
 static bool
 check_domain_match (const char *cookie_domain, const char *host)
 {
+
+#ifdef HAVE_LIBPSL
+  char *cookie_domain_lower = NULL;
+  char *host_lower = NULL;
+  const psl_ctx_t *psl;
+  int is_acceptable;
+
   DEBUGP (("cdm: 1"));
+  if (!(psl = psl_builtin()))
+    {
+      DEBUGP (("\nlibpsl not built with a public suffix list. "
+               "Falling back to simple heuristics.\n"));
+      goto no_psl;
+    }
 
-  /* Numeric address requires exact match.  It also requires HOST to
-     be an IP address.  */
-  if (numeric_address_p (cookie_domain))
-    return 0 == strcmp (cookie_domain, host);
+  if (psl_str_to_utf8lower (cookie_domain, NULL, NULL, &cookie_domain_lower) == PSL_SUCCESS &&
+      psl_str_to_utf8lower (host, NULL, NULL, &host_lower) == PSL_SUCCESS)
+    {
+      is_acceptable = psl_is_cookie_domain_acceptable (psl, host_lower, cookie_domain_lower);
+    }
+  else
+    {
+        DEBUGP (("libpsl unable to parse domain name. "
+                 "Falling back to simple heuristics.\n"));
+        goto no_psl;
+    }
 
-  DEBUGP ((" 2"));
+  xfree (cookie_domain_lower);
+  xfree (host_lower);
+
+  return is_acceptable == 1;
+
+no_psl:
+  /* Cleanup the PSL pointers first */
+  xfree (cookie_domain_lower);
+  xfree (host_lower);
+#endif
+
+  /* For efficiency make some elementary checks first */
+  DEBUGP (("cdm: 2"));
 
   /* For the sake of efficiency, check for exact match first. */
   if (0 == strcasecmp (cookie_domain, host))
@@ -650,7 +699,7 @@ check_path_match (const char *cookie_path, const char *path)
   s = PS_newstr;                                                \
 } while (0)
 
-
+
 /* Process the HTTP `Set-Cookie' header.  This results in storing the
    cookie or discarding a matching one, or ignoring it completely, all
    depending on the contents.  */
@@ -676,9 +725,6 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
 
   if (!cookie->domain)
     {
-    copy_domain:
-      /* If the domain was not provided, we use the one we're talking
-         to, and set exact match.  */
       cookie->domain = xstrdup (host);
       cookie->domain_exact = 1;
       /* Set the port, but only if it's non-default. */
@@ -690,11 +736,12 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
       if (!check_domain_match (cookie->domain, host))
         {
           logprintf (LOG_NOTQUIET,
-                     _("Cookie coming from %s attempted to set domain to %s\n"),
-                     quotearg_style (escape_quoting_style, host),
+                     _("Cookie coming from %s attempted to set domain to "),
+                     quotearg_style (escape_quoting_style, host));
+          logprintf (LOG_NOTQUIET,
+                     _("%s\n"),
                      quotearg_style (escape_quoting_style, cookie->domain));
-          xfree (cookie->domain);
-          goto copy_domain;
+          cookie->discard_requested = true;
         }
     }
 
@@ -736,7 +783,7 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
   if (cookie)
     delete_cookie (cookie);
 }
-
+
 /* Support for sending out cookies in HTTP requests, based on
    previously stored cookies.  Entry point is
    `build_cookies_request'.  */
@@ -1067,7 +1114,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
   assert (pos == result_size);
   return result;
 }
-
+
 /* Support for loading and saving cookies.  The format used for
    loading and saving should be the format of the `cookies.txt' file
    used by Netscape and Mozilla, at least the Unix versions.
@@ -1134,7 +1181,9 @@ domain_port (const char *domain_b, const char *domain_e,
 void
 cookie_jar_load (struct cookie_jar *jar, const char *file)
 {
-  char *line;
+  char *line = NULL;
+  size_t bufsize = 0;
+
   FILE *fp = fopen (file, "r");
   if (!fp)
     {
@@ -1142,9 +1191,10 @@ cookie_jar_load (struct cookie_jar *jar, const char *file)
                  quote (file), strerror (errno));
       return;
     }
+
   cookies_now = time (NULL);
 
-  for (; ((line = read_whole_line (fp)) != NULL); xfree (line))
+  while (getline (&line, &bufsize, fp) > 0)
     {
       struct cookie *cookie;
       char *p = line;
@@ -1238,6 +1288,8 @@ cookie_jar_load (struct cookie_jar *jar, const char *file)
     abort_cookie:
       delete_cookie (cookie);
     }
+
+  xfree(line);
   fclose (fp);
 }
 
@@ -1301,7 +1353,7 @@ cookie_jar_save (struct cookie_jar *jar, const char *file)
 
   DEBUGP (("Done saving cookies.\n"));
 }
-
+
 /* Clean up cookie-related data. */
 
 void
@@ -1324,7 +1376,7 @@ cookie_jar_delete (struct cookie_jar *jar)
   hash_table_destroy (jar->chains);
   xfree (jar);
 }
-
+
 /* Test cases.  Currently this is only tests parse_set_cookies.  To
    use, recompile Wget with -DTEST_COOKIES and call test_cookies()
    from main.  */
@@ -1377,7 +1429,7 @@ test_cookies (void)
         param_token name, value;
         const char *ptr = data;
         int j = 0;
-        while (extract_param (&ptr, &name, &value, ';'))
+        while (extract_param (&ptr, &name, &value, ';', NULL))
           {
             char *n = strdupdelim (name.b, name.e);
             char *v = strdupdelim (value.b, value.e);
@@ -1393,8 +1445,8 @@ test_cookies (void)
               printf ("Invalid value %d for '%s' (expected '%s', got '%s')\n",
                       j / 2 + 1, data, expected[j + 1], v);
             j += 2;
-            free (n);
-            free (v);
+            xfree (n);
+            xfree (v);
           }
         if (expected[j])
           printf ("Too few parameters for '%s'\n", data);

src/cookies.h

@@ -37,9 +37,9 @@ struct cookie_jar *cookie_jar_new (void);
 void cookie_jar_delete (struct cookie_jar *);
 
 void cookie_handle_set_cookie (struct cookie_jar *, const char *, int,
-			       const char *, const char *);
+                               const char *, const char *);
 char *cookie_header (struct cookie_jar *, const char *, int,
-		     const char *, bool);
+                     const char *, bool);
 
 void cookie_jar_load (struct cookie_jar *, const char *);
 void cookie_jar_save (struct cookie_jar *, const char *);

src/css-url.c

@@ -1,5 +1,5 @@
 /* Collect URLs from CSS source.
-   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011 Free
+   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011, 2014 Free
    Software Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -41,11 +41,7 @@ as that of the covered work.  */
 #include <wget.h>
 
 #include <stdio.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif
+#include <string.h>
 #include <stdlib.h>
 #include <ctype.h>
 #include <errno.h>
@@ -55,6 +51,8 @@ as that of the covered work.  */
 #include "convert.h"
 #include "html-url.h"
 #include "css-tokens.h"
+#include "css-url.h"
+#include "xstrndup.h"
 
 /* from lex.yy.c */
 extern char *yytext;
@@ -63,41 +61,6 @@ typedef struct yy_buffer_state *YY_BUFFER_STATE;
 extern YY_BUFFER_STATE yy_scan_bytes (const char *bytes,int len  );
 extern int yylex (void);
 
-#if 1
-const char *token_names[] = {
-  "CSSEOF",
-  "S",
-  "CDO",
-  "CDC",
-  "INCLUDES",
-  "DASHMATCH",
-  "LBRACE",
-  "PLUS",
-  "GREATER",
-  "COMMA",
-  "STRING",
-  "INVALID",
-  "IDENT",
-  "HASH",
-  "IMPORT_SYM",
-  "PAGE_SYM",
-  "MEDIA_SYM",
-  "CHARSET_SYM",
-  "IMPORTANT_SYM",
-  "EMS",
-  "EXS",
-  "LENGTH",
-  "ANGLE",
-  "TIME",
-  "FREQ",
-  "DIMENSION",
-  "PERCENTAGE",
-  "NUMBER",
-  "URI",
-  "FUNCTION"
-};
-#endif
-
 /*
   Given a detected URI token, get only the URI specified within.
   Also adjust the starting position and length of the string.
@@ -107,15 +70,9 @@ const char *token_names[] = {
   whitespace after the opening parenthesis and before the closing
   parenthesis.
 */
-char *
+static char *
 get_uri_string (const char *at, int *pos, int *length)
 {
-  char *uri;
-  /*char buf[1024];
-  strncpy(buf,at + *pos, *length);
-  buf[*length] = '\0';
-  DEBUGP (("get_uri_string: \"%s\"\n", buf));*/
-
   if (0 != strncasecmp (at + *pos, "url(", 4))
     return NULL;
 
@@ -141,14 +98,7 @@ get_uri_string (const char *at, int *pos, int *length)
       *length -= 2;
     }
 
-  uri = xmalloc (*length + 1);
-  if (uri)
-    {
-      strncpy (uri, at + *pos, *length);
-      uri[*length] = '\0';
-    }
-
-  return uri;
+  return xstrndup (at + *pos, *length);
 }
 
 void
@@ -160,12 +110,6 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
   int pos, length;
   char *uri;
 
-  /*
-  strncpy(tmp,ctx->text + offset, buf_length);
-  tmp[buf_length] = '\0';
-  DEBUGP (("get_urls_css: \"%s\"\n", tmp));
-  */
-
   /* tell flex to scan from this buffer */
   yy_scan_bytes (ctx->text + offset, buf_length);
 
@@ -199,7 +143,7 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
                   pos++;
                   length -= 2;
                   uri = xmalloc (length + 1);
-                  strncpy (uri, yytext + 1, length);
+                  memcpy (uri, yytext + 1, length);
                   uri[length] = '\0';
                 }
 

src/css-url.h

@@ -30,6 +30,7 @@ as that of the covered work.  */
 #ifndef CSS_URL_H
 #define CSS_URL_H
 
+void get_urls_css (struct map_context *, int, int);
 void get_urls_css (struct map_context *, int, int);
 struct urlpos *get_urls_css_file (const char *, const char *);
 

src/decc_ver.c

@@ -2,6 +2,15 @@
 
 int main( void)
 {
+
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
 #ifdef __VMS_VERSION
 printf( " __VMS_VERSION: >%s<.\n", __VMS_VERSION);
 #else /* def __VMS_VERSION */

src/exits.c

@@ -1,7 +1,5 @@
-/* Command line parsing.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+/* Exit status handling.
+   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
 
    This file is part of GNU Wget.
 
@@ -22,24 +20,6 @@
 #include "wget.h"
 #include "exits.h"
 
-/* Final exit code possibilities. Exit codes 1 and 2 are reserved
- * for situations that lead to direct exits from Wget, not using the
- * value of final_exit_status. */
-enum
-  {
-    WGET_EXIT_SUCCESS = 0,
-
-    WGET_EXIT_MINIMUM = 3,
-    WGET_EXIT_IO_FAIL = WGET_EXIT_MINIMUM,
-    WGET_EXIT_NETWORK_FAIL = 4,
-    WGET_EXIT_SSL_AUTH_FAIL = 5,
-    WGET_EXIT_SERVER_AUTH_FAIL = 6,
-    WGET_EXIT_PROTOCOL_ERROR = 7,
-    WGET_EXIT_SERVER_ERROR = 8,
-
-    WGET_EXIT_UNKNOWN
-  };
-
 static int final_exit_status = WGET_EXIT_SUCCESS;
 
 /* XXX: I don't like that newly-added uerr_t codes will doubtless fall
@@ -60,7 +40,7 @@ get_status_for_err (uerr_t err)
     case RETROK:
       return WGET_EXIT_SUCCESS;
     case FOPENERR: case FOPEN_EXCL_ERR: case FWRITEERR: case WRITEFAILED:
-    case UNLINKERR:
+    case UNLINKERR: case CLOSEFAILED: case FILEBADFILE:
       return WGET_EXIT_IO_FAIL;
     case NOCONERROR: case HOSTERR: case CONSOCKERR: case CONERROR:
     case CONSSLERR: case CONIMPOSSIBLE: case FTPRERR: case FTPINVPASV:
@@ -70,15 +50,15 @@ get_status_for_err (uerr_t err)
       return WGET_EXIT_SSL_AUTH_FAIL;
     case FTPLOGINC: case FTPLOGREFUSED: case AUTHFAILED:
       return WGET_EXIT_SERVER_AUTH_FAIL;
-    case HEOF: case HERR:
+    case HEOF: case HERR: case ATTRMISSING:
       return WGET_EXIT_PROTOCOL_ERROR;
     case WRONGCODE: case FTPPORTERR: case FTPSYSERR:
     case FTPNSFOD: case FTPUNKNOWNTYPE: case FTPSRVERR:
     case FTPRETRINT: case FTPRESTFAIL: case FTPNOPASV:
     case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN:
-    case PROXERR:
+    case PROXERR: case GATEWAYTIMEOUT:
       return WGET_EXIT_SERVER_ERROR;
-    case URLERROR: case QUOTEXC: case SSLINITFAILED:
+    case URLERROR: case QUOTEXC: case SSLINITFAILED: case UNKNOWNATTR:
     default:
       return WGET_EXIT_UNKNOWN;
     }
@@ -110,4 +90,3 @@ get_exit_status (void)
       ? 1
       : final_exit_status;
 }
-

src/exits.h

@@ -1,5 +1,5 @@
-/* Internationalization related declarations.
-   Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+/* Exit status related declarations.
+   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -21,6 +21,23 @@ along with Wget.  If not, see <http://www.gnu.org/licenses/>.  */
 
 #include "wget.h"
 
+/* Final exit code possibilities. Exit codes 1 and 2 are reserved
+ * for situations that lead to direct exits from Wget, not using the
+ * value of final_exit_status. */
+enum
+  {
+    WGET_EXIT_SUCCESS = 0,
+    WGET_EXIT_GENERIC_ERROR = 1,
+    WGET_EXIT_PARSE_ERROR = 2,
+    WGET_EXIT_IO_FAIL = 3,
+    WGET_EXIT_NETWORK_FAIL = 4,
+    WGET_EXIT_SSL_AUTH_FAIL = 5,
+    WGET_EXIT_SERVER_AUTH_FAIL = 6,
+    WGET_EXIT_PROTOCOL_ERROR = 7,
+    WGET_EXIT_SERVER_ERROR = 8,
+
+    WGET_EXIT_UNKNOWN
+  };
 
 void inform_exit_status (uerr_t err);
 

src/ftp-basic.c

@@ -1,6 +1,6 @@
 /* Basic FTP routines.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -43,10 +43,9 @@ as that of the covered work.  */
 #include "host.h"
 #include "ftp.h"
 #include "retr.h"
+#include "c-strcase.h"
 
-char ftp_last_respline[128];
 
-
 /* Get the response of FTP server and allocate enough room to handle
    it.  <CR> and <LF> characters are stripped from the line, and the
    line is 0-terminated.  All the response lines but the last one are
@@ -84,8 +83,6 @@ ftp_response (int fd, char **ret_line)
       if (c_isdigit (line[0]) && c_isdigit (line[1]) && c_isdigit (line[2])
           && line[3] == ' ')
         {
-          strncpy (ftp_last_respline, line, sizeof (ftp_last_respline));
-          ftp_last_respline[sizeof (ftp_last_respline) - 1] = '\0';
           *ret_line = line;
           return FTPOK;
         }
@@ -194,7 +191,7 @@ ftp_login (int csock, const char *acc, const char *pass)
     for (i = 0; i < countof (skey_head); i++)
       {
         int l = strlen (skey_head[i]);
-        if (0 == strncasecmp (skey_head[i], respline, l))
+        if (0 == c_strncasecmp (skey_head[i], respline, l))
           {
             seed = respline + l;
             break;
@@ -524,7 +521,10 @@ ftp_pasv (int csock, ip_address *addr, int *port)
   for (s += 4; *s && !c_isdigit (*s); s++)
     ;
   if (!*s)
-    return FTPINVPASV;
+    {
+      xfree (respline);
+      return FTPINVPASV;
+    }
   for (i = 0; i < 6; i++)
     {
       tmp[i] = 0;
@@ -593,7 +593,10 @@ ftp_lpsv (int csock, ip_address *addr, int *port)
   for (s += 4; *s && !c_isdigit (*s); s++)
     ;
   if (!*s)
-    return FTPINVPASV;
+    {
+      xfree (respline);
+      return FTPINVPASV;
+    }
 
   /* First, get the address family */
   af = 0;
@@ -782,15 +785,13 @@ ftp_epsv (int csock, ip_address *ip, int *port)
     }
 
   /* Finally, get the port number */
-  tport = 0;
-  for (i = 1; c_isdigit (*s); s++)
-    {
-      if (i > 5)
-        {
-          xfree (respline);
-          return FTPINVPASV;
-        }
+  for (tport = 0, i = 0; i < 5 && c_isdigit (*s); i++, s++)
       tport = (*s - '0') + 10 * tport;
+
+  if (i >= 5)
+    {
+      xfree (respline);
+      return FTPINVPASV;
     }
 
   /* Make sure that the response terminates correcty */
@@ -954,23 +955,34 @@ ftp_retr (int csock, const char *file)
 /* Sends the LIST command to the server.  If FILE is NULL, send just
    `LIST' (no space).  */
 uerr_t
-ftp_list (int csock, const char *file, enum stype rs)
+ftp_list (int csock, const char *file, bool avoid_list_a, bool avoid_list,
+          bool *list_a_used)
 {
   char *request, *respline;
   int nwritten;
   uerr_t err;
   bool ok = false;
   size_t i = 0;
-  /* Try `LIST -a' first and revert to `LIST' in case of failure.  */
-  const char *list_commands[] = { "LIST -a",
-                                  "LIST" };
 
-  /* 2008-01-29  SMS.  For a VMS FTP server, where "LIST -a" may not
-     fail, but will never do what is desired here, skip directly to the
-     simple "LIST" command (assumed to be the last one in the list).
-  */
-  if (rs == ST_VMS)
-    i = countof (list_commands)- 1;
+  /* 2013-10-12 Andrea Urbani (matfanjol)
+     For more information about LIST and "LIST -a" please look at ftp.c,
+     function getftp, text "__LIST_A_EXPLANATION__".
+
+     If somebody changes the following commands, please, checks also the
+     later "i" variable.  */
+  static const char *list_commands[] = {
+    "LIST -a",
+    "LIST"
+  };
+
+  *list_a_used = false;
+
+  if (avoid_list_a)
+    {
+      i = countof (list_commands)- 1;
+      DEBUGP (("(skipping \"LIST -a\")"));
+    }
+
 
   do {
     /* Send request.  */
@@ -994,6 +1006,8 @@ ftp_list (int csock, const char *file, enum stype rs)
           {
             err = FTPOK;
             ok = true;
+            /* Which list command was used? */
+            *list_a_used = (i == 0);
           }
         else
           {
@@ -1002,6 +1016,12 @@ ftp_list (int csock, const char *file, enum stype rs)
         xfree (respline);
       }
     ++i;
+    if ((avoid_list) && (i == 1))
+      {
+        /* I skip LIST */
+        ++i;
+        DEBUGP (("(skipping \"LIST\")"));
+      }
   } while (i < countof (list_commands) && !ok);
 
   return err;
@@ -1009,11 +1029,12 @@ ftp_list (int csock, const char *file, enum stype rs)
 
 /* Sends the SYST command to the server. */
 uerr_t
-ftp_syst (int csock, enum stype *server_type)
+ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)
 {
   char *request, *respline;
   int nwritten;
   uerr_t err;
+  char *ftp_last_respline;
 
   /* Send SYST request.  */
   request = ftp_request ("SYST", NULL);
@@ -1035,6 +1056,8 @@ ftp_syst (int csock, enum stype *server_type)
       return FTPSRVERR;
     }
 
+  ftp_last_respline = strdup (respline);
+
   /* Skip the number (215, but 200 (!!!) in case of VMS) */
   strtok (respline, " ");
 
@@ -1042,22 +1065,34 @@ ftp_syst (int csock, enum stype *server_type)
      first word of the server response)?  */
   request = strtok (NULL, " ");
 
+  *unix_type = UST_OTHER;
+
   if (request == NULL)
     *server_type = ST_OTHER;
-  else if (!strcasecmp (request, "VMS"))
+  else if (!c_strcasecmp (request, "VMS"))
     *server_type = ST_VMS;
-  else if (!strcasecmp (request, "UNIX"))
-    *server_type = ST_UNIX;
-  else if (!strcasecmp (request, "WINDOWS_NT")
-           || !strcasecmp (request, "WINDOWS2000"))
+  else if (!c_strcasecmp (request, "UNIX"))
+    {
+      *server_type = ST_UNIX;
+      /* 2013-10-17 Andrea Urbani (matfanjol)
+         I check more in depth the system type */
+      if (!c_strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
+        *unix_type = UST_TYPE_L8;
+      else if (!c_strncasecmp (ftp_last_respline,
+                             "215 UNIX MultiNet Unix Emulation V5.3(93)", 41))
+        *unix_type = UST_MULTINET;
+    }
+  else if (!c_strcasecmp (request, "WINDOWS_NT")
+           || !c_strcasecmp (request, "WINDOWS2000"))
     *server_type = ST_WINNT;
-  else if (!strcasecmp (request, "MACOS"))
+  else if (!c_strcasecmp (request, "MACOS"))
     *server_type = ST_MACOS;
-  else if (!strcasecmp (request, "OS/400"))
+  else if (!c_strcasecmp (request, "OS/400"))
     *server_type = ST_OS400;
   else
     *server_type = ST_OTHER;
 
+  xfree (ftp_last_respline);
   xfree (respline);
   /* All OK.  */
   return FTPOK;
@@ -1101,7 +1136,7 @@ ftp_pwd (int csock, char **pwd)
     goto err;
 
   /* Has the `pwd' been already allocated?  Free! */
-  xfree_null (*pwd);
+  xfree (*pwd);
 
   *pwd = xstrdup (request);
 

src/ftp-ls.c

@@ -42,6 +42,7 @@ as that of the covered work.  */
 #include "url.h"
 #include "convert.h"            /* for html_quote_string prototype */
 #include "retr.h"               /* for output_stream */
+#include "c-strcase.h"
 
 /* Converts symbolic permissions to number-style ones, e.g. string
    rwxr-xr-x to 755.  For now, it knows nothing of
@@ -68,16 +69,17 @@ symperms (const char *s)
    replaces all <TAB> character with <SPACE>. Returns the length of the
    modified line. */
 static int
-clean_line(char *line)
+clean_line (char *line, int len)
 {
-  int len = strlen (line);
-  if (!len) return 0;
-  if (line[len - 1] == '\n')
+  if (len <= 0) return 0;
+
+  while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
     line[--len] = '\0';
+
   if (!len) return 0;
-  if (line[len - 1] == '\r')
-    line[--len] = '\0';
+
   for ( ; *line ; line++ ) if (*line == '\t') *line = ' ';
+
   return len;
 }
 
@@ -102,9 +104,10 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
   int hour, min, sec, ptype;
   struct tm timestruct, *tnow;
   time_t timenow;
+  size_t bufsize = 0;
 
-  char *line, *tok, *ptok;      /* tokenizer */
-  struct fileinfo *dir, *l, cur; /* list creation */
+  char *line = NULL, *tok, *ptok;      /* tokenizer */
+  struct fileinfo *dir, *l, cur;       /* list creation */
 
   fp = fopen (file, "rb");
   if (!fp)
@@ -115,22 +118,16 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
   dir = l = NULL;
 
   /* Line loop to end of file: */
-  while ((line = read_whole_line (fp)) != NULL)
+  while ((len = getline (&line, &bufsize, fp)) > 0)
     {
-      len = clean_line (line);
+      len = clean_line (line, len);
       /* Skip if total...  */
-      if (!strncasecmp (line, "total", 5))
-        {
-          xfree (line);
-          continue;
-        }
+      if (!c_strncasecmp (line, "total", 5))
+        continue;
       /* Get the first token (permissions).  */
       tok = strtok (line, " ");
       if (!tok)
-        {
-          xfree (line);
-          continue;
-        }
+        continue;
 
       cur.name = NULL;
       cur.linkto = NULL;
@@ -203,7 +200,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
           if (next < 0)         /* a month name was not encountered */
             {
               for (i = 0; i < 12; i++)
-                if (!strcmp (tok, months[i]))
+                if (!c_strcasecmp (tok, months[i]))
                   break;
               /* If we got a month, it means the token before it is the
                  size, and the filename is three tokens away.  */
@@ -366,9 +363,8 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
       if (error || ignore)
         {
           DEBUGP (("Skipping.\n"));
-          xfree_null (cur.name);
-          xfree_null (cur.linkto);
-          xfree (line);
+          xfree (cur.name);
+          xfree (cur.linkto);
           continue;
         }
 
@@ -416,10 +412,9 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
       timestruct.tm_isdst = -1;
       l->tstamp = mktime (&timestruct); /* store the time-stamp */
       l->ptype = ptype;
-
-      xfree (line);
     }
 
+  xfree (line);
   fclose (fp);
   return dir;
 }
@@ -431,9 +426,11 @@ ftp_parse_winnt_ls (const char *file)
   int len;
   int year, month, day;         /* for time analysis */
   int hour, min;
+  size_t bufsize = 0;
   struct tm timestruct;
 
-  char *line, *tok;             /* tokenizer */
+  char *line = NULL, *tok;             /* tokenizer */
+  char *filename;
   struct fileinfo *dir, *l, cur; /* list creation */
 
   fp = fopen (file, "rb");
@@ -445,43 +442,53 @@ ftp_parse_winnt_ls (const char *file)
   dir = l = NULL;
 
   /* Line loop to end of file: */
-  while ((line = read_whole_line (fp)) != NULL)
+  while ((len = getline (&line, &bufsize, fp)) > 0)
     {
-      len = clean_line (line);
+      len = clean_line (line, len);
 
-      /* Extracting name is a bit of black magic and we have to do it
-         before `strtok' inserted extra \0 characters in the line
-         string. For the moment let us just suppose that the name starts at
-         column 39 of the listing. This way we could also recognize
-         filenames that begin with a series of space characters (but who
-         really wants to use such filenames anyway?). */
-      if (len < 40) goto continue_loop;
-      tok = line + 39;
-      cur.name = xstrdup(tok);
-      DEBUGP (("Name: '%s'\n", cur.name));
+      /* Name begins at 39 column of the listing if date presented in `mm-dd-yy'
+         format or at 41 column if date presented in `mm-dd-yyyy' format. Thus,
+         we cannot extract name before we parse date. Using this information we
+         also can recognize filenames that begin with a series of space
+         characters (but who really wants to use such filenames anyway?). */
+      if (len < 40) continue;
+      filename = line + 39;
 
-      /* First column: mm-dd-yy. Should atoi() on the month fail, january
-         will be assumed.  */
+      /* First column: mm-dd-yy or mm-dd-yyyy. Should atoi() on the month fail,
+         january will be assumed.  */
       tok = strtok(line, "-");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       month = atoi(tok) - 1;
       if (month < 0) month = 0;
       tok = strtok(NULL, "-");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       day = atoi(tok);
       tok = strtok(NULL, " ");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       year = atoi(tok);
       /* Assuming the epoch starting at 1.1.1970 */
-      if (year <= 70) year += 100;
+      if (year <= 70)
+        {
+          year += 100;
+        }
+      else if (year >= 1900)
+        {
+          year -= 1900;
+          filename += 2;
+        }
+      /* Now it is possible to determine the position of the first symbol in
+         filename. */
+      cur.name = xstrdup(filename);
+      DEBUGP (("Name: '%s'\n", cur.name));
+
 
       /* Second column: hh:mm[AP]M, listing does not contain value for
          seconds */
       tok = strtok(NULL,  ":");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       hour = atoi(tok);
       tok = strtok(NULL,  "M");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       min = atoi(tok);
       /* Adjust hour from AM/PM. Just for the record, the sequence goes
          11:00AM, 12:00PM, 01:00PM ... 11:00PM, 12:00AM, 01:00AM . */
@@ -512,9 +519,9 @@ ftp_parse_winnt_ls (const char *file)
          directories as the listing does not give us a clue) and filetype
          here. */
       tok = strtok(NULL, " ");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       while ((tok != NULL) && (*tok == '\0'))  tok = strtok(NULL, " ");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       if (*tok == '<')
         {
           cur.type  = FT_DIRECTORY;
@@ -553,11 +560,9 @@ ftp_parse_winnt_ls (const char *file)
           memcpy (l, &cur, sizeof (cur));
           l->next = NULL;
         }
-
-continue_loop:
-      xfree (line);
     }
 
+  xfree (line);
   fclose(fp);
   return dir;
 }
@@ -678,11 +683,12 @@ ftp_parse_vms_ls (const char *file)
   FILE *fp;
   int dt, i, j, len;
   int perms;
+  size_t bufsize = 0;
   time_t timenow;
   struct tm *timestruct;
   char date_str[ 32];
 
-  char *line, *tok;		 /* tokenizer */
+  char *line = NULL, *tok; /* tokenizer */
   struct fileinfo *dir, *l, cur; /* list creation */
 
   fp = fopen (file, "r");
@@ -695,52 +701,34 @@ ftp_parse_vms_ls (const char *file)
 
   /* Skip blank lines, Directory heading, and more blank lines. */
 
-  j = 0; /* Expecting initial blank line(s). */
-  while (1)
+  for (j = 0; (i = getline (&line, &bufsize, fp)) > 0; )
     {
-      line = read_whole_line (fp);
-      if (line == NULL)
+      i = clean_line (line, i);
+      if (i <= 0)
+        continue; /* Ignore blank line. */
+
+      if ((j == 0) && (line[i - 1] == ']'))
         {
-        break;
+          /* Found Directory heading line.  Next non-blank line
+          is significant. */
+          j = 1;
+        }
+      else if (!strncmp (line, "Total of ", 9))
+        {
+          /* Found "Total of ..." footing line.  No valid data
+             will follow (empty directory). */
+          i = 0; /* Arrange for early exit. */
+          break;
         }
       else
         {
-          i = clean_line (line);
-          if (i <= 0)
-            {
-              xfree (line); /* Free useless line storage. */
-              continue; /* Blank line.  Keep looking. */
-            }
-          else
-            {
-              if ((j == 0) && (line[ i- 1] == ']'))
-                {
-                  /* Found Directory heading line.  Next non-blank line
-                  is significant.
-                  */
-                  j = 1;
-                }
-              else if (!strncmp (line, "Total of ", 9))
-                {
-                  /* Found "Total of ..." footing line.  No valid data
-                     will follow (empty directory).
-                  */
-                  xfree (line); /* Free useless line storage. */
-                  line = NULL; /* Arrange for early exit. */
-                  break;
-                }
-              else
-                {
-                  break; /* Must be significant data. */
-                }
-            }
-          xfree (line); /* Free useless line storage. */
+          break; /* Must be significant data. */
         }
     }
 
   /* Read remainder of file until the next blank line or EOF. */
 
-  while (line != NULL)
+  while (i > 0)
     {
       char *p;
 
@@ -788,14 +776,14 @@ ftp_parse_vms_ls (const char *file)
          what will work in a CWD command.
       */
       len = strlen (tok);
-      if (!strncasecmp((tok + (len - 4)), ".DIR", 4))
+      if (!c_strncasecmp((tok + (len - 4)), ".DIR", 4))
         {
           *(tok+ (len - 4)) = '\0'; /* Discard ".DIR". */
           cur.type  = FT_DIRECTORY;
           cur.perms = VMS_DEFAULT_PROT_DIR;
           DEBUGP (("Directory (nv)\n"));
         }
-      else if (!strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
+      else if (!c_strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
         {
           *(tok+ (len - 6)) = '\0'; /* Discard ".DIR;1". */
           cur.type  = FT_DIRECTORY;
@@ -831,9 +819,8 @@ ftp_parse_vms_ls (const char *file)
       if (tok == NULL)
         {
           DEBUGP (("Getting additional line.\n"));
-          xfree (line);
-          line = read_whole_line (fp);
-          if (!line)
+          i = getline (&line, &bufsize, fp);
+          if (i <= 0)
             {
               DEBUGP (("EOF.  Leaving listing parser.\n"));
               break;
@@ -842,14 +829,14 @@ ftp_parse_vms_ls (const char *file)
           /* Second line must begin with " ".  Otherwise, it's a first
              line (and we may be confused).
           */
+          i = clean_line (line, i);
           if (i <= 0)
             {
               /* Blank line.  End of significant file listing. */
               DEBUGP (("Blank line.  Leaving listing parser.\n"));
-              xfree (line); /* Free useless line storage. */
               break;
             }
-          else if (line[ 0] != ' ')
+          else if (line[0] != ' ')
             {
               DEBUGP (("Non-blank in column 1.  Must be a new file name?\n"));
               continue;
@@ -861,7 +848,6 @@ ftp_parse_vms_ls (const char *file)
                 {
                   /* Unexpected non-empty but apparently blank line. */
                   DEBUGP (("Null token.  Leaving listing parser.\n"));
-                  xfree (line); /* Free useless line storage. */
                   break;
                 }
             }
@@ -900,7 +886,7 @@ ftp_parse_vms_ls (const char *file)
                        (sizeof( date_str)- strlen (date_str) - 1));
               DEBUGP (("Date time: >%s<\n", date_str));
             }
-          else if (strchr ( tok, '[') != NULL)
+          else if (strchr (tok, '[') != NULL)
             {
               /* Owner.  (Ignore.) */
               DEBUGP (("Owner.\n"));
@@ -910,7 +896,8 @@ ftp_parse_vms_ls (const char *file)
               /* Protections (permissions). */
               perms = 0;
               j = 0;
-              for (i = 0; i < strlen( tok); i++)
+              /*FIXME: Should not be using the variable like this. */
+              for (i = 0; i < (int) strlen(tok); i++)
                 {
                   switch (tok[ i])
                     {
@@ -1004,21 +991,19 @@ ftp_parse_vms_ls (const char *file)
           l->next = NULL;
         }
 
-      /* Free old line storage.  Read a new line. */
-      xfree (line);
-      line = read_whole_line (fp);
-      if (line != NULL)
+      i = getline (&line, &bufsize, fp);
+      if (i > 0)
         {
-          i = clean_line (line);
+          i = clean_line (line, i);
           if (i <= 0)
-	    {
+            {
               /* Blank line.  End of significant file listing. */
-              xfree (line); /* Free useless line storage. */
-	      break;
-	    }
+              break;
+            }
         }
     }
 
+  xfree (line);
   fclose (fp);
   return dir;
 }
@@ -1067,7 +1052,7 @@ Unsupported listing type, trying Unix listing parser.\n"));
       return ftp_parse_unix_ls (file, 0);
     }
 }
-
+
 /* Stuff for creating FTP index. */
 
 /* The function creates an HTML index containing references to given
@@ -1104,7 +1089,7 @@ ftp_index (const char *file, struct url *u, struct fileinfo *f)
       else
         upwd = concat_strings (tmpu, "@", (char *) 0);
       xfree (tmpu);
-      xfree_null (tmpp);
+      xfree (tmpp);
     }
   else
     upwd = xstrdup ("");

src/ftp.c

@@ -1,6 +1,6 @@
 /* File Transfer Protocol support.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -49,6 +49,8 @@ as that of the covered work.  */
 #include "netrc.h"
 #include "convert.h"            /* for downloaded_file */
 #include "recur.h"              /* for INFINITE_RECURSION */
+#include "warc.h"
+#include "c-strcase.h"
 
 #ifdef __VMS
 # include "vms.h"
@@ -69,12 +71,12 @@ typedef struct
   int csock;                    /* control connection socket */
   double dltime;                /* time of the download in msecs */
   enum stype rs;                /* remote system reported by ftp server */
+  enum ustype rsu;              /* when rs is ST_UNIX, here there are more details */
   char *id;                     /* initial directory */
   char *target;                 /* target file name */
   struct url *proxy;            /* FTWK-style proxy */
 } ccon;
 
-extern int numurls;
 
 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
    the string S, and return the number converted to wgint, if found, 0
@@ -100,7 +102,7 @@ ftp_expected_bytes (const char *s)
         return 0;
       if (c_tolower (*s) != 'b')
         continue;
-      if (strncasecmp (s, "byte", 4))
+      if (c_strncasecmp (s, "byte", 4))
         continue;
       else
         break;
@@ -219,13 +221,13 @@ print_length (wgint size, wgint start, bool authoritative)
 {
   logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
   if (size >= 1024)
-    logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
+    logprintf (LOG_VERBOSE, " (%s)", human_readable (size, 10, 1));
   if (start > 0)
     {
       if (size - start >= 1024)
         logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
                    number_to_static_string (size - start),
-                   human_readable (size - start));
+                   human_readable (size - start, 10, 1));
       else
         logprintf (LOG_VERBOSE, _(", %s remaining"),
                    number_to_static_string (size - start));
@@ -237,25 +239,28 @@ static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
 
 /* Retrieves a file with denoted parameters through opening an FTP
    connection to the server.  It always closes the data connection,
-   and closes the control connection in case of error.  */
+   and closes the control connection in case of error.  If warc_tmp
+   is non-NULL, the downloaded data will be written there as well.  */
 static uerr_t
 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
-        wgint restval, ccon *con, int count)
+        wgint restval, ccon *con, int count, wgint *last_expected_bytes,
+        FILE *warc_tmp)
 {
   int csock, dtsock, local_sock, res;
   uerr_t err = RETROK;          /* appease the compiler */
   FILE *fp;
-  char *user, *passwd, *respline;
-  char *tms;
-  const char *tmrate;
+  char *respline, *tms;
+  const char *user, *passwd, *tmrate;
   int cmd = con->cmd;
   bool pasv_mode_open = false;
   wgint expected_bytes = 0;
   bool got_expected_bytes = false;
   bool rest_failed = false;
   int flags;
-  wgint rd_size;
+  wgint rd_size, previous_rd_size = 0;
   char type_char;
+  bool try_again;
+  bool list_a_used = false;
 
   assert (con != NULL);
   assert (con->target != NULL);
@@ -287,13 +292,6 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
     {
       char    *host = con->proxy ? con->proxy->host : u->host;
       int      port = con->proxy ? con->proxy->port : u->port;
-      char *logname = user;
-
-      if (con->proxy)
-        {
-          /* If proxy is in use, log in as username@target-site. */
-          logname = concat_strings (user, "@", u->host, (char *) 0);
-        }
 
       /* Login to the server: */
 
@@ -301,20 +299,10 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
 
       csock = connect_to_host (host, port);
       if (csock == E_HOST)
-        {
-          if (con->proxy)
-            xfree (logname);
-
           return HOSTERR;
-        }
       else if (csock < 0)
-        {
-          if (con->proxy)
-            xfree (logname);
-
           return (retryable_socket_connect_error (errno)
                   ? CONERROR : CONIMPOSSIBLE);
-        }
 
       if (cmd & LEAVE_PENDING)
         con->csock = csock;
@@ -326,10 +314,15 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
                  quotearg_style (escape_quoting_style, user));
       if (opt.server_response)
         logputs (LOG_ALWAYS, "\n");
-      err = ftp_login (csock, logname, passwd);
-
       if (con->proxy)
-        xfree (logname);
+        {
+          /* If proxy is in use, log in as username@target-site. */
+          char *logname = concat_strings (user, "@", u->host, (char *) 0);
+          err = ftp_login (csock, logname, passwd);
+          xfree (logname);
+        }
+      else
+        err = ftp_login (csock, user, passwd);
 
       /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
       switch (err)
@@ -376,7 +369,7 @@ Error in server response, closing control connection.\n"));
       /* Third: Get the system type */
       if (!opt.server_response)
         logprintf (LOG_VERBOSE, "==> SYST ... ");
-      err = ftp_syst (csock, &con->rs);
+      err = ftp_syst (csock, &con->rs, &con->rsu);
       /* FTPRERR */
       switch (err)
         {
@@ -401,6 +394,44 @@ Error in server response, closing control connection.\n"));
       if (!opt.server_response && err != FTPSRVERR)
         logputs (LOG_VERBOSE, _("done.    "));
 
+      /* 2013-10-17 Andrea Urbani (matfanjol)
+         According to the system type I choose which
+         list command will be used.
+         If I don't know that system, I will try, the
+         first time of each session, "LIST -a" and
+         "LIST". (see __LIST_A_EXPLANATION__ below) */
+      switch (con->rs)
+        {
+        case ST_VMS:
+          /* About ST_VMS there is an old note:
+             2008-01-29  SMS.  For a VMS FTP server, where "LIST -a" may not
+             fail, but will never do what is desired here,
+             skip directly to the simple "LIST" command
+             (assumed to be the last one in the list).  */
+          DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
+          con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+          con->st |= AVOID_LIST_A;
+          break;
+        case ST_UNIX:
+          if (con->rsu == UST_MULTINET)
+            {
+              DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
+                       "as standard list command\n"));
+              con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+              con->st |= AVOID_LIST_A;
+            }
+          else if (con->rsu == UST_TYPE_L8)
+            {
+              DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
+                       "as standard list command\n"));
+              con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+              con->st |= AVOID_LIST;
+            }
+          break;
+        default:
+          break;
+        }
+
       /* Fourth: Find the initial ftp directory */
 
       if (!opt.server_response)
@@ -418,7 +449,7 @@ Error in server response, closing control connection.\n"));
           return err;
         case FTPSRVERR :
           /* PWD unsupported -- assume "/". */
-          xfree_null (con->id);
+          xfree (con->id);
           con->id = xstrdup ("/");
           break;
         case FTPOK:
@@ -512,7 +543,7 @@ Error in server response, closing control connection.\n"));
         logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
       else
         {
-          char *targ = NULL;
+          const char *targ = NULL;
           int cwd_count;
           int cwd_end;
           int cwd_start;
@@ -627,16 +658,16 @@ Error in server response, closing control connection.\n"));
              The VMS restriction may be relaxed when the squirrely code
              above is reformed.
           */
-	  if ((con->rs == ST_VMS) && (target[0] != '/'))
-	    {
-	      cwd_start = 0;
-	      DEBUGP (("Using two-step CWD for relative path.\n"));
-	    }
-	  else
-	    {
+          if ((con->rs == ST_VMS) && (target[0] != '/'))
+            {
+              cwd_start = 0;
+              DEBUGP (("Using two-step CWD for relative path.\n"));
+            }
+          else
+            {
               /* Go straight to the target. */
-	      cwd_start = 1;
-	    }
+              cwd_start = 1;
+            }
 
           /* At least one VMS FTP server (TCPware V5.6-2) can switch to
              a UNIX emulation mode when given a UNIX-like directory
@@ -654,10 +685,10 @@ Error in server response, closing control connection.\n"));
              Unlike the rest of this block, this particular behavior
              _is_ VMS-specific, so it gets its own VMS test.
           */
-	  if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
+          if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
             {
               cwd_end = 3;
-	      DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
+              DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
             }
           else
             {
@@ -667,76 +698,80 @@ Error in server response, closing control connection.\n"));
           /* 2004-09-20 SMS. */
           /* Sorry about the deviant indenting.  Laziness. */
 
-	  for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
-	{
-          switch (cwd_count)
+          for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
             {
-              case 0:
-	        /* Step one (optional): Go to the initial directory,
-	           exactly as reported by the server.
-	        */
-	        targ = con->id;
-                break;
+              switch (cwd_count)
+                {
+                  case 0:
+                    /* Step one (optional): Go to the initial directory,
+                       exactly as reported by the server.
+                    */
+                    targ = con->id;
+                    break;
 
-              case 1:
-	        /* Step two: Go to the target directory.  (Absolute or
-	           relative will work now.)
-	        */
-	        targ = target;
-                break;
+                  case 1:
+                    /* Step two: Go to the target directory.  (Absolute or
+                       relative will work now.)
+                    */
+                    targ = target;
+                    break;
 
-              case 2:
-                /* Step three (optional): "CWD []" to restore server
-                   VMS-ness.
-                */
-                targ = "[]";
-                break;
+                  case 2:
+                    /* Step three (optional): "CWD []" to restore server
+                       VMS-ness.
+                    */
+                    targ = "[]";
+                    break;
 
-              default:
-                /* Can't happen. */
-                assert (1);
-	    }
+                  default:
+                    logprintf (LOG_ALWAYS, _("Logically impossible section reached in getftp()"));
+                    logprintf (LOG_ALWAYS, _("cwd_count: %d\ncwd_start: %d\ncwd_end: %d\n"),
+                                             cwd_count, cwd_start, cwd_end);
+                    abort ();
+                }
 
-          if (!opt.server_response)
-            logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
-                       quotearg_style (escape_quoting_style, target));
-          err = ftp_cwd (csock, targ);
-          /* FTPRERR, WRITEFAILED, FTPNSFOD */
-          switch (err)
-            {
-            case FTPRERR:
-              logputs (LOG_VERBOSE, "\n");
-              logputs (LOG_NOTQUIET, _("\
+              if (!opt.server_response)
+                logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
+                           quotearg_style (escape_quoting_style, target));
+
+              err = ftp_cwd (csock, targ);
+
+              /* FTPRERR, WRITEFAILED, FTPNSFOD */
+              switch (err)
+                {
+                  case FTPRERR:
+                    logputs (LOG_VERBOSE, "\n");
+                    logputs (LOG_NOTQUIET, _("\
 Error in server response, closing control connection.\n"));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case WRITEFAILED:
-              logputs (LOG_VERBOSE, "\n");
-              logputs (LOG_NOTQUIET,
-                       _("Write failed, closing control connection.\n"));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case FTPNSFOD:
-              logputs (LOG_VERBOSE, "\n");
-              logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
-                         quote (u->dir));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case FTPOK:
-              break;
-            default:
-              abort ();
-            }
-          if (!opt.server_response)
-            logputs (LOG_VERBOSE, _("done.\n"));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case WRITEFAILED:
+                    logputs (LOG_VERBOSE, "\n");
+                    logputs (LOG_NOTQUIET,
+                             _("Write failed, closing control connection.\n"));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case FTPNSFOD:
+                    logputs (LOG_VERBOSE, "\n");
+                    logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
+                               quote (u->dir));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case FTPOK:
+                    break;
+                  default:
+                    abort ();
+                }
 
-        } /* for */
+              if (!opt.server_response)
+                logputs (LOG_VERBOSE, _("done.\n"));
+
+            } /* for */
 
           /* 2004-09-20 SMS. */
-          /* End of deviant indenting. */
 
         } /* else */
     }
@@ -772,8 +807,12 @@ Error in server response, closing control connection.\n"));
           abort ();
         }
         if (!opt.server_response)
-          logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
-                     number_to_static_string (expected_bytes));
+          {
+            logprintf (LOG_VERBOSE, "%s\n",
+                    expected_bytes ?
+                    number_to_static_string (expected_bytes) :
+                    _("done.\n"));
+          }
     }
 
   if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
@@ -786,6 +825,9 @@ Error in server response, closing control connection.\n"));
       return RETRFINISHED;
     }
 
+  do
+  {
+  try_again = false;
   /* If anything is to be retrieved, PORT (or PASV) must be sent.  */
   if (cmd & (DO_LIST | DO_RETR))
     {
@@ -949,42 +991,41 @@ Error in server response, closing control connection.\n"));
   if (cmd & DO_RETR)
     {
       /* If we're in spider mode, don't really retrieve anything except
-	 the directory listing and verify whether the given "file" exists.  */
+         the directory listing and verify whether the given "file" exists.  */
       if (opt.spider)
         {
-	  bool exists = false;
-	  uerr_t res;
-	  struct fileinfo *f;
-	  res = ftp_get_listing (u, con, &f);
-	  /* Set the DO_RETR command flag again, because it gets unset when
-	     calling ftp_get_listing() and would otherwise cause an assertion
-	     failure earlier on when this function gets repeatedly called
-	     (e.g., when recursing).  */
-	  con->cmd |= DO_RETR;
-	  if (res == RETROK)
-	    {
-	      while (f)
-		{
-		  if (!strcmp (f->name, u->file))
-		    {
-		      exists = true;
-		      break;
-		    }
-		  f = f->next;
-		}
+          bool exists = false;
+          struct fileinfo *f;
+          uerr_t _res = ftp_get_listing (u, con, &f);
+          /* Set the DO_RETR command flag again, because it gets unset when
+             calling ftp_get_listing() and would otherwise cause an assertion
+             failure earlier on when this function gets repeatedly called
+             (e.g., when recursing).  */
+          con->cmd |= DO_RETR;
+          if (_res == RETROK)
+            {
+              while (f)
+                {
+                  if (!strcmp (f->name, u->file))
+                    {
+                      exists = true;
+                      break;
+                    }
+                  f = f->next;
+                }
               if (exists)
                 {
                   logputs (LOG_VERBOSE, "\n");
                   logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
                              quote (u->file));
                 }
-	      else
+              else
                 {
-		  logputs (LOG_VERBOSE, "\n");
-		  logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
-			     quote (u->file));
-		}
-	    }
+                  logputs (LOG_VERBOSE, "\n");
+                  logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
+                             quote (u->file));
+                }
+            }
           fd_close (csock);
           con->csock = -1;
           fd_close (dtsock);
@@ -1042,7 +1083,7 @@ Error in server response, closing control connection.\n"));
         logputs (LOG_VERBOSE, _("done.\n"));
 
       if (! got_expected_bytes)
-        expected_bytes = ftp_expected_bytes (ftp_last_respline);
+        expected_bytes = *last_expected_bytes;
     } /* do retrieve */
 
   if (cmd & DO_LIST)
@@ -1052,7 +1093,8 @@ Error in server response, closing control connection.\n"));
       /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
          without arguments is better than `LIST .'; confirmed by
          RFC959.  */
-      err = ftp_list (csock, NULL, con->rs);
+      err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);
+
       /* FTPRERR, WRITEFAILED */
       switch (err)
         {
@@ -1090,7 +1132,7 @@ Error in server response, closing control connection.\n"));
         logputs (LOG_VERBOSE, _("done.\n"));
 
       if (! got_expected_bytes)
-        expected_bytes = ftp_expected_bytes (ftp_last_respline);
+        expected_bytes = *last_expected_bytes;
     } /* cmd & DO_LIST */
 
   if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
@@ -1122,7 +1164,7 @@ Error in server response, closing control connection.\n"));
   /* Open the file -- if output_stream is set, use it instead.  */
 
   /* 2005-04-17 SMS.
-     Note that having the output_stream ("-O") file opened in main()
+     Note that having the output_stream ("-O") file opened in main
      (main.c) rather limits the ability in VMS to open the file
      differently for ASCII versus binary FTP here.  (Of course, doing it
      there allows a open failure to be detected immediately, without first
@@ -1152,13 +1194,25 @@ Error in server response, closing control connection.\n"));
    Elsewhere, define a constant "binary" flag.
    Isn't it nice to have distinct text and binary file types?
 */
-# define BIN_TYPE_TRANSFER (type_char != 'A')
+/* 2011-09-30 SMS.
+   Added listing files to the set of non-"binary" (text, Stream_LF)
+   files.  (Wget works either way, but other programs, like, say, text
+   editors, work better on listing files which have text attributes.)
+   Now we use "binary" attributes for a binary ("IMAGE") transfer,
+   unless "--ftp-stmlf" was specified, and we always use non-"binary"
+   (text, Stream_LF) attributes for a listing file, or for an ASCII
+   transfer.
+   Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
+   fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
+*/
 #ifdef __VMS
+# define BIN_TYPE_TRANSFER (type_char != 'A')
+# define BIN_TYPE_FILE \
+   ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
-# define BIN_TYPE_FILE (BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
 #else /* def __VMS */
-# define BIN_TYPE_FILE 1
+# define BIN_TYPE_FILE true
 #endif /* def __VMS [else] */
 
       if (restval && !(con->cmd & DO_LIST))
@@ -1182,21 +1236,20 @@ Error in server response, closing control connection.\n"));
         }
       else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
                || opt.output_document || count > 0)
-        {	  
-	  if (opt.unlink && file_exists_p (con->target))
-	    {
-	      int res = unlink (con->target);
-	      if (res < 0)
-		{
-		  logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
-			     strerror (errno));
-		  fd_close (csock);
-		  con->csock = -1;
-		  fd_close (dtsock);
-		  fd_close (local_sock);
-		  return UNLINKERR;
-		}
-	    }
+        {
+          if (opt.unlink && file_exists_p (con->target))
+            {
+              if (unlink (con->target) < 0)
+                {
+                  logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
+                    strerror (errno));
+                    fd_close (csock);
+                    con->csock = -1;
+                    fd_close (dtsock);
+                    fd_close (local_sock);
+                    return UNLINKERR;
+                }
+            }
 
 #ifdef __VMS
           int open_id;
@@ -1217,7 +1270,7 @@ Error in server response, closing control connection.\n"));
         }
       else
         {
-          fp = fopen_excl (con->target, true);
+          fp = fopen_excl (con->target, BIN_TYPE_FILE);
           if (!fp && errno == EEXIST)
             {
               /* We cannot just invent a new name and use it (which is
@@ -1260,9 +1313,9 @@ Error in server response, closing control connection.\n"));
   if (restval && rest_failed)
     flags |= rb_skip_startpos;
   rd_size = 0;
-  res = fd_read_body (dtsock, fp,
+  res = fd_read_body (con->target, dtsock, fp,
                       expected_bytes ? expected_bytes - restval : 0,
-                      restval, &rd_size, qtyread, &con->dltime, flags);
+                      restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
 
   tms = datetime_str (time (NULL));
   tmrate = retr_rate (rd_size, con->dltime);
@@ -1273,15 +1326,18 @@ Error in server response, closing control connection.\n"));
   if (!output_stream || con->cmd & DO_LIST)
     fclose (fp);
 
-  /* If fd_read_body couldn't write to fp, bail out.  */
-  if (res == -2)
+  /* If fd_read_body couldn't write to fp or warc_tmp, bail out.  */
+  if (res == -2 || (warc_tmp != NULL && res == -3))
     {
       logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
                  con->target, strerror (errno));
       fd_close (csock);
       con->csock = -1;
       fd_close (dtsock);
-      return FWRITEERR;
+      if (res == -2)
+        return FWRITEERR;
+      else if (res == -3)
+        return WARC_TMP_FWRITEERR;
     }
   else if (res == -1)
     {
@@ -1294,6 +1350,7 @@ Error in server response, closing control connection.\n"));
 
   /* Get the server to tell us if everything is retrieved.  */
   err = ftp_response (csock, &respline);
+  *last_expected_bytes = ftp_expected_bytes (respline);
   if (err != FTPOK)
     {
       /* The control connection is decidedly closed.  Print the time
@@ -1339,8 +1396,10 @@ Error in server response, closing control connection.\n"));
     }
   /* If it was a listing, and opt.server_response is true,
      print it out.  */
-  if (opt.server_response && (con->cmd & DO_LIST))
+  if (con->cmd & DO_LIST)
     {
+      if (opt.server_response)
+        {
 /* 2005-02-25 SMS.
    Much of this work may already have been done, but repeating it should
    do no damage beyond wasting time.
@@ -1363,22 +1422,115 @@ Error in server response, closing control connection.\n"));
         logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
       else
         {
-          char *line;
-          /* The lines are being read with read_whole_line because of
+          char *line = NULL;
+          size_t bufsize = 0;
+          ssize_t len;
+
+          /* The lines are being read with getline because of
              no-buffering on opt.lfile.  */
-          while ((line = read_whole_line (fp)) != NULL)
+          while ((len = getline (&line, &bufsize, fp)) > 0)
             {
-              char *p = strchr (line, '\0');
-              while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
-                *--p = '\0';
+              while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
+                line[--len] = '\0';
               logprintf (LOG_ALWAYS, "%s\n",
                          quotearg_style (escape_quoting_style, line));
-              xfree (line);
             }
+          xfree (line);
           fclose (fp);
         }
-    } /* con->cmd & DO_LIST && server_response */
+        } /* server_response */
 
+      /* 2013-10-17 Andrea Urbani (matfanjol)
+         < __LIST_A_EXPLANATION__ >
+          After the SYST command, looks if it knows that system.
+          If yes, wget will force the use of "LIST" or "LIST -a".
+          If no, wget will try, only the first time of each session, before the
+          "LIST -a" command and after the "LIST".
+          If "LIST -a" works and returns more or equal data of the "LIST",
+          "LIST -a" will be the standard list command for all the session.
+          If "LIST -a" fails or returns less data than "LIST" (think on the case
+          of an existing file called "-a"), "LIST" will be the standard list
+          command for all the session.
+          ("LIST -a" is used to get also the hidden files)
+
+          */
+      if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
+        {
+          /* We still have to check "LIST" after the first "LIST -a" to see
+             if with "LIST" we get more data than "LIST -a", that means
+             "LIST -a" returned files/folders with "-a" name. */
+          if (con->st & AVOID_LIST_A)
+            {
+              /* LIST was used in this cycle.
+                 Let's see the result. */
+              if (rd_size > previous_rd_size)
+                {
+                  /* LIST returns more data than "LIST -a".
+                     "LIST" is the official command to use. */
+                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                  DEBUGP (("LIST returned more data than \"LIST -a\": "
+                           "I will use \"LIST\" as standard list command\n"));
+                }
+              else if (previous_rd_size > rd_size)
+                {
+                  /* "LIST -a" returned more data then LIST.
+                     "LIST -a" is the official command to use. */
+                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                  con->st |= AVOID_LIST;
+                  con->st &= ~AVOID_LIST_A;
+                  /* Sorry, please, download again the "LIST -a"... */
+                  try_again = true;
+                  DEBUGP (("LIST returned less data than \"LIST -a\": I will "
+                           "use \"LIST -a\" as standard list command\n"));
+                }
+              else
+                {
+                  /* LIST and "LIST -a" return the same data. */
+                  if (rd_size == 0)
+                    {
+                      /* Same empty data. We will check both again because
+                         we cannot check if "LIST -a" has returned an empty
+                         folder instead of a folder content. */
+                      con->st &= ~AVOID_LIST_A;
+                    }
+                  else
+                    {
+                      /* Same data, so, better to take "LIST -a" that
+                         shows also hidden files/folders (when present) */
+                      con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                      con->st |= AVOID_LIST;
+                      con->st &= ~AVOID_LIST_A;
+                      DEBUGP (("LIST returned the same amount of data of "
+                               "\"LIST -a\": I will use \"LIST -a\" as standard "
+                               "list command\n"));
+                    }
+                }
+            }
+          else
+            {
+              /* In this cycle "LIST -a" should being used. Is it true? */
+              if (list_a_used)
+                {
+                  /* Yes, it is.
+                     OK, let's save the amount of data and try again
+                     with LIST */
+                  previous_rd_size = rd_size;
+                  try_again = true;
+                  con->st |= AVOID_LIST_A;
+                }
+              else
+                {
+                  /* No: something happens and LIST was used.
+                     This means "LIST -a" raises an error. */
+                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                  con->st |= AVOID_LIST_A;
+                  DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
+                           "as standard list command\n"));
+                }
+            }
+        }
+    }
+  } while (try_again);
   return RETRFINISHED;
 }
 
@@ -1397,6 +1549,12 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   uerr_t err;
   struct_stat st;
 
+  /* Declare WARC variables. */
+  bool warc_enabled = (opt.warc_filename != NULL);
+  FILE *warc_tmp = NULL;
+  ip_address *warc_ip = NULL;
+  wgint last_expected_bytes = 0;
+
   /* Get the target, and set the name for the message accordingly. */
   if ((f == NULL) && (con->target))
     {
@@ -1406,6 +1564,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   else
     {
       /* URL-derived file.  Consider "-O file" name. */
+      xfree (con->target);
       con->target = url_file_name (u, NULL);
       if (!opt.output_document)
         locf = con->target;
@@ -1415,7 +1574,12 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
 
   /* If the output_document was given, then this check was already done and
      the file didn't exist. Hence the !opt.output_document */
-  if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
+
+  /* If we receive .listing file it is necessary to determine system type of the ftp
+     server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
+     order to establish connection with the server and get system type. */
+  if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
+      && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
     {
       logprintf (LOG_VERBOSE,
                  _("File %s already there; not retrieving.\n"), quote (con->target));
@@ -1460,9 +1624,26 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
             con->cmd |= DO_CWD;
         }
 
+      /* For file RETR requests, we can write a WARC record.
+         We record the file contents to a temporary file. */
+      if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
+        {
+          warc_tmp = warc_tempfile ();
+          if (warc_tmp == NULL)
+            return WARC_TMP_FOPENERR;
+
+          if (!con->proxy && con->csock != -1)
+            {
+              warc_ip = (ip_address *) alloca (sizeof (ip_address));
+              socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
+            }
+        }
+
       /* Decide whether or not to restart.  */
       if (con->cmd & DO_LIST)
         restval = 0;
+      else if (opt.start_pos >= 0)
+        restval = opt.start_pos;
       else if (opt.always_rest
           && stat (locf, &st) == 0
           && S_ISREG (st.st_mode))
@@ -1497,7 +1678,11 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
         len = f->size;
       else
         len = 0;
-      err = getftp (u, len, &qtyread, restval, con, count);
+
+      /* If we are working on a WARC record, getftp should also write
+         to the warc_tmp file. */
+      err = getftp (u, len, &qtyread, restval, con, count, &last_expected_bytes,
+                    warc_tmp);
 
       if (con->csock == -1)
         con->st &= ~DONE_CWD;
@@ -1508,8 +1693,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
         {
         case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
         case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
-        case UNLINKERR:
+        case UNLINKERR: case WARC_TMP_FWRITEERR:
           /* Fatal errors, give up.  */
+          if (warc_tmp != NULL)
+            fclose (warc_tmp);
           return err;
         case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
         case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
@@ -1520,7 +1707,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
           if (err == FOPEN_EXCL_ERR)
             {
               /* Re-determine the file name. */
-              xfree_null (con->target);
+              xfree (con->target);
               con->target = url_file_name (u, NULL);
               locf = con->target;
             }
@@ -1577,7 +1764,20 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
           xfree (hurl);
         }
 
-      if ((con->cmd & DO_LIST))
+      if (warc_enabled && (con->cmd & DO_RETR))
+        {
+          /* Create and store a WARC resource record for the retrieved file. */
+          bool warc_res;
+
+          warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
+                                                  warc_ip, NULL, warc_tmp, -1);
+          if (! warc_res)
+            return WARC_ERR;
+
+          /* warc_write_resource_record has also closed warc_tmp. */
+        }
+
+      if (con->cmd & DO_LIST)
         /* This is a directory listing file. */
         {
           if (!opt.remove_listing)
@@ -1880,8 +2080,10 @@ Already have correct symlink %s -> %s\n\n"),
 
       set_local_file (&actual_target, con->target);
 
-      /* If downloading a plain file, set valid (non-zero) permissions. */
-      if (dlthis && (actual_target != NULL) && (f->type == FT_PLAINFILE))
+      /* If downloading a plain file, and the user requested it, then
+         set valid (non-zero) permissions. */
+      if (dlthis && (actual_target != NULL) &&
+       (f->type == FT_PLAINFILE) && opt.preserve_perm)
         {
           if (f->perms)
             chmod (actual_target, f->perms);
@@ -1914,7 +2116,9 @@ Already have correct symlink %s -> %s\n\n"),
       xfree (ofile);
 
       /* Break on fatals.  */
-      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
+      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
+          || err == WARC_ERR || err == WARC_TMP_FOPENERR
+          || err == WARC_TMP_FWRITEERR)
         break;
       con->cmd &= ~ (DO_CWD | DO_LOGIN);
       f = f->next;
@@ -2012,6 +2216,29 @@ has_insecure_name_p (const char *s)
   return false;
 }
 
+/* Test if the file node is invalid. This can occur due to malformed or
+ * maliciously crafted listing files being returned by the server.
+ *
+ * Currently, this function only tests if there are multiple entries in the
+ * listing file by the same name. However this function can be expanded as more
+ * such illegal listing formats are discovered. */
+static bool
+is_invalid_entry (struct fileinfo *f)
+{
+  struct fileinfo *cur = f;
+  char *f_name = f->name;
+
+  /* If the node we're currently checking has a duplicate later, we eliminate
+   * the current node and leave the next one intact. */
+  while (cur->next)
+    {
+      cur = cur->next;
+      if (strcmp(f_name, cur->name) == 0)
+          return true;
+    }
+  return false;
+}
+
 /* A near-top-level function to retrieve the files in a directory.
    The function calls ftp_get_listing, to get a linked list of files.
    Then it weeds out the file names that do not match the pattern.
@@ -2049,11 +2276,11 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
             f = f->next;
         }
     }
-  /* Remove all files with possible harmful names */
+  /* Remove all files with possible harmful names or invalid entries. */
   f = start;
   while (f)
     {
-      if (has_insecure_name_p (f->name))
+      if (has_insecure_name_p (f->name) || is_invalid_entry (f))
         {
           logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
                      quote (f->name));
@@ -2125,7 +2352,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
   if (start)
     {
       /* Just get everything.  */
-      ftp_retrieve_list (u, start, con);
+      res = ftp_retrieve_list (u, start, con);
     }
   else
     {
@@ -2154,8 +2381,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
   if (opt.quota && total_downloaded_bytes > opt.quota)
     return QUOTEXC;
   else
-    /* #### Should we return `res' here?  */
-    return RETROK;
+    return res;
 }
 
 /* The wrapper that calls an appropriate routine according to contents
@@ -2233,11 +2459,11 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
             file_part = u->path;
           ispattern = has_wildcards_p (file_part);
         }
-      if (ispattern || recursive || opt.timestamping)
+      if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
         {
           /* ftp_retrieve_glob is a catch-all function that gets called
-             if we need globbing, time-stamping or recursion.  Its
-             third argument is just what we really need.  */
+             if we need globbing, time-stamping, recursion or preserve
+             permissions.  Its third argument is just what we really need.  */
           res = ftp_retrieve_glob (u, &con,
                                    ispattern ? GLOB_GLOBALL : GLOB_GETONE);
         }
@@ -2251,10 +2477,8 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
   /* If a connection was left, quench it.  */
   if (con.csock != -1)
     fd_close (con.csock);
-  xfree_null (con.id);
-  con.id = NULL;
-  xfree_null (con.target);
-  con.target = NULL;
+  xfree (con.id);
+  xfree (con.target);
   return res;
 }
 
@@ -2268,7 +2492,7 @@ delelement (struct fileinfo *f, struct fileinfo **start)
   struct fileinfo *next = f->next;
 
   xfree (f->name);
-  xfree_null (f->linkto);
+  xfree (f->linkto);
   xfree (f);
 
   if (next)

src/ftp.h

@@ -45,7 +45,13 @@ enum stype
   ST_OTHER
 };
 
-extern char ftp_last_respline[];
+/* Extensions of the ST_UNIX */
+enum ustype
+{
+  UST_TYPE_L8,
+  UST_MULTINET,
+  UST_OTHER
+};
 
 uerr_t ftp_response (int, char **);
 uerr_t ftp_login (int, const char *, const char *);
@@ -61,8 +67,8 @@ uerr_t ftp_type (int, int);
 uerr_t ftp_cwd (int, const char *);
 uerr_t ftp_retr (int, const char *);
 uerr_t ftp_rest (int, wgint);
-uerr_t ftp_list (int, const char *, enum stype);
-uerr_t ftp_syst (int, enum stype *);
+uerr_t ftp_list (int, const char *, bool, bool, bool *);
+uerr_t ftp_syst (int, enum stype *, enum ustype *);
 uerr_t ftp_pwd (int, char **);
 uerr_t ftp_size (int, const char *, wgint *);
 
@@ -98,34 +104,49 @@ enum parsetype
 /* Information about one filename in a linked list.  */
 struct fileinfo
 {
-  enum ftype type;		/* file type */
-  char *name;			/* file name */
-  wgint size;			/* file size */
-  long tstamp;			/* time-stamp */
-  enum parsetype ptype; /* time parsing */
-  int perms;			/* file permissions */
-  char *linkto;			/* link to which file points */
-  struct fileinfo *prev;	/* previous... */
-  struct fileinfo *next;	/* ...and next structure. */
+  enum ftype type;          /* file type */
+  char *name;               /* file name */
+  wgint size;               /* file size */
+  long tstamp;              /* time-stamp */
+  enum parsetype ptype;     /* time parsing */
+  int perms;                /* file permissions */
+  char *linkto;             /* link to which file points */
+  struct fileinfo *prev;    /* previous... */
+  struct fileinfo *next;    /* ...and next structure. */
 };
 
 /* Commands for FTP functions.  */
 enum wget_ftp_command
 {
-  DO_LOGIN      = 0x0001,	/* Connect and login to the server.  */
-  DO_CWD        = 0x0002,	/* Change current directory.  */
-  DO_RETR       = 0x0004,	/* Retrieve the file.  */
-  DO_LIST       = 0x0008,	/* Retrieve the directory list.  */
-  LEAVE_PENDING = 0x0010	/* Do not close the socket.  */
+  DO_LOGIN      = 0x0001,   /* Connect and login to the server.  */
+  DO_CWD        = 0x0002,   /* Change current directory.  */
+  DO_RETR       = 0x0004,   /* Retrieve the file.  */
+  DO_LIST       = 0x0008,   /* Retrieve the directory list.  */
+  LEAVE_PENDING = 0x0010    /* Do not close the socket.  */
 };
 
 enum wget_ftp_fstatus
 {
-  NOTHING       = 0x0000,	/* Nothing done yet.  */
-  ON_YOUR_OWN   = 0x0001,	/* The ftp_loop_internal sets the
-				   defaults.  */
-  DONE_CWD      = 0x0002	/* The current working directory is
-				   correct.  */
+  NOTHING       = 0x0000,   /* Nothing done yet.  */
+  ON_YOUR_OWN   = 0x0001,   /* The ftp_loop_internal sets the
+                               defaults.  */
+  DONE_CWD      = 0x0002,   /* The current working directory is
+                               correct.  */
+
+  /* 2013-10-17 Andrea Urbani (matfanjol)
+     For more information about the following entries, please,
+     look at ftp.c, function getftp, text "__LIST_A_EXPLANATION__". */
+  AVOID_LIST_A  = 0x0004,   /* It tells us if during this
+                               session we have to avoid the use
+                               of "LIST -a".*/
+  AVOID_LIST    = 0x0008,   /* It tells us if during this
+                               session we have to avoid to use
+                               "LIST". */
+  LIST_AFTER_LIST_A_CHECK_DONE  = 0x0010
+                            /* It tells us if we have already
+                               checked "LIST" after the first
+                               "LIST -a" to handle the case of
+                               file/folders named "-a". */
 };
 
 struct fileinfo *ftp_parse_ls (const char *, const enum stype);

src/gettext.h

@@ -1,272 +0,0 @@
-/* Convenience header for conditional use of GNU <libintl.h>.
-   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001, 2002, 2004, 2005,
-   2006, 2009, 2010, 2011 Free Software Foundation, Inc.
-
-   This program is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 2, or (at your option)
-   any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU General Public
-   License along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-   USA.  */
-
-#ifndef _LIBGETTEXT_H
-#define _LIBGETTEXT_H 1
-
-/* NLS can be disabled through the configure --disable-nls option.  */
-#if ENABLE_NLS
-
-/* Get declarations of GNU message catalog functions.  */
-# include <libintl.h>
-
-/* You can set the DEFAULT_TEXT_DOMAIN macro to specify the domain used by
-   the gettext() and ngettext() macros.  This is an alternative to calling
-   textdomain(), and is useful for libraries.  */
-# ifdef DEFAULT_TEXT_DOMAIN
-#  undef gettext
-#  define gettext(Msgid) \
-     dgettext (DEFAULT_TEXT_DOMAIN, Msgid)
-#  undef ngettext
-#  define ngettext(Msgid1, Msgid2, N) \
-     dngettext (DEFAULT_TEXT_DOMAIN, Msgid1, Msgid2, N)
-# endif
-
-#else
-
-/* Solaris /usr/include/locale.h includes /usr/include/libintl.h, which
-   chokes if dcgettext is defined as a macro.  So include it now, to make
-   later inclusions of <locale.h> a NOP.  We don't include <libintl.h>
-   as well because people using "gettext.h" will not include <libintl.h>,
-   and also including <libintl.h> would fail on SunOS 4, whereas <locale.h>
-   is OK.  */
-#if defined(__sun)
-# include <locale.h>
-#endif
-
-/* Many header files from the libstdc++ coming with g++ 3.3 or newer include
-   <libintl.h>, which chokes if dcgettext is defined as a macro.  So include
-   it now, to make later inclusions of <libintl.h> a NOP.  */
-#if defined(__cplusplus) && defined(__GNUG__) && (__GNUC__ >= 3)
-# include <cstdlib>
-# if (__GLIBC__ >= 2) || _GLIBCXX_HAVE_LIBINTL_H
-#  include <libintl.h>
-# endif
-#endif
-
-/* Disabled NLS.
-   The casts to 'const char *' serve the purpose of producing warnings
-   for invalid uses of the value returned from these functions.
-   On pre-ANSI systems without 'const', the config.h file is supposed to
-   contain "#define const".  */
-# define gettext(Msgid) ((const char *) (Msgid))
-# define dgettext(Domainname, Msgid) ((void) (Domainname), gettext (Msgid))
-# define dcgettext(Domainname, Msgid, Category) \
-    ((void) (Category), dgettext (Domainname, Msgid))
-# define ngettext(Msgid1, Msgid2, N) \
-    ((N) == 1 \
-     ? ((void) (Msgid2), (const char *) (Msgid1)) \
-     : ((void) (Msgid1), (const char *) (Msgid2)))
-# define dngettext(Domainname, Msgid1, Msgid2, N) \
-    ((void) (Domainname), ngettext (Msgid1, Msgid2, N))
-# define dcngettext(Domainname, Msgid1, Msgid2, N, Category) \
-    ((void) (Category), dngettext(Domainname, Msgid1, Msgid2, N))
-# define textdomain(Domainname) ((const char *) (Domainname))
-# define bindtextdomain(Domainname, Dirname) \
-    ((void) (Domainname), (const char *) (Dirname))
-# define bind_textdomain_codeset(Domainname, Codeset) \
-    ((void) (Domainname), (const char *) (Codeset))
-
-#endif
-
-/* A pseudo function call that serves as a marker for the automated
-   extraction of messages, but does not call gettext().  The run-time
-   translation is done at a different place in the code.
-   The argument, String, should be a literal string.  Concatenated strings
-   and other string expressions won't work.
-   The macro's expansion is not parenthesized, so that it is suitable as
-   initializer for static 'char[]' or 'const char[]' variables.  */
-#define gettext_noop(String) String
-
-/* The separator between msgctxt and msgid in a .mo file.  */
-#define GETTEXT_CONTEXT_GLUE "\004"
-
-/* Pseudo function calls, taking a MSGCTXT and a MSGID instead of just a
-   MSGID.  MSGCTXT and MSGID must be string literals.  MSGCTXT should be
-   short and rarely need to change.
-   The letter 'p' stands for 'particular' or 'special'.  */
-#ifdef DEFAULT_TEXT_DOMAIN
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#else
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#endif
-#define dpgettext(Domainname, Msgctxt, Msgid) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#define dcpgettext(Domainname, Msgctxt, Msgid, Category) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, Category)
-#ifdef DEFAULT_TEXT_DOMAIN
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#else
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#endif
-#define dnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dcnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N, Category) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, Category)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-pgettext_aux (const char *domain,
-	      const char *msg_ctxt_id, const char *msgid,
-	      int category)
-{
-  const char *translation = dcgettext (domain, msg_ctxt_id, category);
-  if (translation == msg_ctxt_id)
-    return msgid;
-  else
-    return translation;
-}
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-npgettext_aux (const char *domain,
-	       const char *msg_ctxt_id, const char *msgid,
-	       const char *msgid_plural, unsigned long int n,
-	       int category)
-{
-  const char *translation =
-    dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-  if (translation == msg_ctxt_id || translation == msgid_plural)
-    return (n == 1 ? msgid : msgid_plural);
-  else
-    return translation;
-}
-
-/* The same thing extended for non-constant arguments.  Here MSGCTXT and MSGID
-   can be arbitrary expressions.  But for string literals these macros are
-   less efficient than those above.  */
-
-#include <string.h>
-
-#define _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS \
-  (((__GNUC__ >= 3 || __GNUG__ >= 2) && !__STRICT_ANSI__) \
-   /* || __STDC_VERSION__ >= 199901L */ )
-
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-#include <stdlib.h>
-#endif
-
-#define pgettext_expr(Msgctxt, Msgid) \
-  dcpgettext_expr (NULL, Msgctxt, Msgid, LC_MESSAGES)
-#define dpgettext_expr(Domainname, Msgctxt, Msgid) \
-  dcpgettext_expr (Domainname, Msgctxt, Msgid, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcpgettext_expr (const char *domain,
-		 const char *msgctxt, const char *msgid,
-		 int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcgettext (domain, msg_ctxt_id, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-	free (msg_ctxt_id);
-#endif
-      if (translation != msg_ctxt_id)
-	return translation;
-    }
-  return msgid;
-}
-
-#define npgettext_expr(Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (NULL, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dnpgettext_expr(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (Domainname, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcnpgettext_expr (const char *domain,
-		  const char *msgctxt, const char *msgid,
-		  const char *msgid_plural, unsigned long int n,
-		  int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-	free (msg_ctxt_id);
-#endif
-      if (!(translation == msg_ctxt_id || translation == msgid_plural))
-	return translation;
-    }
-  return (n == 1 ? msgid : msgid_plural);
-}
-
-#endif /* _LIBGETTEXT_H */

src/gnutls.c

@@ -1,5 +1,5 @@
 /* SSL support via GnuTLS library.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -30,7 +30,6 @@ as that of the covered work.  */
 
 #include "wget.h"
 
-#include <assert.h>
 #include <errno.h>
 #include <unistd.h>
 #include <string.h>
@@ -45,69 +44,181 @@ as that of the covered work.  */
 #include "utils.h"
 #include "connect.h"
 #include "url.h"
+#include "ptimer.h"
+#include "hash.h"
 #include "ssl.h"
 
+#include <sys/fcntl.h>
+
 #ifdef WIN32
 # include "w32sock.h"
 #endif
 
+#include "host.h"
+
+static int
+key_type_to_gnutls_type (enum keyfile_type type)
+{
+  switch (type)
+    {
+    case keyfile_pem:
+      return GNUTLS_X509_FMT_PEM;
+    case keyfile_asn1:
+      return GNUTLS_X509_FMT_DER;
+    default:
+      abort ();
+    }
+}
+
 /* Note: some of the functions private to this file have names that
    begin with "wgnutls_" (e.g. wgnutls_read) so that they wouldn't be
    confused with actual gnutls functions -- such as the gnutls_read
    preprocessor macro.  */
 
-static gnutls_certificate_credentials credentials;
-
+static gnutls_certificate_credentials_t credentials;
 bool
-ssl_init ()
+ssl_init (void)
 {
+  /* Becomes true if GnuTLS is initialized. */
+  static bool ssl_initialized = false;
   const char *ca_directory;
   DIR *dir;
+  int ncerts = -1;
+
+  /* GnuTLS should be initialized only once. */
+  if (ssl_initialized)
+    return true;
 
   gnutls_global_init ();
   gnutls_certificate_allocate_credentials (&credentials);
-  gnutls_certificate_set_verify_flags(credentials,
-                                      GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
+  gnutls_certificate_set_verify_flags (credentials,
+                                       GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
 
-  ca_directory = opt.ca_directory ? opt.ca_directory : "/etc/ssl/certs";
+#if GNUTLS_VERSION_MAJOR >= 3
+  if (!opt.ca_directory)
+    ncerts = gnutls_certificate_set_x509_system_trust (credentials);
+#endif
 
-  dir = opendir (ca_directory);
-  if (dir == NULL)
+  /* If GnuTLS version is too old or CA loading failed, fallback to old behaviour.
+   * Also use old behaviour if the CA directory is user-provided.  */
+  if (ncerts <= 0)
     {
-      if (opt.ca_directory)
-        logprintf (LOG_NOTQUIET, _("ERROR: Cannot open directory %s.\n"),
-                   opt.ca_directory);
-    }
-  else
-    {
-      struct dirent *dent;
-      while ((dent = readdir (dir)) != NULL)
+      ca_directory = opt.ca_directory ? opt.ca_directory : "/etc/ssl/certs";
+      if ((dir = opendir (ca_directory)) == NULL)
         {
-          struct stat st;
-          char *ca_file;
-          asprintf (&ca_file, "%s/%s", ca_directory, dent->d_name);
-
-          stat (ca_file, &st);
-
-          if (S_ISREG (st.st_mode))
-            gnutls_certificate_set_x509_trust_file (credentials, ca_file,
-                                                    GNUTLS_X509_FMT_PEM);
-
-          free (ca_file);
+          if (opt.ca_directory && *opt.ca_directory)
+            logprintf (LOG_NOTQUIET, _("ERROR: Cannot open directory %s.\n"),
+                       opt.ca_directory);
         }
+      else
+        {
+          struct hash_table *inode_map = hash_table_new (196, NULL, NULL);
+          struct dirent *dent;
+          size_t dirlen = strlen(ca_directory);
+          int rc;
 
-      closedir (dir);
+          ncerts = 0;
+
+          while ((dent = readdir (dir)) != NULL)
+            {
+              struct stat st;
+              size_t ca_file_length = dirlen + strlen(dent->d_name) + 2;
+              char *ca_file = alloca(ca_file_length);
+
+              snprintf (ca_file, ca_file_length, "%s/%s", ca_directory, dent->d_name);
+              if (stat (ca_file, &st) != 0)
+                continue;
+
+              if (! S_ISREG (st.st_mode))
+                continue;
+
+              /* avoid loading the same file twice by checking the inode.  */
+              if (hash_table_contains (inode_map, (void *)(intptr_t) st.st_ino))
+                continue;
+
+              hash_table_put (inode_map, (void *)(intptr_t) st.st_ino, NULL);
+              if ((rc = gnutls_certificate_set_x509_trust_file (credentials, ca_file,
+                                                                GNUTLS_X509_FMT_PEM)) <= 0)
+                DEBUGP (("WARNING: Failed to open cert %s: (%d).\n", ca_file, rc));
+              else
+                ncerts += rc;
+            }
+
+          hash_table_destroy (inode_map);
+          closedir (dir);
+        }
     }
 
   if (opt.ca_cert)
-    gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
-                                            GNUTLS_X509_FMT_PEM);
+    {
+      int rc;
+
+      ncerts = 0;
+
+      if ((rc = gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
+                                                        GNUTLS_X509_FMT_PEM)) <= 0)
+        logprintf (LOG_NOTQUIET, _ ("ERROR: Failed to open cert %s: (%d).\n"),
+                   opt.ca_cert, rc);
+      else
+        {
+          ncerts += rc;
+          logprintf (LOG_NOTQUIET, _ ("Loaded CA certificate '%s'\n"), opt.ca_cert);
+        }
+    }
+
+  if (opt.crl_file)
+    {
+      int rc;
+
+      if ((rc = gnutls_certificate_set_x509_crl_file (credentials, opt.crl_file, GNUTLS_X509_FMT_PEM)) <= 0)
+        {
+          logprintf (LOG_NOTQUIET, _("ERROR: Failed to load CRL file '%s': (%d)\n"), opt.crl_file, rc);
+          return false;
+        }
+
+      logprintf (LOG_NOTQUIET, _ ("Loaded CRL file '%s'\n"), opt.crl_file);
+    }
+
+  DEBUGP (("Certificates loaded: %d\n", ncerts));
+
+  /* Use the private key from the cert file unless otherwise specified. */
+  if (opt.cert_file && !opt.private_key)
+    {
+      opt.private_key = opt.cert_file;
+      opt.private_key_type = opt.cert_type;
+    }
+  /* Use the cert from the private key file unless otherwise specified. */
+  if (!opt.cert_file && opt.private_key)
+    {
+      opt.cert_file = opt.private_key;
+      opt.cert_type = opt.private_key_type;
+    }
+
+  if (opt.cert_file && opt.private_key)
+    {
+      int type;
+      if (opt.private_key_type != opt.cert_type)
+        {
+          /* GnuTLS can't handle this */
+          logprintf (LOG_NOTQUIET, _("ERROR: GnuTLS requires the key and the \
+cert to be of the same type.\n"));
+        }
+
+      type = key_type_to_gnutls_type (opt.private_key_type);
+
+      gnutls_certificate_set_x509_key_file (credentials, opt.cert_file,
+                                            opt.private_key,
+                                            type);
+    }
+
+  ssl_initialized = true;
+
   return true;
 }
 
 struct wgnutls_transport_context
 {
-  gnutls_session session;       /* GnuTLS session handle */
+  gnutls_session_t session;       /* GnuTLS session handle */
   int last_error;               /* last error returned by read/write/... */
 
   /* Since GnuTLS doesn't support the equivalent to recv(...,
@@ -118,9 +229,76 @@ struct wgnutls_transport_context
   int peeklen;
 };
 
-#ifndef MIN
-# define MIN(i, j) ((i) <= (j) ? (i) : (j))
+static int
+wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout)
+{
+#ifdef F_GETFL
+  int flags = 0;
 #endif
+  int ret = 0;
+  struct ptimer *timer = NULL;
+  struct wgnutls_transport_context *ctx = arg;
+  int timed_out = 0;
+
+  if (timeout)
+    {
+#ifdef F_GETFL
+      flags = fcntl (fd, F_GETFL, 0);
+      if (flags < 0)
+        return flags;
+      if (fcntl (fd, F_SETFL, flags | O_NONBLOCK))
+        return -1;
+#else
+      /* XXX: Assume it was blocking before.  */
+      const int one = 1;
+      if (ioctl (fd, FIONBIO, &one) < 0)
+        return -1;
+#endif
+
+      timer = ptimer_new ();
+      if (timer == NULL)
+        return -1;
+    }
+
+  do
+    {
+      double next_timeout = 0;
+      if (timeout)
+        {
+          next_timeout = timeout - ptimer_measure (timer);
+          if (next_timeout < 0)
+            break;
+        }
+
+      ret = GNUTLS_E_AGAIN;
+      if (timeout == 0 || gnutls_record_check_pending (ctx->session)
+          || select_fd (fd, next_timeout, WAIT_FOR_READ))
+        {
+          ret = gnutls_record_recv (ctx->session, buf, bufsize);
+          timed_out = timeout && ptimer_measure (timer) >= timeout;
+        }
+    }
+  while (ret == GNUTLS_E_INTERRUPTED || (ret == GNUTLS_E_AGAIN && !timed_out));
+
+  if (timeout)
+    {
+      ptimer_destroy (timer);
+
+#ifdef F_GETFL
+      if (fcntl (fd, F_SETFL, flags) < 0)
+        return -1;
+#else
+      const int zero = 0;
+      if (ioctl (fd, FIONBIO, &zero) < 0)
+        return -1;
+#endif
+
+      if (timed_out && ret == GNUTLS_E_AGAIN)
+        errno = ETIMEDOUT;
+    }
+
+  return ret;
+}
 
 static int
 wgnutls_read (int fd, char *buf, int bufsize, void *arg)
@@ -140,10 +318,7 @@ wgnutls_read (int fd, char *buf, int bufsize, void *arg)
       return copysize;
     }
 
-  do
-    ret = gnutls_record_recv (ctx->session, buf, bufsize);
-  while (ret == GNUTLS_E_INTERRUPTED || ret == GNUTLS_E_AGAIN);
-
+  ret = wgnutls_read_timeout (fd, buf, bufsize, arg, opt.read_timeout);
   if (ret < 0)
     ctx->last_error = ret;
 
@@ -151,7 +326,7 @@ wgnutls_read (int fd, char *buf, int bufsize, void *arg)
 }
 
 static int
-wgnutls_write (int fd, char *buf, int bufsize, void *arg)
+wgnutls_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg)
 {
   int ret;
   struct wgnutls_transport_context *ctx = arg;
@@ -167,8 +342,12 @@ static int
 wgnutls_poll (int fd, double timeout, int wait_for, void *arg)
 {
   struct wgnutls_transport_context *ctx = arg;
-  return ctx->peeklen || gnutls_record_check_pending (ctx->session)
-    || select_fd (fd, timeout, wait_for);
+
+  if (timeout)
+    return ctx->peeklen || gnutls_record_check_pending (ctx->session)
+      || select_fd (fd, timeout, wait_for);
+  else
+    return ctx->peeklen || gnutls_record_check_pending (ctx->session);
 }
 
 static int
@@ -177,21 +356,24 @@ wgnutls_peek (int fd, char *buf, int bufsize, void *arg)
   int read = 0;
   struct wgnutls_transport_context *ctx = arg;
   int offset = MIN (bufsize, ctx->peeklen);
-  if (bufsize > sizeof ctx->peekbuf)
-    bufsize = sizeof ctx->peekbuf;
 
   if (ctx->peeklen)
-    memcpy (buf, ctx->peekbuf, offset);
+    {
+      memcpy (buf, ctx->peekbuf, offset);
+      return offset;
+    }
+
+  if (bufsize > (int) sizeof ctx->peekbuf)
+    bufsize = sizeof ctx->peekbuf;
 
   if (bufsize > offset)
     {
-      if (gnutls_record_check_pending (ctx->session) <= 0
+      if (opt.read_timeout && gnutls_record_check_pending (ctx->session) == 0
           && select_fd (fd, 0.0, WAIT_FOR_READ) <= 0)
         read = 0;
       else
-        read = gnutls_record_recv (ctx->session, buf + offset,
-                                   bufsize - offset);
-
+        read = wgnutls_read_timeout (fd, buf + offset, bufsize - offset,
+                                     ctx, opt.read_timeout);
       if (read < 0)
         {
           if (offset)
@@ -212,7 +394,7 @@ wgnutls_peek (int fd, char *buf, int bufsize, void *arg)
 }
 
 static const char *
-wgnutls_errstr (int fd, void *arg)
+wgnutls_errstr (int fd _GL_UNUSED, void *arg)
 {
   struct wgnutls_transport_context *ctx = arg;
   return gnutls_strerror (ctx->last_error);
@@ -238,33 +420,70 @@ static struct transport_implementation wgnutls_transport =
 };
 
 bool
-ssl_connect_wget (int fd)
+ssl_connect_wget (int fd, const char *hostname)
 {
+#ifdef F_GETFL
+  int flags = 0;
+#endif
   struct wgnutls_transport_context *ctx;
-  gnutls_session session;
+  gnutls_session_t session;
   int err;
+  const char *str;
+
   gnutls_init (&session, GNUTLS_CLIENT);
+
+  /* We set the server name but only if it's not an IP address. */
+  if (! is_valid_ip_address (hostname))
+    {
+      gnutls_server_name_set (session, GNUTLS_NAME_DNS, hostname,
+                              strlen (hostname));
+    }
+
   gnutls_set_default_priority (session);
   gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
 #endif
-  gnutls_transport_set_ptr (session, (gnutls_transport_ptr) FD_TO_SOCKET (fd));
+#ifdef HAVE_INTPTR_T
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) (intptr_t) FD_TO_SOCKET (fd));
+#else
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd));
+#endif
 
-  err = 0;
 #if HAVE_GNUTLS_PRIORITY_SET_DIRECT
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
+      err = gnutls_priority_set_direct (session, "NORMAL:%COMPAT:-VERS-SSL3.0", NULL);
       break;
+
     case secure_protocol_sslv2:
     case secure_protocol_sslv3:
-      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL", NULL);
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL);
       break;
+
     case secure_protocol_tlsv1:
       err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
       break;
+
+    case secure_protocol_tlsv1_1:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1", NULL);
+      break;
+
+    case secure_protocol_pfs:
+      err = gnutls_priority_set_direct (session, "PFS:-VERS-SSL3.0", NULL);
+      if (err != GNUTLS_E_SUCCESS)
+        /* fallback if PFS is not available */
+        err = gnutls_priority_set_direct (session, "NORMAL:-RSA:-VERS-SSL3.0", NULL);
+      break;
+
     default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
 #else
@@ -273,6 +492,7 @@ ssl_connect_wget (int fd)
     {
     case secure_protocol_auto:
       break;
+
     case secure_protocol_sslv2:
     case secure_protocol_sslv3:
       allowed_protocols[0] = GNUTLS_SSL3;
@@ -286,7 +506,20 @@ ssl_connect_wget (int fd)
       err = gnutls_protocol_set_priority (session, allowed_protocols);
       break;
 
+    case secure_protocol_tlsv1_1:
+      allowed_protocols[0] = GNUTLS_TLS1_1;
+      allowed_protocols[1] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      allowed_protocols[0] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
     default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
 #endif
@@ -298,10 +531,82 @@ ssl_connect_wget (int fd)
       return false;
     }
 
-  err = gnutls_handshake (session);
+  if (opt.connect_timeout)
+    {
+#ifdef F_GETFL
+      flags = fcntl (fd, F_GETFL, 0);
+      if (flags < 0)
+        return flags;
+      if (fcntl (fd, F_SETFL, flags | O_NONBLOCK))
+        return -1;
+#else
+      /* XXX: Assume it was blocking before.  */
+      const int one = 1;
+      if (ioctl (fd, FIONBIO, &one) < 0)
+        return -1;
+#endif
+    }
+
+  /* We don't stop the handshake process for non-fatal errors */
+  do
+    {
+      err = gnutls_handshake (session);
+
+      if (opt.connect_timeout && err == GNUTLS_E_AGAIN)
+        {
+          if (gnutls_record_get_direction (session))
+            {
+              /* wait for writeability */
+              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_WRITE);
+            }
+          else
+            {
+              /* wait for readability */
+              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_READ);
+            }
+
+          if (err <= 0)
+            {
+              if (err == 0)
+                {
+                  errno = ETIMEDOUT;
+                  err = -1;
+                }
+              break;
+            }
+
+           err = GNUTLS_E_AGAIN;
+        }
+      else if (err < 0)
+        {
+          logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
+          if (err == GNUTLS_E_WARNING_ALERT_RECEIVED ||
+              err == GNUTLS_E_FATAL_ALERT_RECEIVED)
+            {
+              gnutls_alert_description_t alert = gnutls_alert_get (session);
+              str = gnutls_alert_get_name (alert);
+              if (str == NULL)
+                str = "(unknown)";
+              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%d]: %s\n", alert, str);
+            }
+        }
+    }
+  while (err && gnutls_error_is_fatal (err) == 0);
+
+  if (opt.connect_timeout)
+    {
+#ifdef F_GETFL
+      if (fcntl (fd, F_SETFL, flags) < 0)
+        return -1;
+#else
+      const int zero = 0;
+      if (ioctl (fd, FIONBIO, &zero) < 0)
+        return -1;
+#endif
+    }
+
   if (err < 0)
     {
-      logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
       gnutls_deinit (session);
       return false;
     }
@@ -312,6 +617,14 @@ ssl_connect_wget (int fd)
   return true;
 }
 
+#define _CHECK_CERT(flag,msg) \
+  if (status & (flag))\
+    {\
+      logprintf (LOG_NOTQUIET, (msg),\
+                 severity, quote (host));\
+      success = false;\
+    }
+
 bool
 ssl_check_certificate (int fd, const char *host)
 {
@@ -334,30 +647,19 @@ ssl_check_certificate (int fd, const char *host)
       goto out;
     }
 
-  if (status & GNUTLS_CERT_INVALID)
-    {
-      logprintf (LOG_NOTQUIET, _("%s: The certificate of %s is not trusted.\n"),
-                 severity, quote (host));
-      success = false;
-    }
-  if (status & GNUTLS_CERT_SIGNER_NOT_FOUND)
-    {
-      logprintf (LOG_NOTQUIET, _("%s: The certificate of %s hasn't got a known issuer.\n"),
-                 severity, quote (host));
-      success = false;
-    }
-  if (status & GNUTLS_CERT_REVOKED)
-    {
-      logprintf (LOG_NOTQUIET, _("%s: The certificate of %s has been revoked.\n"),
-                 severity, quote (host));
-      success = false;
-    }
+  _CHECK_CERT (GNUTLS_CERT_INVALID, _("%s: The certificate of %s is not trusted.\n"));
+  _CHECK_CERT (GNUTLS_CERT_SIGNER_NOT_FOUND, _("%s: The certificate of %s hasn't got a known issuer.\n"));
+  _CHECK_CERT (GNUTLS_CERT_REVOKED, _("%s: The certificate of %s has been revoked.\n"));
+  _CHECK_CERT (GNUTLS_CERT_SIGNER_NOT_CA, _("%s: The certificate signer of %s was not a CA.\n"));
+  _CHECK_CERT (GNUTLS_CERT_INSECURE_ALGORITHM, _("%s: The certificate of %s was signed using an insecure algorithm.\n"));
+  _CHECK_CERT (GNUTLS_CERT_NOT_ACTIVATED, _("%s: The certificate of %s is not yet activated.\n"));
+  _CHECK_CERT (GNUTLS_CERT_EXPIRED, _("%s: The certificate of %s has expired.\n"));
 
   if (gnutls_certificate_type_get (ctx->session) == GNUTLS_CRT_X509)
     {
       time_t now = time (NULL);
-      gnutls_x509_crt cert;
-      const gnutls_datum *cert_list;
+      gnutls_x509_crt_t cert;
+      const gnutls_datum_t *cert_list;
       unsigned int cert_list_size;
 
       if ((err = gnutls_x509_crt_init (&cert)) < 0)
@@ -373,7 +675,7 @@ ssl_check_certificate (int fd, const char *host)
         {
           logprintf (LOG_NOTQUIET, _("No certificate found\n"));
           success = false;
-          goto out;
+          goto crt_deinit;
         }
       err = gnutls_x509_crt_import (cert, cert_list, GNUTLS_X509_FMT_DER);
       if (err < 0)
@@ -381,7 +683,7 @@ ssl_check_certificate (int fd, const char *host)
           logprintf (LOG_NOTQUIET, _("Error parsing certificate: %s\n"),
                      gnutls_strerror (err));
           success = false;
-          goto out;
+          goto crt_deinit;
         }
       if (now < gnutls_x509_crt_get_activation_time (cert))
         {
@@ -400,8 +702,14 @@ ssl_check_certificate (int fd, const char *host)
                      quote (host));
           success = false;
         }
+ crt_deinit:
       gnutls_x509_crt_deinit (cert);
-   }
+    }
+  else
+    {
+      logprintf (LOG_NOTQUIET, _("Certificate must be X.509\n"));
+      success = false;
+    }
 
  out:
   return opt.check_cert ? success : true;

src/hash.c

@@ -46,20 +46,17 @@ as that of the covered work.  */
 # include "utils.h"
 #else
 /* Make do without them. */
-# define xnew(x) xmalloc (sizeof (x))
-# define xnew_array(type, x) xmalloc (sizeof (type) * (x))
-# define xmalloc malloc
-# define xfree free
+# define xnew(type) (xmalloc (sizeof (type)))
+# define xnew0(type) (xcalloc (1, sizeof (type)))
+# define xnew_array(type, len) (xmalloc ((len) * sizeof (type)))
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
+
 # ifndef countof
 #  define countof(x) (sizeof (x) / sizeof ((x)[0]))
 # endif
 # include <ctype.h>
 # define c_tolower(x) tolower ((unsigned char) (x))
-# ifdef HAVE_STDINT_H
-#  include <stdint.h>
-# else
-   typedef unsigned long uintptr_t;
-# endif
+# include <stdint.h>
 #endif
 
 #include "hash.h"
@@ -423,14 +420,14 @@ grow_hash_table (struct hash_table *ht)
    table if necessary.  */
 
 void
-hash_table_put (struct hash_table *ht, const void *key, void *value)
+hash_table_put (struct hash_table *ht, const void *key, const void *value)
 {
   struct cell *c = find_cell (ht, key);
   if (CELL_OCCUPIED (c))
     {
       /* update existing item */
       c->key   = (void *)key; /* const? */
-      c->value = value;
+      c->value = (void *)value;
       return;
     }
 
@@ -445,7 +442,7 @@ hash_table_put (struct hash_table *ht, const void *key, void *value)
   /* add new item */
   ++ht->count;
   c->key   = (void *)key;       /* const? */
-  c->value = value;
+  c->value = (void *)value;
 }
 
 /* Remove KEY->value mapping from HT.  Return 0 if there was no such
@@ -589,7 +586,7 @@ hash_table_count (const struct hash_table *ht)
 {
   return ht->count;
 }
-
+
 /* Functions from this point onward are meant for convenience and
    don't strictly belong to this file.  However, this is as good a
    place for them as any.  */
@@ -744,7 +741,7 @@ cmp_pointer (const void *ptr1, const void *ptr2)
 {
   return ptr1 == ptr2;
 }
-
+
 #ifdef TEST
 
 #include <stdio.h>
@@ -767,6 +764,15 @@ main (void)
 {
   struct hash_table *ht = make_string_hash_table (0);
   char line[80];
+
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   while ((fgets (line, sizeof (line), stdin)))
     {
       int len = strlen (line);

src/hash.h

@@ -34,7 +34,7 @@ as that of the covered work.  */
 struct hash_table;
 
 struct hash_table *hash_table_new (int, unsigned long (*) (const void *),
-				   int (*) (const void *, const void *));
+                                   int (*) (const void *, const void *));
 void hash_table_destroy (struct hash_table *);
 
 void *hash_table_get (const struct hash_table *, const void *);
@@ -42,16 +42,16 @@ int hash_table_get_pair (const struct hash_table *, const void *,
                          void *, void *);
 int hash_table_contains (const struct hash_table *, const void *);
 
-void hash_table_put (struct hash_table *, const void *, void *);
+void hash_table_put (struct hash_table *, const void *, const void *);
 int hash_table_remove (struct hash_table *, const void *);
 void hash_table_clear (struct hash_table *);
 
 void hash_table_for_each (struct hash_table *,
-		          int (*) (void *, void *, void *), void *);
+                          int (*) (void *, void *, void *), void *);
 
 typedef struct {
-  void *key, *value;		/* public members */
-  void *pos, *end;		/* private members */
+  void *key, *value;    /* public members */
+  void *pos, *end;      /* private members */
 } hash_table_iterator;
 void hash_table_iterate (struct hash_table *, hash_table_iterator *);
 int hash_table_iter_next (hash_table_iterator *);

src/host.c

@@ -1,6 +1,6 @@
 /* Host name resolution and matching.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -151,6 +151,13 @@ address_list_set_faulty (struct address_list *al, int index)
      "faulty" attempt is always preceded with all-faulty addresses,
      and this is how Wget uses it.  */
   assert (index == al->faulty);
+  if (index != al->faulty)
+    {
+      logprintf (LOG_ALWAYS, "index: %d\nal->faulty: %d\n", index, al->faulty);
+      logprintf (LOG_ALWAYS, _("Error in handling the address list.\n"));
+      logprintf (LOG_ALWAYS, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort();
+    }
 
   ++al->faulty;
   if (al->faulty >= al->count)
@@ -309,7 +316,7 @@ address_list_release (struct address_list *al)
       address_list_delete (al);
     }
 }
-
+
 /* Versions of gethostbyname and getaddrinfo that support timeout. */
 
 #ifndef ENABLE_IPV6
@@ -408,7 +415,7 @@ getaddrinfo_with_timeout (const char *node, const char *service,
 }
 
 #endif /* ENABLE_IPV6 */
-
+
 /* Return a textual representation of ADDR, i.e. the dotted quad for
    IPv4 addresses, and the colon-separated list of hex words (with all
    zeros omitted, etc.) for IPv6 addresses.  */
@@ -570,7 +577,7 @@ is_valid_ipv6_address (const char *str, const char *end)
 
   return true;
 }
-
+
 /* Simple host cache, used by lookup_host to speed up resolving.  The
    cache doesn't handle TTL because Wget is a fairly short-lived
    application.  Refreshing is attempted when connect fails, though --
@@ -592,7 +599,7 @@ cache_query (const char *host)
   al = hash_table_get (host_name_addresses_map, host);
   if (al)
     {
-      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, al));
+      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, (void *) al));
       ++al->refcount;
       return al;
     }
@@ -637,7 +644,7 @@ cache_remove (const char *host)
       hash_table_remove (host_name_addresses_map, host);
     }
 }
-
+
 /* Look up HOST in DNS and return a list of IP addresses.
 
    This function caches its result so that, if the same host is passed
@@ -740,8 +747,7 @@ lookup_host (const char *host, int flags)
       logprintf (LOG_VERBOSE, _("Resolving %s... "),
                  quotearg_style (escape_quoting_style, str ? str : host));
 
-      if (str)
-        xfree (str);
+      xfree (str);
     }
 
 #ifdef ENABLE_IPV6
@@ -848,7 +854,7 @@ lookup_host (const char *host, int flags)
 
   return al;
 }
-
+
 /* Determine whether a URL is acceptable to be followed, according to
    a list of domains to accept.  */
 bool
@@ -914,3 +920,18 @@ host_cleanup (void)
       host_name_addresses_map = NULL;
     }
 }
+
+bool
+is_valid_ip_address (const char *name)
+{
+  const char *endp;
+
+  endp = name + strlen(name);
+  if (is_valid_ipv4_address (name, endp))
+    return true;
+#ifdef ENABLE_IPV6
+  if (is_valid_ipv6_address (name, endp))
+    return true;
+#endif
+  return false;
+}

src/host.h

@@ -1,6 +1,6 @@
 /* Declarations for host.c
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -58,9 +58,9 @@ typedef struct {
 
   /* The actual data, in the form of struct in_addr or in6_addr: */
   union {
-    struct in_addr d4;		/* IPv4 address */
+    struct in_addr d4;      /* IPv4 address */
 #ifdef ENABLE_IPV6
-    struct in6_addr d6;		/* IPv6 address */
+    struct in6_addr d6;     /* IPv6 address */
 #endif
   } data;
 
@@ -98,6 +98,8 @@ const char *print_address (const ip_address *);
 bool is_valid_ipv6_address (const char *, const char *);
 #endif
 
+bool is_valid_ip_address (const char *name);
+
 bool accept_domain (struct url *);
 bool sufmatch (const char **, const char *);
 

src/html-parse.c

@@ -253,7 +253,7 @@ struct pool {
       (sizevar) = ga_newsize;                                                   \
     }                                                                           \
 } while (0)
-
+
 /* Test whether n+1-sized entity name fits in P.  We don't support
    IE-style non-terminated entities, e.g. "&ltfoo" -> "<foo".
    However, "&lt;foo" will work, as will "&lt!foo", "&lt", etc.  In
@@ -280,7 +280,7 @@ struct tagstack_item {
   struct tagstack_item *next;
 };
 
-struct tagstack_item *
+static struct tagstack_item *
 tagstack_push (struct tagstack_item **head, struct tagstack_item **tail)
 {
   struct tagstack_item *ts = xmalloc(sizeof(struct tagstack_item));
@@ -301,7 +301,7 @@ tagstack_push (struct tagstack_item **head, struct tagstack_item **tail)
 }
 
 /* remove ts and everything after it from the stack */
-void
+static void
 tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail,
               struct tagstack_item *ts)
 {
@@ -343,7 +343,7 @@ tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail,
     }
 }
 
-struct tagstack_item *
+static struct tagstack_item *
 tagstack_find (struct tagstack_item *tail, const char *tagname_begin,
                const char *tagname_end)
 {
@@ -517,7 +517,7 @@ convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags
         *p = c_tolower (*p);
     }
 }
-
+
 /* Originally we used to adhere to rfc 1866 here, and allowed only
    letters, digits, periods, and hyphens as names (of tags or
    attributes).  However, this broke too many pages which used
@@ -770,7 +770,7 @@ find_comment_end (const char *beg, const char *end)
       }
   return NULL;
 }
-
+
 /* Return true if the string containing of characters inside [b, e) is
    present in hash table HT.  */
 
@@ -881,7 +881,7 @@ map_html_tags (const char *text, int size,
     if (*p == '!')
       {
         if (!(flags & MHT_STRICT_COMMENTS)
-            && p < end + 3 && p[1] == '-' && p[2] == '-')
+            && p + 3 < end && p[1] == '-' && p[2] == '-')
           {
             /* If strict comments are not enforced and if we know
                we're looking at a comment, simply look for the
@@ -1167,7 +1167,7 @@ map_html_tags (const char *text, int size,
 #undef ADVANCE
 #undef SKIP_WS
 #undef SKIP_NON_WS
-
+
 #ifdef STANDALONE
 static void
 test_mapper (struct taginfo *taginfo, void *arg)
@@ -1189,6 +1189,14 @@ int main ()
   int read_count;
   int tag_counter = 0;
 
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   while ((read_count = fread (x + length, 1, size - length, stdin)))
     {
       length += read_count;

src/html-parse.h

@@ -32,8 +32,8 @@ as that of the covered work.  */
 #define HTML_PARSE_H
 
 struct attr_pair {
-  char *name;			/* attribute name */
-  char *value;			/* attribute value */
+  char *name;           /* attribute name */
+  char *value;          /* attribute value */
 
   /* Needed for URL conversion; the places where the value begins and
      ends, including the quotes and everything. */
@@ -45,19 +45,19 @@ struct attr_pair {
 };
 
 struct taginfo {
-  char *name;			/* tag name */
-  int end_tag_p;		/* whether this is an end-tag */
-  int nattrs;			/* number of attributes */
-  struct attr_pair *attrs;	/* attributes */
+  char *name;                   /* tag name */
+  int end_tag_p;                /* whether this is an end-tag */
+  int nattrs;                   /* number of attributes */
+  struct attr_pair *attrs;      /* attributes */
 
-  const char *start_position;	/* start position of tag */
-  const char *end_position;	/* end position of tag */
+  const char *start_position;   /* start position of tag */
+  const char *end_position;     /* end position of tag */
 
   const char *contents_begin;   /* delimiters of tag contents */
   const char *contents_end;     /* only valid if end_tag_p */
 };
 
-struct hash_table;		/* forward declaration */
+struct hash_table;              /* forward declaration */
 
 /* Flags for map_html_tags: */
 #define MHT_STRICT_COMMENTS  1  /* use strict comment interpretation */
@@ -65,7 +65,7 @@ struct hash_table;		/* forward declaration */
                                    <a href=" foo "> as "foo" */
 
 void map_html_tags (const char *, int,
-		    void (*) (struct taginfo *, void *), void *, int,
-		    const struct hash_table *, const struct hash_table *);
+                    void (*) (struct taginfo *, void *), void *, int,
+                    const struct hash_table *, const struct hash_table *);
 
 #endif /* HTML_PARSE_H */

src/html-url.c

@@ -1,6 +1,6 @@
 /* Collect URLs from HTML source.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -45,6 +45,7 @@ as that of the covered work.  */
 #include "recur.h"
 #include "html-url.h"
 #include "css-url.h"
+#include "c-strcase.h"
 
 typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
 
@@ -79,7 +80,10 @@ enum {
   TAG_SCRIPT,
   TAG_TABLE,
   TAG_TD,
-  TAG_TH
+  TAG_TH,
+  TAG_VIDEO,
+  TAG_AUDIO,
+  TAG_SOURCE
 };
 
 /* The list of known tags and functions used for handling them.  Most
@@ -110,7 +114,10 @@ static struct known_tag {
   { TAG_SCRIPT,  "script",      tag_find_urls },
   { TAG_TABLE,   "table",       tag_find_urls },
   { TAG_TD,      "td",          tag_find_urls },
-  { TAG_TH,      "th",          tag_find_urls }
+  { TAG_TH,      "th",          tag_find_urls },
+  { TAG_VIDEO,   "video",       tag_find_urls },
+  { TAG_AUDIO,   "audio",       tag_find_urls },
+  { TAG_SOURCE,  "source",      tag_find_urls }
 };
 
 /* tag_url_attributes documents which attributes of which tags contain
@@ -157,7 +164,12 @@ static struct {
   { TAG_SCRIPT,         "src",          ATTR_INLINE },
   { TAG_TABLE,          "background",   ATTR_INLINE },
   { TAG_TD,             "background",   ATTR_INLINE },
-  { TAG_TH,             "background",   ATTR_INLINE }
+  { TAG_TH,             "background",   ATTR_INLINE },
+  { TAG_VIDEO,          "src",          ATTR_INLINE },
+  { TAG_VIDEO,          "poster",       ATTR_INLINE },
+  { TAG_AUDIO,          "src",          ATTR_INLINE },
+  { TAG_AUDIO,          "poster",       ATTR_INLINE },
+  { TAG_SOURCE,         "src",          ATTR_INLINE }
 };
 
 /* The lists of interesting tags and attributes are built dynamically,
@@ -244,7 +256,7 @@ find_attr (struct taginfo *tag, const char *name, int *attrind)
 {
   int i;
   for (i = 0; i < tag->nattrs; i++)
-    if (!strcasecmp (tag->attrs[i].name, name))
+    if (!c_strcasecmp (tag->attrs[i].name, name))
       {
         if (attrind)
           *attrind = i;
@@ -273,6 +285,10 @@ append_url (const char *link_uri, int position, int size,
   const char *base = ctx->base ? ctx->base : ctx->parent_base;
   struct url *url;
 
+  struct iri *iri = iri_new ();
+  set_uri_encoding (iri, opt.locale, true);
+  iri->utf8_encode = true;
+
   if (!base)
     {
       DEBUGP (("%s: no base, merge will use \"%s\".\n",
@@ -290,7 +306,7 @@ append_url (const char *link_uri, int position, int size,
           return NULL;
         }
 
-      url = url_parse (link_uri, NULL, NULL, false);
+      url = url_parse (link_uri, NULL, iri, false);
       if (!url)
         {
           DEBUGP (("%s: link \"%s\" doesn't parse.\n",
@@ -312,7 +328,7 @@ append_url (const char *link_uri, int position, int size,
                quote_n (2, link_uri),
                quotearg_n_style (3, escape_quoting_style, complete_uri)));
 
-      url = url_parse (complete_uri, NULL, NULL, false);
+      url = url_parse (complete_uri, NULL, iri, false);
       if (!url)
         {
           DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
@@ -323,6 +339,8 @@ append_url (const char *link_uri, int position, int size,
       xfree (complete_uri);
     }
 
+  iri_free (iri);
+
   DEBUGP (("appending %s to urlpos.\n", quote (url->url)));
 
   newel = xnew0 (struct urlpos);
@@ -361,7 +379,7 @@ append_url (const char *link_uri, int position, int size,
 
   return newel;
 }
-
+
 static void
 check_style_attr (struct taginfo *tag, struct map_context *ctx)
 {
@@ -455,7 +473,7 @@ tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx)
 /* Handle the BASE tag, for <base href=...>. */
 
 static void
-tag_handle_base (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_base (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   struct urlpos *base_urlpos;
   int attrind;
@@ -470,8 +488,7 @@ tag_handle_base (int tagid, struct taginfo *tag, struct map_context *ctx)
   base_urlpos->ignore_when_downloading = 1;
   base_urlpos->link_base_p = 1;
 
-  if (ctx->base)
-    xfree (ctx->base);
+  xfree (ctx->base);
   if (ctx->parent_base)
     ctx->base = uri_merge (ctx->parent_base, newbase);
   else
@@ -481,7 +498,7 @@ tag_handle_base (int tagid, struct taginfo *tag, struct map_context *ctx)
 /* Mark the URL found in <form action=...> for conversion. */
 
 static void
-tag_handle_form (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_form (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   int attrind;
   char *action = find_attr (tag, "action", &attrind);
@@ -499,7 +516,7 @@ tag_handle_form (int tagid, struct taginfo *tag, struct map_context *ctx)
    links will be followed in -p mode depends on the REL attribute.  */
 
 static void
-tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   int attrind;
   char *href = find_attr (tag, "href", &attrind);
@@ -519,12 +536,12 @@ tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
           char *rel = find_attr (tag, "rel", NULL);
           if (rel)
             {
-              if (0 == strcasecmp (rel, "stylesheet"))
+              if (0 == c_strcasecmp (rel, "stylesheet"))
                 {
                   up->link_inline_p = 1;
                   up->link_expect_css = 1;
                 }
-              else if (0 == strcasecmp (rel, "shortcut icon"))
+              else if (0 == c_strcasecmp (rel, "shortcut icon"))
                 {
                   up->link_inline_p = 1;
                 }
@@ -536,7 +553,7 @@ tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
                      <link rel="alternate" type="application/rss+xml" href=".../?feed=rss2" />
                   */
                   char *type = find_attr (tag, "type", NULL);
-                  if (!type || strcasecmp (type, "text/html") == 0)
+                  if (!type || c_strcasecmp (type, "text/html") == 0)
                     up->link_expect_html = 1;
                 }
             }
@@ -548,12 +565,12 @@ tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
    refresh feature and because of robot exclusion.  */
 
 static void
-tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   char *name = find_attr (tag, "name", NULL);
   char *http_equiv = find_attr (tag, "http-equiv", NULL);
 
-  if (http_equiv && 0 == strcasecmp (http_equiv, "refresh"))
+  if (http_equiv && 0 == c_strcasecmp (http_equiv, "refresh"))
     {
       /* Some pages use a META tag to specify that the page be
          refreshed by a new page after a given number of seconds.  The
@@ -598,7 +615,7 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
           entry->link_expect_html = 1;
         }
     }
-  else if (http_equiv && 0 == strcasecmp (http_equiv, "content-type"))
+  else if (http_equiv && 0 == c_strcasecmp (http_equiv, "content-type"))
     {
       /* Handle stuff like:
          <meta http-equiv="Content-Type" content="text/html; charset=CHARSET"> */
@@ -612,17 +629,17 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
       if (!mcharset)
         return;
 
-      xfree_null (meta_charset);
+      xfree (meta_charset);
       meta_charset = mcharset;
     }
-  else if (name && 0 == strcasecmp (name, "robots"))
+  else if (name && 0 == c_strcasecmp (name, "robots"))
     {
       /* Handle stuff like:
          <meta name="robots" content="index,nofollow"> */
       char *content = find_attr (tag, "content", NULL);
       if (!content)
         return;
-      if (!strcasecmp (content, "none"))
+      if (!c_strcasecmp (content, "none"))
         ctx->nofollow = true;
       else
         {
@@ -634,7 +651,7 @@ tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
               /* Find the next occurrence of ',' or whitespace,
                * or the end of the string.  */
               end = content + strcspn (content, ", \f\n\r\t\v");
-              if (!strncasecmp (content, "nofollow", end - content))
+              if (!c_strncasecmp (content, "nofollow", end - content))
                 ctx->nofollow = true;
               /* Skip past the next comma, if any. */
               if (*end == ',')
@@ -675,15 +692,16 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
 
   check_style_attr (tag, ctx);
 
-  if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style")) &&
-      tag->contents_begin && tag->contents_end)
+  if (tag->end_tag_p && (0 == c_strcasecmp (tag->name, "style"))
+      && tag->contents_begin && tag->contents_end
+      && tag->contents_begin <= tag->contents_end)
   {
     /* parse contents */
     get_urls_css (ctx, tag->contents_begin - ctx->text,
                   tag->contents_end - tag->contents_begin);
   }
 }
-
+
 /* Analyze HTML tags FILE and construct a list of URLs referenced from
    it.  It merges relative links in FILE with URL.  It is aware of
    <base href=...> and does the right thing.  */
@@ -730,15 +748,16 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
   map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
                  NULL, interesting_attributes);
 
-  /* If meta charset isn't null, override content encoding */
-  if (iri && meta_charset)
+  /* Meta charset is only valid if there was no HTTP header Content-Type charset. */
+  /* This is true for HTTP 1.0 and 1.1. */
+  if (iri && !iri->content_encoding && meta_charset)
     set_content_encoding (iri, meta_charset);
 
   DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
   if (meta_disallow_follow)
     *meta_disallow_follow = ctx.nofollow;
 
-  xfree_null (ctx.base);
+  xfree (ctx.base);
   wget_read_file_free (fm);
   return ctx.head;
 }
@@ -769,6 +788,7 @@ get_urls_file (const char *file)
     {
       int up_error_code;
       char *url_text;
+      char *new_url;
       struct urlpos *entry;
       struct url *url;
 
@@ -803,6 +823,13 @@ get_urls_file (const char *file)
           url_text = merged;
         }
 
+      new_url = rewrite_shorthand_url (url_text);
+      if (new_url)
+        {
+          xfree (url_text);
+          url_text = new_url;
+        }
+
       url = url_parse (url_text, &up_error_code, NULL, false);
       if (!url)
         {

src/html-url.h

@@ -32,20 +32,21 @@ as that of the covered work.  */
 #define HTML_URL_H
 
 struct map_context {
-  char *text;			/* HTML text. */
-  char *base;			/* Base URI of the document, possibly
-				   changed through <base href=...>. */
-  const char *parent_base;	/* Base of the current document. */
-  const char *document_file;	/* File name of this document. */
-  bool nofollow;		/* whether NOFOLLOW was specified in a
+  char *text;                   /* HTML text. */
+  char *base;                   /* Base URI of the document, possibly
+                                   changed through <base href=...>. */
+  const char *parent_base;      /* Base of the current document. */
+  const char *document_file;    /* File name of this document. */
+  bool nofollow;                /* whether NOFOLLOW was specified in a
                                    <meta name=robots> tag. */
 
-  struct urlpos *head;	/* List of URLs that is being built. */
+  struct urlpos *head;          /* List of URLs that is being built. */
 };
 
 struct urlpos *get_urls_file (const char *);
 struct urlpos *get_urls_html (const char *, const char *, bool *, struct iri *);
 struct urlpos *append_url (const char *, int, int, struct map_context *);
 void free_urlpos (struct urlpos *);
+void cleanup_html_url (void);
 
 #endif /* HTML_URL_H */

src/http-ntlm.c

@@ -42,68 +42,45 @@ as that of the covered work.  */
 #include <string.h>
 #include <stdlib.h>
 
-#include <openssl/des.h>
-#include <openssl/md4.h>
-#include <openssl/opensslv.h>
-
 #include "utils.h"
 #include "http-ntlm.h"
 
-#if OPENSSL_VERSION_NUMBER < 0x00907001L
-#define DES_key_schedule des_key_schedule
-#define DES_cblock des_cblock
-#define DES_set_odd_parity des_set_odd_parity
-#define DES_set_key des_set_key
-#define DES_ecb_encrypt des_ecb_encrypt
+#ifdef HAVE_NETTLE
+# include <nettle/md4.h>
+# include <nettle/des.h>
+#else
+# include <openssl/des.h>
+# include <openssl/md4.h>
+# include <openssl/opensslv.h>
+
+# if OPENSSL_VERSION_NUMBER < 0x00907001L
+#  define DES_key_schedule des_key_schedule
+#  define DES_cblock des_cblock
+#  define DES_set_odd_parity des_set_odd_parity
+#  define DES_set_key des_set_key
+#  define DES_ecb_encrypt des_ecb_encrypt
 
 /* This is how things were done in the old days */
-#define DESKEY(x) x
-#define DESKEYARG(x) x
-#else
+#  define DESKEY(x) x
+#  define DESKEYARG(x) x
+# else
 /* Modern version */
-#define DESKEYARG(x) *x
-#define DESKEY(x) &x
+#  define DESKEYARG(x) *x
+#  define DESKEY(x) &x
+# endif
+
 #endif
 
 /* Define this to make the type-3 message include the NT response message */
 #define USE_NTRESPONSES 1
-
+
+
 /* Flag bits definitions available at on
    http://davenport.sourceforge.net/ntlm.html */
 
-#define NTLMFLAG_NEGOTIATE_UNICODE               (1<<0)
 #define NTLMFLAG_NEGOTIATE_OEM                   (1<<1)
-#define NTLMFLAG_REQUEST_TARGET                  (1<<2)
-/* unknown (1<<3) */
-#define NTLMFLAG_NEGOTIATE_SIGN                  (1<<4)
-#define NTLMFLAG_NEGOTIATE_SEAL                  (1<<5)
-#define NTLMFLAG_NEGOTIATE_DATAGRAM_STYLE        (1<<6)
-#define NTLMFLAG_NEGOTIATE_LM_KEY                (1<<7)
-#define NTLMFLAG_NEGOTIATE_NETWARE               (1<<8)
 #define NTLMFLAG_NEGOTIATE_NTLM_KEY              (1<<9)
-/* unknown (1<<10) */
-/* unknown (1<<11) */
-#define NTLMFLAG_NEGOTIATE_DOMAIN_SUPPLIED       (1<<12)
-#define NTLMFLAG_NEGOTIATE_WORKSTATION_SUPPLIED  (1<<13)
-#define NTLMFLAG_NEGOTIATE_LOCAL_CALL            (1<<14)
-#define NTLMFLAG_NEGOTIATE_ALWAYS_SIGN           (1<<15)
-#define NTLMFLAG_TARGET_TYPE_DOMAIN              (1<<16)
-#define NTLMFLAG_TARGET_TYPE_SERVER              (1<<17)
-#define NTLMFLAG_TARGET_TYPE_SHARE               (1<<18)
-#define NTLMFLAG_NEGOTIATE_NTLM2_KEY             (1<<19)
-#define NTLMFLAG_REQUEST_INIT_RESPONSE           (1<<20)
-#define NTLMFLAG_REQUEST_ACCEPT_RESPONSE         (1<<21)
-#define NTLMFLAG_REQUEST_NONNT_SESSION_KEY       (1<<22)
-#define NTLMFLAG_NEGOTIATE_TARGET_INFO           (1<<23)
-/* unknown (1<24) */
-/* unknown (1<25) */
-/* unknown (1<26) */
-/* unknown (1<27) */
-/* unknown (1<28) */
-#define NTLMFLAG_NEGOTIATE_128                   (1<<29)
-#define NTLMFLAG_NEGOTIATE_KEY_EXCHANGE          (1<<30)
-#define NTLMFLAG_NEGOTIATE_56                    (1<<31)
-
+
 /*
   (*) = A "security buffer" is a triplet consisting of two shorts and one
   long:
@@ -140,7 +117,7 @@ ntlm_input (struct ntlmdata *ntlm, const char *header)
          (40)    Target Information  (optional) security buffer(*)
          32 (48) start of data block
       */
-      int size;
+      ssize_t size;
       char *buffer = (char *) alloca (strlen (header));
 
       DEBUGP (("Received a type-2 NTLM message.\n"));
@@ -176,6 +153,25 @@ ntlm_input (struct ntlmdata *ntlm, const char *header)
  * Turns a 56 bit key into the 64 bit, odd parity key and sets the key.  The
  * key schedule ks is also set.
  */
+#ifdef HAVE_NETTLE
+static void
+setup_des_key(unsigned char *key_56,
+              struct des_ctx *des)
+{
+  unsigned char key[8];
+
+  key[0] = key_56[0];
+  key[1] = ((key_56[0] << 7) & 0xFF) | (key_56[1] >> 1);
+  key[2] = ((key_56[1] << 6) & 0xFF) | (key_56[2] >> 2);
+  key[3] = ((key_56[2] << 5) & 0xFF) | (key_56[3] >> 3);
+  key[4] = ((key_56[3] << 4) & 0xFF) | (key_56[4] >> 4);
+  key[5] = ((key_56[4] << 3) & 0xFF) | (key_56[5] >> 5);
+  key[6] = ((key_56[5] << 2) & 0xFF) | (key_56[6] >> 6);
+  key[7] =  (key_56[6] << 1) & 0xFF;
+
+  nettle_des_set_key(des, key);
+}
+#else
 static void
 setup_des_key(unsigned char *key_56,
               DES_key_schedule DESKEYARG(ks))
@@ -194,6 +190,7 @@ setup_des_key(unsigned char *key_56,
   DES_set_odd_parity(&key);
   DES_set_key(&key, ks);
 }
+#endif
 
  /*
   * takes a 21 byte array and treats it as 3 56-bit DES keys. The
@@ -203,6 +200,18 @@ setup_des_key(unsigned char *key_56,
 static void
 calc_resp(unsigned char *keys, unsigned char *plaintext, unsigned char *results)
 {
+#ifdef HAVE_NETTLE
+  struct des_ctx des;
+
+  setup_des_key(keys, &des);
+  nettle_des_encrypt(&des, 8, results, plaintext);
+
+  setup_des_key(keys + 7, &des);
+  nettle_des_encrypt(&des, 8, results + 8, plaintext);
+
+  setup_des_key(keys + 14, &des);
+  nettle_des_encrypt(&des, 8, results + 16, plaintext);
+#else
   DES_key_schedule ks;
 
   setup_des_key(keys, DESKEY(ks));
@@ -216,6 +225,7 @@ calc_resp(unsigned char *keys, unsigned char *plaintext, unsigned char *results)
   setup_des_key(keys+14, DESKEY(ks));
   DES_ecb_encrypt((DES_cblock*) plaintext, (DES_cblock*) (results+16),
                   DESKEY(ks), DES_ENCRYPT);
+#endif
 }
 
 /*
@@ -238,8 +248,7 @@ mkhash(const char *password,
   static const unsigned char magic[] = {
     0x4B, 0x47, 0x53, 0x21, 0x40, 0x23, 0x24, 0x25
   };
-  int i;
-  int len = strlen(password);
+  size_t i, len = strlen(password);
 
   /* make it fit at least 14 bytes */
   pw = (unsigned char *) alloca (len < 7 ? 14 : len * 2);
@@ -248,13 +257,22 @@ mkhash(const char *password,
     len = 14;
 
   for (i=0; i<len; i++)
-    pw[i] = c_toupper (password[i]);
+    pw[i] = (unsigned char) c_toupper (password[i]);
 
   for (; i<14; i++)
     pw[i] = 0;
 
   {
     /* create LanManager hashed password */
+#ifdef HAVE_NETTLE
+    struct des_ctx des;
+
+    setup_des_key(pw, &des);
+    nettle_des_encrypt(&des, 8, lmbuffer, magic);
+
+    setup_des_key(pw + 7, &des);
+    nettle_des_encrypt(&des, 8, lmbuffer + 8, magic);
+#else
     DES_key_schedule ks;
 
     setup_des_key(pw, DESKEY(ks));
@@ -264,6 +282,7 @@ mkhash(const char *password,
     setup_des_key(pw+7, DESKEY(ks));
     DES_ecb_encrypt((DES_cblock *)magic, (DES_cblock *)(lmbuffer+8),
                     DESKEY(ks), DES_ENCRYPT);
+#endif
 
     memset(lmbuffer+16, 0, 5);
   }
@@ -272,19 +291,29 @@ mkhash(const char *password,
 
 #ifdef USE_NTRESPONSES
   {
-    /* create NT hashed password */
+#ifdef HAVE_NETTLE
+    struct md4_ctx MD4;
+#else
     MD4_CTX MD4;
+#endif
 
     len = strlen(password);
 
     for (i=0; i<len; i++) {
-      pw[2*i]   = password[i];
+      pw[2*i]   = (unsigned char) password[i];
       pw[2*i+1] = 0;
     }
 
+#ifdef HAVE_NETTLE
+    nettle_md4_init(&MD4);
+    nettle_md4_update(&MD4, (unsigned) (2 * len), pw);
+    nettle_md4_digest(&MD4, MD4_DIGEST_SIZE, ntbuffer);
+#else
+    /* create NT hashed password */
     MD4_Init(&MD4);
     MD4_Update(&MD4, pw, 2*len);
     MD4_Final(ntbuffer, &MD4);
+#endif
 
     memset(ntbuffer+16, 0, 5);
   }
@@ -293,7 +322,7 @@ mkhash(const char *password,
 #endif
 }
 
-#define SHORTPAIR(x) ((x) & 0xff), ((x) >> 8)
+#define SHORTPAIR(x) (char) ((x) & 0xff), (char) ((x) >> 8)
 #define LONGQUARTET(x) ((x) & 0xff), (((x) >> 8)&0xff), \
   (((x) >>16)&0xff), ((x)>>24)
 
@@ -302,19 +331,19 @@ char *
 ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
              bool *ready)
 {
-  const char *domain=""; /* empty */
-  const char *host=""; /* empty */
-  int domlen=strlen(domain);
-  int hostlen = strlen(host);
-  int hostoff; /* host name offset */
-  int domoff;  /* domain name offset */
-  int size;
+  const char *domain = ""; /* empty */
+  const char *host = ""; /* empty */
+  size_t domlen = strlen(domain);
+  size_t hostlen = strlen(host);
+  size_t hostoff; /* host name offset */
+  size_t domoff;  /* domain name offset */
+  size_t size;
   char *base64;
   char ntlmbuf[256]; /* enough, unless the host/domain is very long */
 
   /* point to the address of the pointer that holds the string to sent to the
      server, which is for a plain host or for a HTTP proxy */
-  char *output;
+  char *output = NULL;
 
   *ready = false;
 
@@ -327,7 +356,8 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
   switch(ntlm->state) {
   case NTLMSTATE_TYPE1:
-  default: /* for the weird cases we (re)start here */
+  case NTLMSTATE_NONE:
+  case NTLMSTATE_LAST:
     hostoff = 32;
     domoff = hostoff + hostlen;
 
@@ -405,15 +435,15 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
     */
 
   {
-    int lmrespoff;
-    int ntrespoff;
-    int useroff;
+    size_t lmrespoff;
+    size_t ntrespoff;
+    size_t useroff;
     unsigned char lmresp[0x18]; /* fixed-size */
 #ifdef USE_NTRESPONSES
     unsigned char ntresp[0x18]; /* fixed-size */
 #endif
     const char *usr;
-    int userlen;
+    size_t userlen;
 
     DEBUGP (("Creating a type-3 NTLM message.\n"));
 
@@ -423,7 +453,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     if (usr) {
       domain = user;
-      domlen = usr - domain;
+      domlen = (size_t) (usr - domain);
       usr++;
     }
     else
@@ -444,82 +474,82 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* Create the big type-3 message binary blob */
 
-    size = snprintf (ntlmbuf, sizeof(ntlmbuf),
-                     "NTLMSSP%c"
-                     "\x03%c%c%c" /* type-3, 32 bits */
+    snprintf (ntlmbuf, sizeof (ntlmbuf),
+              "NTLMSSP%c"
+              "\x03%c%c%c" /* type-3, 32 bits */
 
-                     "%c%c%c%c" /* LanManager length + allocated space */
-                     "%c%c" /* LanManager offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c%c%c" /* LanManager length + allocated space */
+              "%c%c" /* LanManager offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c" /* NT-response length */
-                     "%c%c" /* NT-response allocated space */
-                     "%c%c" /* NT-response offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c" /* NT-response length */
+              "%c%c" /* NT-response allocated space */
+              "%c%c" /* NT-response offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* domain length */
-                     "%c%c"  /* domain allocated space */
-                     "%c%c"  /* domain name offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* domain length */
+              "%c%c" /* domain allocated space */
+              "%c%c" /* domain name offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* user length */
-                     "%c%c"  /* user allocated space */
-                     "%c%c"  /* user offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* user length */
+              "%c%c" /* user allocated space */
+              "%c%c" /* user offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* host length */
-                     "%c%c"  /* host allocated space */
-                     "%c%c"  /* host offset */
-                     "%c%c%c%c%c%c"  /* 6 zeroes */
+              "%c%c" /* host length */
+              "%c%c" /* host allocated space */
+              "%c%c" /* host offset */
+              "%c%c%c%c%c%c" /* 6 zeroes */
 
-                     "\xff\xff"  /* message length */
-                     "%c%c"  /* 2 zeroes */
+              "\xff\xff" /* message length */
+              "%c%c" /* 2 zeroes */
 
-                     "\x01\x82" /* flags */
-                     "%c%c"  /* 2 zeroes */
+              "\x01\x82" /* flags */
+              "%c%c" /* 2 zeroes */
 
-                     /* domain string */
-                     /* user string */
-                     /* host string */
-                     /* LanManager response */
-                     /* NT response */
-                     ,
-                     0, /* zero termination */
-                     0,0,0, /* type-3 long, the 24 upper bits */
+              /* domain string */
+              /* user string */
+              /* host string */
+              /* LanManager response */
+              /* NT response */
+              ,
+              0, /* zero termination */
+              0, 0, 0, /* type-3 long, the 24 upper bits */
 
-                     SHORTPAIR(0x18),  /* LanManager response length, twice */
-                     SHORTPAIR(0x18),
-                     SHORTPAIR(lmrespoff),
-                     0x0, 0x0,
+              SHORTPAIR (0x18), /* LanManager response length, twice */
+              SHORTPAIR (0x18),
+              SHORTPAIR (lmrespoff),
+              0x0, 0x0,
 
 #ifdef USE_NTRESPONSES
-                     SHORTPAIR(0x18),  /* NT-response length, twice */
-                     SHORTPAIR(0x18),
+                SHORTPAIR (0x18), /* NT-response length, twice */
+              SHORTPAIR (0x18),
 #else
-                     0x0, 0x0,
-                     0x0, 0x0,
+                0x0, 0x0,
+              0x0, 0x0,
 #endif
-                     SHORTPAIR(ntrespoff),
-                     0x0, 0x0,
+                SHORTPAIR (ntrespoff),
+              0x0, 0x0,
 
-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domoff),
-                     0x0, 0x0,
+              SHORTPAIR (domlen),
+              SHORTPAIR (domlen),
+              SHORTPAIR (domoff),
+              0x0, 0x0,
 
-                     SHORTPAIR(userlen),
-                     SHORTPAIR(userlen),
-                     SHORTPAIR(useroff),
-                     0x0, 0x0,
+              SHORTPAIR (userlen),
+              SHORTPAIR (userlen),
+              SHORTPAIR (useroff),
+              0x0, 0x0,
 
-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostoff),
-                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostoff),
+              0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 
-                     0x0, 0x0,
+              0x0, 0x0,
 
-                     0x0, 0x0);
+              0x0, 0x0);
 
     /* size is now 64 */
     size=64;
@@ -527,7 +557,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* Make sure that the user and domain strings fit in the target buffer
        before we copy them there. */
-    if(((size_t) size + userlen + domlen) >= sizeof(ntlmbuf))
+    if((size + userlen + domlen) >= sizeof(ntlmbuf))
       return NULL;
 
     memcpy(&ntlmbuf[size], domain, domlen);
@@ -537,20 +567,20 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
     size += userlen;
 
     /* we append the binary hashes to the end of the blob */
-    if(size < ((int)sizeof(ntlmbuf) - 0x18)) {
+    if(size < (sizeof(ntlmbuf) - 0x18)) {
       memcpy(&ntlmbuf[size], lmresp, 0x18);
       size += 0x18;
     }
 
 #ifdef USE_NTRESPONSES
-    if(size < ((int)sizeof(ntlmbuf) - 0x18)) {
+    if(size < (sizeof(ntlmbuf) - 0x18)) {
       memcpy(&ntlmbuf[size], ntresp, 0x18);
       size += 0x18;
     }
 #endif
 
-    ntlmbuf[56] = size & 0xff;
-    ntlmbuf[57] = size >> 8;
+    ntlmbuf[56] = (char) (size & 0xff);
+    ntlmbuf[57] = (char) (size >> 8);
 
     /* convert the binary blob into base64 */
     base64 = (char *) alloca (BASE64_LENGTH (size) + 1);

src/http.h

@@ -43,7 +43,7 @@ typedef struct {
   /* A token consists of characters in the [b, e) range. */
   const char *b, *e;
 } param_token;
-bool extract_param (const char **, param_token *, param_token *, char);
+bool extract_param (const char **, param_token *, param_token *, char, bool *);
 
 
 #endif /* HTTP_H */

src/init.c

@@ -1,6 +1,6 @@
 /* Reading/parsing the initialization file.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -30,6 +30,7 @@ shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
 #include "wget.h"
+#include "exits.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -46,6 +47,10 @@ as that of the covered work.  */
 # endif
 #endif
 
+#include <regex.h>
+#ifdef HAVE_LIBPCRE
+# include <pcre.h>
+#endif
 
 #ifdef HAVE_PWD_H
 # include <pwd.h>
@@ -62,6 +67,10 @@ as that of the covered work.  */
 #include "res.h"                /* for res_cleanup */
 #include "http.h"               /* for http_cleanup */
 #include "retr.h"               /* for output_stream */
+#include "warc.h"               /* for warc_close */
+#include "spider.h"             /* for spider_cleanup */
+#include "html-url.h"           /* for cleanup_html_url */
+#include "c-strcase.h"
 
 #ifdef TESTING
 #include "test.h"
@@ -81,19 +90,24 @@ CMD_DECLARE (cmd_directory_vector);
 CMD_DECLARE (cmd_number);
 CMD_DECLARE (cmd_number_inf);
 CMD_DECLARE (cmd_string);
+CMD_DECLARE (cmd_string_uppercase);
 CMD_DECLARE (cmd_file);
+CMD_DECLARE (cmd_file_once);
 CMD_DECLARE (cmd_directory);
 CMD_DECLARE (cmd_time);
 CMD_DECLARE (cmd_vector);
 
 CMD_DECLARE (cmd_spec_dirstruct);
 CMD_DECLARE (cmd_spec_header);
+CMD_DECLARE (cmd_spec_warc_header);
 CMD_DECLARE (cmd_spec_htmlify);
 CMD_DECLARE (cmd_spec_mirror);
 CMD_DECLARE (cmd_spec_prefer_family);
 CMD_DECLARE (cmd_spec_progress);
 CMD_DECLARE (cmd_spec_recursive);
+CMD_DECLARE (cmd_spec_regex_type);
 CMD_DECLARE (cmd_spec_restrict_file_names);
+CMD_DECLARE (cmd_spec_report_speed);
 #ifdef HAVE_SSL
 CMD_DECLARE (cmd_spec_secure_protocol);
 #endif
@@ -115,6 +129,7 @@ static const struct {
 } commands[] = {
   /* KEEP THIS LIST ALPHABETICALLY SORTED */
   { "accept",           &opt.accepts,           cmd_vector },
+  { "acceptregex",      &opt.acceptregex_s,     cmd_string },
   { "addhostdir",       &opt.add_hostdir,       cmd_boolean },
   { "adjustextension",  &opt.adjust_extension,  cmd_boolean },
   { "alwaysrest",       &opt.always_rest,       cmd_boolean }, /* deprecated */
@@ -126,6 +141,8 @@ static const struct {
   { "backups",          &opt.backups,           cmd_number },
   { "base",             &opt.base_href,         cmd_string },
   { "bindaddress",      &opt.bind_address,      cmd_string },
+  { "bodydata",         &opt.body_data,         cmd_string },
+  { "bodyfile",         &opt.body_file,         cmd_string },
 #ifdef HAVE_SSL
   { "cacertificate",    &opt.ca_cert,           cmd_file },
 #endif
@@ -136,17 +153,19 @@ static const struct {
   { "certificatetype",  &opt.cert_type,         cmd_cert_type },
   { "checkcertificate", &opt.check_cert,        cmd_boolean },
 #endif
-  { "chooseconfig",     &opt.choose_config,	cmd_file },
+  { "chooseconfig",     &opt.choose_config,     cmd_file },
   { "connecttimeout",   &opt.connect_timeout,   cmd_time },
   { "contentdisposition", &opt.content_disposition, cmd_boolean },
+  { "contentonerror",   &opt.content_on_error,  cmd_boolean },
   { "continue",         &opt.always_rest,       cmd_boolean },
   { "convertlinks",     &opt.convert_links,     cmd_boolean },
   { "cookies",          &opt.cookies,           cmd_boolean },
-  { "cutdirs",          &opt.cut_dirs,          cmd_number },
-#ifdef ENABLE_DEBUG
-  { "debug",            &opt.debug,             cmd_boolean },
+#ifdef HAVE_SSL
+  { "crlfile",      &opt.crl_file,          cmd_file_once },
 #endif
-  { "defaultpage", 	&opt.default_page,      cmd_string},
+  { "cutdirs",          &opt.cut_dirs,          cmd_number },
+  { "debug",            &opt.debug,             cmd_boolean },
+  { "defaultpage",      &opt.default_page,      cmd_string },
   { "deleteafter",      &opt.delete_after,      cmd_boolean },
   { "dirprefix",        &opt.dir_prefix,        cmd_directory },
   { "dirstruct",        NULL,                   cmd_spec_dirstruct },
@@ -180,6 +199,9 @@ static const struct {
   { "httppasswd",       &opt.http_passwd,       cmd_string }, /* deprecated */
   { "httppassword",     &opt.http_passwd,       cmd_string },
   { "httpproxy",        &opt.http_proxy,        cmd_string },
+#ifdef HAVE_SSL
+  { "httpsonly",        &opt.https_only,        cmd_boolean },
+#endif
   { "httpsproxy",       &opt.https_proxy,       cmd_string },
   { "httpuser",         &opt.http_user,         cmd_string },
   { "ignorecase",       &opt.ignore_case,       cmd_boolean },
@@ -199,9 +221,11 @@ static const struct {
   { "logfile",          &opt.lfilename,         cmd_file },
   { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
   { "maxredirect",      &opt.max_redirect,      cmd_number },
+  { "method",           &opt.method,            cmd_string_uppercase },
   { "mirror",           NULL,                   cmd_spec_mirror },
   { "netrc",            &opt.netrc,             cmd_boolean },
   { "noclobber",        &opt.noclobber,         cmd_boolean },
+  { "noconfig",         &opt.noconfig,          cmd_boolean },
   { "noparent",         &opt.no_parent,         cmd_boolean },
   { "noproxy",          &opt.no_proxy,          cmd_vector },
   { "numtries",         &opt.ntry,              cmd_number_inf },/* deprecated*/
@@ -213,7 +237,7 @@ static const struct {
   { "postdata",         &opt.post_data,         cmd_string },
   { "postfile",         &opt.post_file_name,    cmd_file },
   { "preferfamily",     NULL,                   cmd_spec_prefer_family },
-  { "preservepermissions", &opt.preserve_perm,  cmd_boolean },/* deprecated */
+  { "preservepermissions", &opt.preserve_perm,  cmd_boolean },
 #ifdef HAVE_SSL
   { "privatekey",       &opt.private_key,       cmd_file },
   { "privatekeytype",   &opt.private_key_type,  cmd_cert_type },
@@ -233,10 +257,13 @@ static const struct {
   { "reclevel",         &opt.reclevel,          cmd_number_inf },
   { "recursive",        NULL,                   cmd_spec_recursive },
   { "referer",          &opt.referer,           cmd_string },
+  { "regextype",        &opt.regex_type,        cmd_spec_regex_type },
   { "reject",           &opt.rejects,           cmd_vector },
+  { "rejectregex",      &opt.rejectregex_s,     cmd_string },
   { "relativeonly",     &opt.relative_only,     cmd_boolean },
   { "remoteencoding",   &opt.encoding_remote,   cmd_string },
   { "removelisting",    &opt.remove_listing,    cmd_boolean },
+  { "reportspeed",             &opt.report_bps, cmd_spec_report_speed},
   { "restrictfilenames", NULL,                  cmd_spec_restrict_file_names },
   { "retrsymlinks",     &opt.retr_symlinks,     cmd_boolean },
   { "retryconnrefused", &opt.retry_connrefused, cmd_boolean },
@@ -248,8 +275,10 @@ static const struct {
 #endif
   { "serverresponse",   &opt.server_response,   cmd_boolean },
   { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean },
+  { "showprogress",     &opt.show_progress,      cmd_boolean },
   { "spanhosts",        &opt.spanhost,          cmd_boolean },
   { "spider",           &opt.spider,            cmd_boolean },
+  { "startpos",         &opt.start_pos,         cmd_bytes },
   { "strictcomments",   &opt.strict_comments,   cmd_boolean },
   { "timeout",          NULL,                   cmd_spec_timeout },
   { "timestamping",     &opt.timestamping,      cmd_boolean },
@@ -263,6 +292,17 @@ static const struct {
   { "verbose",          NULL,                   cmd_spec_verbose },
   { "wait",             &opt.wait,              cmd_time },
   { "waitretry",        &opt.waitretry,         cmd_time },
+  { "warccdx",          &opt.warc_cdx_enabled,  cmd_boolean },
+  { "warccdxdedup",     &opt.warc_cdx_dedup_filename,  cmd_file },
+#ifdef HAVE_LIBZ
+  { "warccompression",  &opt.warc_compression_enabled, cmd_boolean },
+#endif
+  { "warcdigests",      &opt.warc_digests_enabled, cmd_boolean },
+  { "warcfile",         &opt.warc_filename,     cmd_file },
+  { "warcheader",       NULL,                   cmd_spec_warc_header },
+  { "warckeeplog",      &opt.warc_keep_log,     cmd_boolean },
+  { "warcmaxsize",      &opt.warc_maxsize,      cmd_bytes },
+  { "warctempdir",      &opt.warc_tempdir,      cmd_directory },
 #ifdef USE_WATT32
   { "wdebug",           &opt.wdebug,            cmd_boolean },
 #endif
@@ -281,7 +321,7 @@ command_by_name (const char *cmdname)
   while (lo <= hi)
     {
       int mid = (lo + hi) >> 1;
-      int cmp = strcasecmp (cmdname, commands[mid].name);
+      int cmp = c_strcasecmp (cmdname, commands[mid].name);
       if (cmp < 0)
         hi = mid - 1;
       else if (cmp > 0)
@@ -291,7 +331,7 @@ command_by_name (const char *cmdname)
     }
   return -1;
 }
-
+
 /* Reset the variables to default values.  */
 void
 defaults (void)
@@ -332,6 +372,22 @@ defaults (void)
 
   opt.dns_cache = true;
   opt.ftp_pasv = true;
+  /* 2014-09-07  Darshit Shah  <darnir@gmail.com>
+   * opt.retr_symlinks is set to true by default. Creating symbolic links on the
+   * local filesystem pose a security threat by malicious FTP Servers that
+   * server a specially crafted .listing file akin to this:
+   *
+   * lrwxrwxrwx   1 root     root           33 Dec 25  2012 JoCxl6d8rFU -> /
+   * drwxrwxr-x  15 1024     106          4096 Aug 28 02:02 JoCxl6d8rFU
+   *
+   * A .listing file in this fashion makes Wget susceptiple to a symlink attack
+   * wherein the attacker is able to create arbitrary files, directories and
+   * symbolic links on the target system and even set permissions.
+   *
+   * Hence, by default Wget attempts to retrieve the pointed-to files and does
+   * not create the symbolic links locally.
+   */
+  opt.retr_symlinks = true;
 
 #ifdef HAVE_SSL
   opt.check_cert = true;
@@ -347,6 +403,8 @@ defaults (void)
   opt.restrict_files_nonascii = false;
   opt.restrict_files_case = restrict_no_case_restriction;
 
+  opt.regex_type = regex_type_posix;
+
   opt.max_redirect = 20;
 
   opt.waitretry = 10;
@@ -361,8 +419,25 @@ defaults (void)
 
   opt.useservertimestamps = true;
   opt.show_all_dns_entries = false;
+
+  opt.warc_maxsize = 0; /* 1024 * 1024 * 1024; */
+#ifdef HAVE_LIBZ
+  opt.warc_compression_enabled = true;
+#else
+  opt.warc_compression_enabled = false;
+#endif
+  opt.warc_digests_enabled = true;
+  opt.warc_cdx_enabled = false;
+  opt.warc_cdx_dedup_filename = NULL;
+  opt.warc_tempdir = NULL;
+  opt.warc_keep_log = true;
+
+  /* Use a negative value to mark the absence of --start-pos option */
+  opt.start_pos = -1;
+  opt.show_progress = false;
+  opt.noscroll = false;
 }
-
+
 /* Return the user's home directory (strdup-ed), or NULL if none is
    found.  */
 char *
@@ -370,7 +445,6 @@ home_dir (void)
 {
   static char *buf = NULL;
   static char *home, *ret;
-  int len;
 
   if (!home)
     {
@@ -378,6 +452,8 @@ home_dir (void)
       if (!home)
         {
 #if defined(MSDOS)
+          int len;
+
           /* Under MSDOS, if $HOME isn't defined, use the directory where
              `wget.exe' resides.  */
           const char *_w32_get_argv0 (void); /* in libwatt.a/pcconfig.c */
@@ -391,12 +467,7 @@ home_dir (void)
           assert (p);
 
           len = p - buff + 1;
-          buff = malloc (len + 1);
-          if (buff == NULL)
-            return NULL;
-
-          strncpy (buff, _w32_get_argv0 (), len);
-          buff[len] = '\0';
+          buff = strdup (_w32_get_argv0 ());
 
           home = buf;
 #elif !defined(WINDOWS)
@@ -415,8 +486,7 @@ home_dir (void)
     }
 
   ret = home ? xstrdup (home) : NULL;
-  if (buf)
-    free (buf);
+  xfree (buf);
 
   return ret;
 }
@@ -435,14 +505,14 @@ wgetrc_env_file_name (void)
         {
           fprintf (stderr, _("%s: WGETRC points to %s, which doesn't exist.\n"),
                    exec_name, env);
-          exit (1);
+          exit (WGET_EXIT_GENERIC_ERROR);
         }
       return xstrdup (env);
     }
   return NULL;
 }
 
-/* Check for the existance of '$HOME/.wgetrc' and return it's path
+/* Check for the existance of '$HOME/.wgetrc' and return its path
    if it exists and is set.  */
 char *
 wgetrc_user_file_name (void)
@@ -457,7 +527,7 @@ wgetrc_user_file_name (void)
   home = home_dir ();
   if (home)
     file = aprintf ("%s/.wgetrc", home);
-  xfree_null (home);
+  xfree (home);
 #endif /* def __VMS [else] */
 
   if (!file)
@@ -492,8 +562,7 @@ wgetrc_file_name (void)
   if (!file)
     {
       char *home = home_dir ();
-      xfree_null (file);
-      file = NULL;
+      xfree (file);
       home = ws_mypath ();
       if (home)
         {
@@ -501,7 +570,6 @@ wgetrc_file_name (void)
           if (!file_exists_p (file))
             {
               xfree (file);
-              file = NULL;
             }
           xfree (home);
         }
@@ -530,7 +598,8 @@ bool
 run_wgetrc (const char *file)
 {
   FILE *fp;
-  char *line;
+  char *line = NULL;
+  size_t bufsize = 0;
   int ln;
   int errcnt = 0;
 
@@ -542,7 +611,7 @@ run_wgetrc (const char *file)
       return true;                      /* not a fatal error */
     }
   ln = 1;
-  while ((line = read_whole_line (fp)) != NULL)
+  while (getline (&line, &bufsize, fp) > 0)
     {
       char *com = NULL, *val = NULL;
       int comind;
@@ -574,11 +643,11 @@ run_wgetrc (const char *file)
         default:
           abort ();
         }
-      xfree_null (com);
-      xfree_null (val);
-      xfree (line);
+      xfree (com);
+      xfree (val);
       ++ln;
     }
+  xfree (line);
   fclose (fp);
 
   return errcnt == 0;
@@ -596,21 +665,34 @@ initialize (void)
      variable has been set. For internal testing purposes only!  */
   env_sysrc = getenv ("SYSTEM_WGETRC");
   if (env_sysrc && file_exists_p (env_sysrc))
-    ok &= run_wgetrc (env_sysrc);
+    {
+      ok &= run_wgetrc (env_sysrc);
+      /* If there are any problems parsing the system wgetrc file, tell
+         the user and exit */
+      if (! ok)
+        {
+          fprintf (stderr, _("\
+Parsing system wgetrc file (env SYSTEM_WGETRC) failed.  Please check\n\
+'%s',\n\
+or specify a different file using --config.\n"), env_sysrc);
+          exit (WGET_EXIT_PARSE_ERROR);
+        }
+    }
   /* Otherwise, if SYSTEM_WGETRC is defined, use it.  */
 #ifdef SYSTEM_WGETRC
   else if (file_exists_p (SYSTEM_WGETRC))
     ok &= run_wgetrc (SYSTEM_WGETRC);
-#endif
   /* If there are any problems parsing the system wgetrc file, tell
      the user and exit */
   if (! ok)
     {
       fprintf (stderr, _("\
-Parsing system wgetrc file failed, please check '%s'.           \
-Or specify a different file using --config\n"), SYSTEM_WGETRC);
-      exit (2);
+Parsing system wgetrc file failed.  Please check\n\
+'%s',\n\
+or specify a different file using --config.\n"), SYSTEM_WGETRC);
+      exit (WGET_EXIT_PARSE_ERROR);
     }
+#endif
   /* Override it with your own, if one exists.  */
   file = wgetrc_file_name ();
   if (!file)
@@ -630,7 +712,7 @@ Or specify a different file using --config\n"), SYSTEM_WGETRC);
 
   /* If there were errors processing either `.wgetrc', abort. */
   if (!ok)
-    exit (2);
+    exit (WGET_EXIT_PARSE_ERROR);
 
   xfree (file);
   return;
@@ -754,16 +836,16 @@ setval_internal_tilde (int comind, const char *com, const char *val)
       pstring = commands[comind].place;
       home = home_dir ();
       if (home)
-	{
-	  homelen = strlen (home);
-	  while (homelen && ISSEP (home[homelen - 1]))
-            home[--homelen] = '\0';
+        {
+          homelen = strlen (home);
+          while (homelen && ISSEP (home[homelen - 1]))
+                 home[--homelen] = '\0';
 
-	  /* Skip the leading "~/". */
-	  for (++val; ISSEP (*val); val++)
-  	    ;
-	  *pstring = concat_strings (home, "/", val, (char *)0);
-	}
+          /* Skip the leading "~/". */
+          for (++val; ISSEP (*val); val++)
+            ;
+          *pstring = concat_strings (home, "/", val, (char *)0);
+        }
     }
   return ret;
 }
@@ -771,8 +853,8 @@ setval_internal_tilde (int comind, const char *com, const char *val)
 /* Run command COM with value VAL.  If running the command produces an
    error, report the error and exit.
 
-   This is intended to be called from main() to modify Wget's behavior
-   through command-line switches.  Since COM is hard-coded in main(),
+   This is intended to be called from main to modify Wget's behavior
+   through command-line switches.  Since COM is hard-coded in main,
    it is not canonicalized, and this aborts when COM is not found.
 
    If COMIND's are exported to init.h, this function will be changed
@@ -789,7 +871,7 @@ setoptval (const char *com, const char *val, const char *optname)
 
   assert (val != NULL);
   if (!setval_internal (command_by_name (com), dd_optname, val))
-    exit (2);
+    exit (WGET_EXIT_PARSE_ERROR);
 }
 
 /* Parse OPT into command and value and run it.  For example,
@@ -797,25 +879,25 @@ setoptval (const char *com, const char *val, const char *optname)
    This is used by the `--execute' flag in main.c.  */
 
 void
-run_command (const char *opt)
+run_command (const char *cmdopt)
 {
   char *com, *val;
   int comind;
-  switch (parse_line (opt, &com, &val, &comind))
+  switch (parse_line (cmdopt, &com, &val, &comind))
     {
     case line_ok:
       if (!setval_internal (comind, com, val))
-        exit (2);
+        exit (WGET_EXIT_PARSE_ERROR);
       xfree (com);
       xfree (val);
       break;
     default:
       fprintf (stderr, _("%s: Invalid --execute command %s\n"),
-               exec_name, quote (opt));
-      exit (2);
+               exec_name, quote (cmdopt));
+      exit (WGET_EXIT_PARSE_ERROR);
     }
 }
-
+
 /* Generic helper functions, for use with `commands'. */
 
 /* Forward declarations: */
@@ -883,7 +965,7 @@ cmd_number (const char *com, const char *val, void *place)
 static bool
 cmd_number_inf (const char *com, const char *val, void *place)
 {
-  if (!strcasecmp (val, "inf"))
+  if (!c_strcasecmp (val, "inf"))
     {
       *(int *) place = 0;
       return true;
@@ -894,25 +976,42 @@ cmd_number_inf (const char *com, const char *val, void *place)
 /* Copy (strdup) the string at COM to a new location and place a
    pointer to *PLACE.  */
 static bool
-cmd_string (const char *com, const char *val, void *place)
+cmd_string (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
-  xfree_null (*pstring);
+  xfree (*pstring);
   *pstring = xstrdup (val);
   return true;
 }
 
+/* Like cmd_string but ensure the string is upper case.  */
+static bool
+cmd_string_uppercase (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  char *q, **pstring;
+  pstring = (char **)place;
+  xfree (*pstring);
 
-/* Like the above, but handles tilde-expansion when reading a user's
+  *pstring = xmalloc (strlen (val) + 1);
+
+  for (q = *pstring; *val; val++, q++)
+    *q = c_toupper (*val);
+
+  *q = '\0';
+  return true;
+}
+
+
+/* Like cmd_string, but handles tilde-expansion when reading a user's
    `.wgetrc'.  In that case, and if VAL begins with `~', the tilde
    gets expanded to the user's home directory.  */
 static bool
-cmd_file (const char *com, const char *val, void *place)
+cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
-  xfree_null (*pstring);
+  xfree (*pstring);
 
   /* #### If VAL is empty, perhaps should set *PLACE to NULL.  */
 
@@ -930,6 +1029,20 @@ cmd_file (const char *com, const char *val, void *place)
   return true;
 }
 
+/* like cmd_file, but insist on just a single option usage */
+static bool
+cmd_file_once (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  if (*(char **)place)
+    {
+      fprintf (stderr, _("%s: %s must only be used once\n"),
+               exec_name, com);
+      return false;
+    }
+
+  return cmd_file(com, val, place);
+}
+
 /* Like cmd_file, but strips trailing '/' characters.  */
 static bool
 cmd_directory (const char *com, const char *val, void *place)
@@ -955,7 +1068,7 @@ cmd_directory (const char *com, const char *val, void *place)
    PLACE vector is cleared instead.  */
 
 static bool
-cmd_vector (const char *com, const char *val, void *place)
+cmd_vector (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char ***pvec = (char ***)place;
 
@@ -970,7 +1083,7 @@ cmd_vector (const char *com, const char *val, void *place)
 }
 
 static bool
-cmd_directory_vector (const char *com, const char *val, void *place)
+cmd_directory_vector (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char ***pvec = (char ***)place;
 
@@ -1179,14 +1292,14 @@ cmd_cert_type (const char *com, const char *val, void *place)
   return ok;
 }
 #endif
-
+
 /* Specialized helper functions, used by `commands' to handle some
    options specially.  */
 
 static bool check_user_specified_header (const char *);
 
 static bool
-cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored)
+cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   if (!cmd_boolean (com, val, &opt.dirstruct))
     return false;
@@ -1200,7 +1313,7 @@ cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_header (const char *com, const char *val, void *place_ignored)
+cmd_spec_header (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   /* Empty value means reset the list of headers. */
   if (*val == '\0')
@@ -1221,7 +1334,28 @@ cmd_spec_header (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_htmlify (const char *com, const char *val, void *place_ignored)
+cmd_spec_warc_header (const char *com, const char *val, void *place_ignored _GL_UNUSED)
+{
+  /* Empty value means reset the list of headers. */
+  if (*val == '\0')
+    {
+      free_vec (opt.warc_user_headers);
+      opt.warc_user_headers = NULL;
+      return true;
+    }
+
+  if (!check_user_specified_header (val))
+    {
+      fprintf (stderr, _("%s: %s: Invalid WARC header %s.\n"),
+               exec_name, com, quote (val));
+      return false;
+    }
+  opt.warc_user_headers = vec_append (opt.warc_user_headers, val);
+  return true;
+}
+
+static bool
+cmd_spec_htmlify (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   int flag = cmd_boolean (com, val, &opt.htmlify);
   if (flag && !opt.htmlify)
@@ -1233,7 +1367,7 @@ cmd_spec_htmlify (const char *com, const char *val, void *place_ignored)
    no limit on max. recursion depth, and don't remove listings.  */
 
 static bool
-cmd_spec_mirror (const char *com, const char *val, void *place_ignored)
+cmd_spec_mirror (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   int mirror;
 
@@ -1255,7 +1389,7 @@ cmd_spec_mirror (const char *com, const char *val, void *place_ignored)
    "IPv4", "IPv6", and "none".  */
 
 static bool
-cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored)
+cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   static const struct decode_item choices[] = {
     { "IPv4", prefer_ipv4 },
@@ -1274,7 +1408,7 @@ cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored)
    implementation before that.  */
 
 static bool
-cmd_spec_progress (const char *com, const char *val, void *place_ignored)
+cmd_spec_progress (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   if (!valid_progress_implementation_p (val))
     {
@@ -1282,10 +1416,10 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored)
                exec_name, com, quote (val));
       return false;
     }
-  xfree_null (opt.progress_type);
+  xfree (opt.progress_type);
 
   /* Don't call set_progress_implementation here.  It will be called
-     in main() when it becomes clear what the log output is.  */
+     in main when it becomes clear what the log output is.  */
   opt.progress_type = xstrdup (val);
   return true;
 }
@@ -1295,7 +1429,7 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored)
    is specified.  */
 
 static bool
-cmd_spec_recursive (const char *com, const char *val, void *place_ignored)
+cmd_spec_recursive (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   if (!cmd_boolean (com, val, &opt.recursive))
     return false;
@@ -1307,8 +1441,27 @@ cmd_spec_recursive (const char *com, const char *val, void *place_ignored)
   return true;
 }
 
+/* Validate --regex-type and set the choice.  */
+
 static bool
-cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored)
+cmd_spec_regex_type (const char *com, const char *val, void *place_ignored _GL_UNUSED)
+{
+  static const struct decode_item choices[] = {
+    { "posix", regex_type_posix },
+#ifdef HAVE_LIBPCRE
+    { "pcre",  regex_type_pcre },
+#endif
+  };
+  int regex_type = regex_type_posix;
+  int ok = decode_string (val, choices, countof (choices), &regex_type);
+  if (!ok)
+    fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val));
+  opt.regex_type = regex_type;
+  return ok;
+}
+
+static bool
+cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   int restrict_os = opt.restrict_files_os;
   int restrict_ctrl = opt.restrict_files_ctrl;
@@ -1361,6 +1514,15 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
   return true;
 }
 
+static bool
+cmd_spec_report_speed (const char *com, const char *val, void *place_ignored _GL_UNUSED)
+{
+  opt.report_bps = c_strcasecmp (val, "bits") == 0;
+  if (!opt.report_bps)
+    fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val));
+  return opt.report_bps;
+}
+
 #ifdef HAVE_SSL
 static bool
 cmd_spec_secure_protocol (const char *com, const char *val, void *place)
@@ -1370,6 +1532,9 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
     { "sslv2", secure_protocol_sslv2 },
     { "sslv3", secure_protocol_sslv3 },
     { "tlsv1", secure_protocol_tlsv1 },
+    { "tlsv1_1", secure_protocol_tlsv1_1 },
+    { "tlsv1_2", secure_protocol_tlsv1_2 },
+    { "pfs", secure_protocol_pfs },
   };
   int ok = decode_string (val, choices, countof (choices), place);
   if (!ok)
@@ -1381,7 +1546,7 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
 /* Set all three timeout values. */
 
 static bool
-cmd_spec_timeout (const char *com, const char *val, void *place_ignored)
+cmd_spec_timeout (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   double value;
   if (!cmd_time (com, val, &value))
@@ -1393,7 +1558,7 @@ cmd_spec_timeout (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_useragent (const char *com, const char *val, void *place_ignored)
+cmd_spec_useragent (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   /* Disallow embedded newlines.  */
   if (strchr (val, '\n'))
@@ -1402,7 +1567,7 @@ cmd_spec_useragent (const char *com, const char *val, void *place_ignored)
                exec_name, com, quote (val));
       return false;
     }
-  xfree_null (opt.useragent);
+  xfree (opt.useragent);
   opt.useragent = xstrdup (val);
   return true;
 }
@@ -1412,7 +1577,7 @@ cmd_spec_useragent (const char *com, const char *val, void *place_ignored)
    some random hackery for disallowing -q -v).  */
 
 static bool
-cmd_spec_verbose (const char *com, const char *val, void *place_ignored)
+cmd_spec_verbose (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   bool flag;
   if (cmd_boolean (com, val, &flag))
@@ -1422,7 +1587,7 @@ cmd_spec_verbose (const char *com, const char *val, void *place_ignored)
     }
   return false;
 }
-
+
 /* Miscellaneous useful routines.  */
 
 /* A very simple atoi clone, more useful than atoi because it works on
@@ -1557,7 +1722,7 @@ decode_string (const char *val, const struct decode_item *items, int itemcount,
 {
   int i;
   for (i = 0; i < itemcount; i++)
-    if (0 == strcasecmp (val, items[i].name))
+    if (0 == c_strcasecmp (val, items[i].name))
       {
         *place = items[i].code;
         return true;
@@ -1565,18 +1730,22 @@ decode_string (const char *val, const struct decode_item *items, int itemcount,
   return false;
 }
 
-
-void cleanup_html_url (void);
-
-
 /* Free the memory allocated by global variables.  */
 void
 cleanup (void)
 {
   /* Free external resources, close files, etc. */
 
+  /* Close WARC file. */
+  if (opt.warc_filename != 0)
+    warc_close ();
+
+  log_close ();
+
   if (output_stream)
-    fclose (output_stream);
+    if (fclose (output_stream) == EOF)
+      inform_exit_status (CLOSEFAILED);
+
   /* No need to check for error because Wget flushes its output (and
      checks for errors) after any data arrives.  */
 
@@ -1593,90 +1762,88 @@ cleanup (void)
   res_cleanup ();
   http_cleanup ();
   cleanup_html_url ();
+  spider_cleanup ();
   host_cleanup ();
   log_cleanup ();
+  netrc_cleanup ();
 
-  {
-    extern acc_t *netrc_list;
-    free_netrc (netrc_list);
-  }
-  xfree_null (opt.choose_config);
-  xfree_null (opt.lfilename);
-  xfree_null (opt.dir_prefix);
-  xfree_null (opt.input_filename);
-  xfree_null (opt.output_document);
+  xfree (opt.choose_config);
+  xfree (opt.lfilename);
+  xfree (opt.dir_prefix);
+  xfree (opt.input_filename);
+  xfree (opt.output_document);
   free_vec (opt.accepts);
   free_vec (opt.rejects);
-  free_vec (opt.excludes);
-  free_vec (opt.includes);
+  free_vec ((char **)opt.excludes);
+  free_vec ((char **)opt.includes);
   free_vec (opt.domains);
   free_vec (opt.follow_tags);
   free_vec (opt.ignore_tags);
-  xfree_null (opt.progress_type);
-  xfree_null (opt.ftp_user);
-  xfree_null (opt.ftp_passwd);
-  xfree_null (opt.ftp_proxy);
-  xfree_null (opt.https_proxy);
-  xfree_null (opt.http_proxy);
+  xfree (opt.progress_type);
+  xfree (opt.ftp_user);
+  xfree (opt.ftp_passwd);
+  xfree (opt.ftp_proxy);
+  xfree (opt.https_proxy);
+  xfree (opt.http_proxy);
   free_vec (opt.no_proxy);
-  xfree_null (opt.useragent);
-  xfree_null (opt.referer);
-  xfree_null (opt.http_user);
-  xfree_null (opt.http_passwd);
+  xfree (opt.useragent);
+  xfree (opt.referer);
+  xfree (opt.http_user);
+  xfree (opt.http_passwd);
   free_vec (opt.user_headers);
+  free_vec (opt.warc_user_headers);
 # ifdef HAVE_SSL
-  xfree_null (opt.cert_file);
-  xfree_null (opt.private_key);
-  xfree_null (opt.ca_directory);
-  xfree_null (opt.ca_cert);
-  xfree_null (opt.random_file);
-  xfree_null (opt.egd_file);
+  xfree (opt.cert_file);
+  xfree (opt.private_key);
+  xfree (opt.ca_directory);
+  xfree (opt.ca_cert);
+  xfree (opt.crl_file);
+  xfree (opt.random_file);
+  xfree (opt.egd_file);
 # endif
-  xfree_null (opt.bind_address);
-  xfree_null (opt.cookies_input);
-  xfree_null (opt.cookies_output);
-  xfree_null (opt.user);
-  xfree_null (opt.passwd);
-  xfree_null (opt.base_href);
+  xfree (opt.bind_address);
+  xfree (opt.cookies_input);
+  xfree (opt.cookies_output);
+  xfree (opt.user);
+  xfree (opt.passwd);
+  xfree (opt.base_href);
+  xfree (opt.method);
+  xfree (opt.post_file_name);
+  xfree (opt.post_data);
+  xfree (opt.body_data);
+  xfree (opt.body_file);
 
 #endif /* DEBUG_MALLOC */
 }
-
+
 /* Unit testing routines.  */
 
 #ifdef TESTING
 
 const char *
-test_commands_sorted()
+test_commands_sorted(void)
 {
-  int prev_idx = 0, next_idx = 1;
-  int command_count = countof (commands) - 1;
-  int cmp = 0;
-  while (next_idx <= command_count)
+  unsigned i;
+
+  for (i = 1; i < countof(commands); ++i)
     {
-      cmp = strcasecmp (commands[prev_idx].name, commands[next_idx].name);
-      if (cmp > 0)
+      if (c_strcasecmp (commands[i - 1].name, commands[i].name) > 0)
         {
           mu_assert ("FAILED", false);
           break;
         }
-      else
-        {
-          prev_idx ++;
-	  next_idx ++;
-        }
     }
   return NULL;
 }
 
 const char *
-test_cmd_spec_restrict_file_names()
+test_cmd_spec_restrict_file_names(void)
 {
-  int i;
-  struct {
-    char *val;
+  unsigned i;
+  static const struct {
+    const char *val;
     int expected_restrict_files_os;
-    int expected_restrict_files_ctrl;
+    bool expected_restrict_files_ctrl;
     int expected_restrict_files_case;
     bool result;
   } test_array[] = {
@@ -1686,7 +1853,7 @@ test_cmd_spec_restrict_file_names()
     { "unix,nocontrol,lowercase,", restrict_unix, false, restrict_lowercase, true },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       bool res;
 
@@ -1701,13 +1868,12 @@ test_cmd_spec_restrict_file_names()
       */
       mu_assert ("test_cmd_spec_restrict_file_names: wrong result",
                  res == test_array[i].result
-                 && opt.restrict_files_os   == test_array[i].expected_restrict_files_os
+                 && (int) opt.restrict_files_os   == test_array[i].expected_restrict_files_os
                  && opt.restrict_files_ctrl == test_array[i].expected_restrict_files_ctrl
-                 && opt.restrict_files_case == test_array[i].expected_restrict_files_case);
+                 && (int) opt.restrict_files_case == test_array[i].expected_restrict_files_case);
     }
 
   return NULL;
 }
 
 #endif /* TESTING */
-

src/iri.c

@@ -38,15 +38,14 @@ as that of the covered work.  */
 #include <errno.h>
 
 #include "utils.h"
+#include "url.h"
+#include "c-strcase.h"
 
 /* RFC3987 section 3.1 mandates STD3 ASCII RULES */
 #define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
 
 /* Note: locale encoding is kept in options struct (opt.locale) */
 
-static bool do_conversion (iconv_t cd, char *in, size_t inlen, char **out);
-
-
 /* Given a string containing "charset=XXX", return the encoding if found,
    or NULL otherwise */
 char *
@@ -110,52 +109,35 @@ check_encoding_name (char *encoding)
   return true;
 }
 
-/* Try converting string str from locale to UTF-8. Return a new string
-   on success, or str on error or if conversion isn't needed. */
-const char *
-locale_to_utf8 (const char *str)
-{
-  iconv_t l2u;
-  char *new;
-
-  /* That shouldn't happen, just in case */
-  if (!opt.locale)
-    {
-      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
-      opt.locale = find_locale ();
-    }
-
-  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
-    return str;
-
-  l2u = iconv_open ("UTF-8", opt.locale);
-  if (l2u != (iconv_t)(-1))
-    {
-      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
-                 quote (opt.locale), quote ("UTF-8"));
-      return str;
-    }
-
-  if (do_conversion (l2u, (char *) str, strlen ((char *) str), &new))
-    return (const char *) new;
-
-  return str;
-}
-
 /* Do the conversion according to the passed conversion descriptor cd. *out
    will contain the transcoded string on success. *out content is
    unspecified otherwise. */
 static bool
-do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
+do_conversion (const char *tocode, const char *fromcode, char *in, size_t inlen, char **out)
 {
+  iconv_t cd;
   /* sXXXav : hummm hard to guess... */
-  size_t len, done, outlen = inlen * 2;
+  size_t len, done, outlen;
   int invalid = 0, tooshort = 0;
-  char *s;
+  char *s, *in_org, *in_save;
 
-  s = xmalloc (outlen + 1);
-  *out = s;
-  len = outlen;
+  cd = iconv_open (tocode, fromcode);
+  if (cd == (iconv_t)(-1))
+    {
+      logprintf (LOG_VERBOSE, _("Conversion from %s to UTF-8 isn't supported\n"),
+                 quote (opt.locale));
+      *out = NULL;
+      return false;
+    }
+
+  /* iconv() has to work on an unescaped string */
+  in_org = in;
+  in_save = in = strndup(in, inlen);
+  url_unescape(in);
+  inlen = strlen(in);
+
+  len = outlen = inlen * 2;
+  *out = s = xmalloc (outlen + 1);
   done = 0;
 
   for (;;)
@@ -164,6 +146,9 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         {
           *out = s;
           *(s + len - outlen - done) = '\0';
+          xfree(in_save);
+          iconv_close(cd);
+          DEBUGP (("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode));
           return true;
         }
 
@@ -202,9 +187,35 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         }
     }
 
+    xfree(in_save);
+    iconv_close(cd);
+    DEBUGP (("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode));
     return false;
 }
 
+/* Try converting string str from locale to UTF-8. Return a new string
+   on success, or str on error or if conversion isn't needed. */
+const char *
+locale_to_utf8 (const char *str)
+{
+  char *new;
+
+  /* That shouldn't happen, just in case */
+  if (!opt.locale)
+    {
+      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
+      opt.locale = find_locale ();
+    }
+
+  if (!opt.locale || !c_strcasecmp (opt.locale, "utf-8"))
+    return str;
+
+  if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new))
+    return (const char *) new;
+
+  return str;
+}
+
 /* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
    on error. */
 char *
@@ -256,22 +267,21 @@ idn_decode (char *host)
 /* Try to transcode string str from remote encoding to UTF-8. On success, *new
    contains the transcoded string. *new content is unspecified otherwise. */
 bool
-remote_to_utf8 (struct iri *i, const char *str, const char **new)
+remote_to_utf8 (struct iri *iri, const char *str, const char **new)
 {
-  iconv_t cd;
   bool ret = false;
 
-  if (!i->uri_encoding)
+  if (!iri->uri_encoding)
     return false;
 
   /* When `i->uri_encoding' == "UTF-8" there is nothing to convert.  But we must
      test for non-ASCII symbols for correct hostname processing in `idn_encode'
      function. */
-  if (!strcmp (i->uri_encoding, "UTF-8"))
+  if (!c_strcasecmp (iri->uri_encoding, "UTF-8"))
     {
-      int i, len = strlen (str);
-      for (i = 0; i < len; i++)
-        if ((unsigned char) str[i] >= (unsigned char) '\200')
+      const unsigned char *p;
+      for (p = (unsigned char *) str; *p; p++)
+        if (*p > 127)
           {
             *new = strdup (str);
             return true;
@@ -279,19 +289,13 @@ remote_to_utf8 (struct iri *i, const char *str, const char **new)
       return false;
     }
 
-  cd = iconv_open ("UTF-8", i->uri_encoding);
-  if (cd == (iconv_t)(-1))
-    return false;
-
-  if (do_conversion (cd, (char *) str, strlen ((char *) str), (char **) new))
+  if (do_conversion ("UTF-8", iri->uri_encoding, (char *) str, strlen (str), (char **) new))
     ret = true;
 
-  iconv_close (cd);
-
   /* Test if something was converted */
-  if (!strcmp (str, *new))
+  if (*new && !strcmp (str, *new))
     {
-      xfree ((char *) *new);
+      xfree (*new);
       return false;
     }
 
@@ -325,10 +329,13 @@ struct iri *iri_dup (const struct iri *src)
 void
 iri_free (struct iri *i)
 {
-  xfree_null (i->uri_encoding);
-  xfree_null (i->content_encoding);
-  xfree_null (i->orig_url);
-  xfree (i);
+  if (i)
+    {
+      xfree (i->uri_encoding);
+      xfree (i->content_encoding);
+      xfree (i->orig_url);
+      xfree (i);
+    }
 }
 
 /* Set uri_encoding of struct iri i. If a remote encoding was specified, use
@@ -341,7 +348,7 @@ set_uri_encoding (struct iri *i, char *charset, bool force)
     return;
   if (i->uri_encoding)
     {
-      if (charset && !strcasecmp (i->uri_encoding, charset))
+      if (charset && !c_strcasecmp (i->uri_encoding, charset))
         return;
       xfree (i->uri_encoding);
     }
@@ -358,11 +365,10 @@ set_content_encoding (struct iri *i, char *charset)
     return;
   if (i->content_encoding)
     {
-      if (charset && !strcasecmp (i->content_encoding, charset))
+      if (charset && !c_strcasecmp (i->content_encoding, charset))
         return;
       xfree (i->content_encoding);
     }
 
   i->content_encoding = charset ? xstrdup (charset) : NULL;
 }
-

src/log.c

@@ -39,6 +39,7 @@ as that of the covered work.  */
 #include <errno.h>
 
 #include "utils.h"
+#include "exits.h"
 #include "log.h"
 
 /* 2005-10-25 SMS.
@@ -72,13 +73,17 @@ as that of the covered work.  */
    - Inhibiting output.  When Wget receives SIGHUP, but redirecting
    the output fails, logging is inhibited.  */
 
-
+
 /* The file descriptor used for logging.  This is NULL before log_init
    is called; logging functions log to stderr then.  log_init sets it
    either to stderr or to a file pointer obtained from fopen().  If
    logging is inhibited, logfp is set back to NULL. */
 static FILE *logfp;
 
+/* A second file descriptor pointing to the temporary log file for the
+   WARC writer.  If WARC writing is disabled, this is NULL.  */
+static FILE *warclogfp;
+
 /* If true, it means logging is inhibited, i.e. nothing is printed or
    stored.  */
 static bool inhibit_logging;
@@ -137,7 +142,7 @@ static int log_line_current = -1;
 static bool trailing_line;
 
 static void check_redirect_output (void);
-
+
 #define ROT_ADVANCE(num) do {                   \
   if (++num >= SAVED_LOG_LINES)                 \
     num = 0;                                    \
@@ -151,11 +156,7 @@ static void
 free_log_line (int num)
 {
   struct log_ln *ln = log_lines + num;
-  if (ln->malloced_line)
-    {
-      xfree (ln->malloced_line);
-      ln->malloced_line = NULL;
-    }
+  xfree (ln->malloced_line);
   ln->content = NULL;
 }
 
@@ -260,7 +261,7 @@ saved_append (const char *s)
       s = end;
     }
 }
-
+
 /* Check X against opt.verbose and opt.quiet.  The semantics is as
    follows:
 
@@ -274,6 +275,10 @@ saved_append (const char *s)
 #define CHECK_VERBOSE(x)                        \
   switch (x)                                    \
     {                                           \
+    case LOG_PROGRESS:                          \
+      if (!opt.show_progress)                   \
+        return;                                 \
+      break;                                    \
     case LOG_ALWAYS:                            \
       break;                                    \
     case LOG_NOTQUIET:                          \
@@ -304,7 +309,32 @@ get_log_fp (void)
     return logfp;
   return stderr;
 }
-
+
+/* Returns the file descriptor for the secondary log file. This is
+   WARCLOGFP, except if called before log_init, in which case it
+   returns stderr.  This is useful in case someone calls a logging
+   function before log_init.
+
+   If logging is inhibited, return NULL.  */
+
+static FILE *
+get_warc_log_fp (void)
+{
+  if (inhibit_logging)
+    return NULL;
+  if (warclogfp)
+    return warclogfp;
+  return NULL;
+}
+
+/* Sets the file descriptor for the secondary log file.  */
+
+void
+log_set_warc_log_fp (FILE * fp)
+{
+  warclogfp = fp;
+}
+
 /* Log a literal string S.  The string is logged as-is, without a
    newline appended.  */
 
@@ -312,13 +342,17 @@ void
 logputs (enum log_options o, const char *s)
 {
   FILE *fp;
+  FILE *warcfp;
 
   check_redirect_output ();
   if ((fp = get_log_fp ()) == NULL)
     return;
+  warcfp = get_warc_log_fp ();
   CHECK_VERBOSE (o);
 
   FPUTS (s, fp);
+  if (warcfp != NULL)
+    FPUTS (s, warcfp);
   if (save_context_p)
     saved_append (s);
   if (flush_log_p)
@@ -347,7 +381,7 @@ struct logvprintf_state {
    (An alternative approach would be to use va_copy, but that's not
    portable.)  */
 
-static bool
+static bool GCC_FORMAT_ATTR (2, 0)
 log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
                       va_list args)
 {
@@ -356,8 +390,9 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
   int available_size = sizeof (smallmsg);
   int numwritten;
   FILE *fp = get_log_fp ();
+  FILE *warcfp = get_warc_log_fp ();
 
-  if (!save_context_p)
+  if (!save_context_p && warcfp == NULL)
     {
       /* In the simple case just call vfprintf(), to avoid needless
          allocation and games with vsnprintf(). */
@@ -407,10 +442,12 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
     }
 
   /* Writing succeeded. */
-  saved_append (write_ptr);
+  if (save_context_p)
+    saved_append (write_ptr);
   FPUTS (write_ptr, fp);
-  if (state->bigmsg)
-    xfree (state->bigmsg);
+  if (warcfp != NULL)
+    FPUTS (write_ptr, warcfp);
+  xfree (state->bigmsg);
 
  flush:
   if (flush_log_p)
@@ -426,6 +463,7 @@ void
 logflush (void)
 {
   FILE *fp = get_log_fp ();
+  FILE *warcfp = get_warc_log_fp ();
   if (fp)
     {
 /* 2005-10-25 SMS.
@@ -440,6 +478,10 @@ logflush (void)
       fflush (fp);
 #endif /* def __VMS [else] */
     }
+
+  if (warcfp != NULL)
+    fflush (warcfp);
+
   needs_flushing = false;
 }
 
@@ -501,7 +543,7 @@ logprintf (enum log_options o, const char *fmt, ...)
       va_end (args);
 
       if (done && errno == EPIPE)
-        exit (1);
+        exit (WGET_EXIT_GENERIC_ERROR);
     }
   while (!done);
 }
@@ -533,7 +575,7 @@ debug_logprintf (const char *fmt, ...)
     }
 }
 #endif /* ENABLE_DEBUG */
-
+
 /* Open FILE and set up a logging stream.  If FILE cannot be opened,
    exit with status of 1.  */
 void
@@ -545,7 +587,7 @@ log_init (const char *file, bool appendp)
       if (!logfp)
         {
           fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
-          exit (1);
+          exit (WGET_EXIT_GENERIC_ERROR);
         }
     }
   else
@@ -573,14 +615,14 @@ log_init (const char *file, bool appendp)
     }
 }
 
-/* Close LOGFP, inhibit further logging and free the memory associated
-   with it.  */
+/* Close LOGFP (only if we opened it, not if it's stderr), inhibit
+   further logging and free the memory associated with it.  */
 void
 log_close (void)
 {
   int i;
 
-  if (logfp)
+  if (logfp && (logfp != stderr))
     fclose (logfp);
   logfp = NULL;
   inhibit_logging = true;
@@ -598,6 +640,7 @@ log_dump_context (void)
 {
   int num = log_line_current;
   FILE *fp = get_log_fp ();
+  FILE *warcfp = get_warc_log_fp ();
   if (!fp)
     return;
 
@@ -609,16 +652,25 @@ log_dump_context (void)
     {
       struct log_ln *ln = log_lines + num;
       if (ln->content)
-        FPUTS (ln->content, fp);
+        {
+          FPUTS (ln->content, fp);
+          if (warcfp != NULL)
+            FPUTS (ln->content, warcfp);
+        }
       ROT_ADVANCE (num);
     }
   while (num != log_line_current);
   if (trailing_line)
     if (log_lines[log_line_current].content)
-      FPUTS (log_lines[log_line_current].content, fp);
+      {
+        FPUTS (log_lines[log_line_current].content, fp);
+        if (warcfp != NULL)
+          FPUTS (log_lines[log_line_current].content, warcfp);
+      }
   fflush (fp);
+  fflush (warcfp);
 }
-
+
 /* String escape functions. */
 
 /* Return the number of non-printable characters in SOURCE.
@@ -790,9 +842,9 @@ log_cleanup (void)
 {
   size_t i;
   for (i = 0; i < countof (ring); i++)
-    xfree_null (ring[i].buffer);
+    xfree (ring[i].buffer);
 }
-
+
 /* When SIGHUP or SIGUSR1 are received, the output is redirected
    elsewhere.  Such redirection is only allowed once. */
 static enum { RR_NONE, RR_REQUESTED, RR_DONE } redirect_request = RR_NONE;
@@ -819,7 +871,7 @@ redirect_output (void)
          can do but disable printing completely. */
       fprintf (stderr, _("\n%s received.\n"), redirect_request_signal_name);
       fprintf (stderr, _("%s: %s; disabling logging.\n"),
-               logfile, strerror (errno));
+               (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
       inhibit_logging = true;
     }
   save_context_p = false;

src/log.h

@@ -34,7 +34,11 @@ as that of the covered work.  */
 /* The log file to which Wget writes to after HUP.  */
 #define DEFAULT_LOGFILE "wget-log"
 
-enum log_options { LOG_VERBOSE, LOG_NOTQUIET, LOG_NONVERBOSE, LOG_ALWAYS };
+#include <stdio.h>
+
+enum log_options { LOG_VERBOSE, LOG_NOTQUIET, LOG_NONVERBOSE, LOG_ALWAYS, LOG_PROGRESS };
+
+void log_set_warc_log_fp (FILE *);
 
 void logprintf (enum log_options, const char *, ...)
      GCC_FORMAT_ATTR (2, 3);

src/mswindows.c

@@ -1,6 +1,6 @@
 /* mswindows.c -- Windows-specific support
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -42,6 +42,7 @@ as that of the covered work.  */
 
 #include "utils.h"
 #include "url.h"
+#include "exits.h"
 
 #ifndef ES_SYSTEM_REQUIRED
 #define ES_SYSTEM_REQUIRED  0x00000001
@@ -84,11 +85,11 @@ windows_main (char **exec_name)
   if (p)
     *p = '\0';
 }
-
+
 static void
 ws_cleanup (void)
 {
-  xfree ((char*)exec_name);
+  xfree (exec_name);
   WSACleanup ();
 }
 
@@ -164,8 +165,8 @@ fake_fork_child (void)
       if (new_log_fp)
         {
           info->logfile_changed = true;
-          strncpy (info->lfilename, opt.lfilename, sizeof (info->lfilename));
-          info->lfilename[sizeof (info->lfilename) - 1] = '\0';
+          snprintf (info->lfilename, sizeof (info->lfilename), "%s",
+                    opt.lfilename);
           fclose (new_log_fp);
         }
     }
@@ -308,7 +309,7 @@ cleanup:
 
   /* We're the parent.  If all is well, terminate.  */
   if (rv)
-    exit (0);
+    exit (WGET_EXIT_SUCCESS);
 
   /* We failed, return.  */
 }
@@ -323,7 +324,7 @@ fork_to_background (void)
   rv = fake_fork_child ();
   if (rv < 0)
     {
-      fprintf (stderr, "fake_fork_child() failed\n");
+      fprintf (stderr, _("fake_fork_child() failed\n"));
       abort ();
     }
   else if (rv == 0)
@@ -331,7 +332,7 @@ fork_to_background (void)
       /* We're the parent.  */
       fake_fork ();
       /* If fake_fork() returns, it failed.  */
-      fprintf (stderr, "fake_fork() failed\n");
+      fprintf (stderr, _("fake_fork() failed\n"));
       abort ();
     }
   /* If we get here, we're the child.  */
@@ -366,8 +367,8 @@ static int old_percentage = -1;
 void
 ws_changetitle (const char *url)
 {
-  xfree_null (title_buf);
-  xfree_null (curr_url);
+  xfree (title_buf);
+  xfree (curr_url);
   title_buf = xmalloc (strlen (url) + 20);
   curr_url = xstrdup (url);
   old_percentage = -1;
@@ -461,7 +462,7 @@ ws_startup (void)
     {
       fprintf (stderr, _("%s: Couldn't find usable socket driver.\n"),
                exec_name);
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   if (data.wVersion < requested)
@@ -469,14 +470,14 @@ ws_startup (void)
       fprintf (stderr, _("%s: Couldn't find usable socket driver.\n"),
                exec_name);
       WSACleanup ();
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   atexit (ws_cleanup);
   set_sleep_mode ();
   SetConsoleCtrlHandler (ws_handler, TRUE);
 }
-
+
 /* run_with_timeout Windows implementation.  */
 
 /* Stack size 0 uses default thread stack-size (reserve+commit).
@@ -620,35 +621,35 @@ inet_ntop (int af, const void *src, char *dst, socklen_t cnt)
 void
 set_windows_fd_as_blocking_socket (int fd)
 {
-	/* 04/2011
+    /* 04/2011
      gnulib select() converts blocking sockets to nonblocking in windows
      discussed here:
      http://old.nabble.com/blocking-socket-is-nonblocking-after-calling-gnulib-
      select%28%29-in-windows-td31432857.html
 
      wget uses blocking sockets so we must convert them back to blocking.
-	*/
-	int ret = 0;
-	int wsagle = 0;
-	const int zero = 0;
+    */
+    int ret = 0;
+    int wsagle = 0;
+    const int zero = 0;
 
-	do
-	{
-		if(wsagle == WSAEINPROGRESS)
-		  Sleep(1);  /* use windows sleep */
-		
-		WSASetLastError (0);
-		ret = ioctl (fd, FIONBIO, &zero);
-		wsagle = WSAGetLastError();
-	}
+    do
+    {
+        if(wsagle == WSAEINPROGRESS)
+          Sleep(1);  /* use windows sleep */
+
+        WSASetLastError (0);
+        ret = ioctl (fd, FIONBIO, &zero);
+        wsagle = WSAGetLastError ();
+    }
   while (ret && (wsagle == WSAEINPROGRESS));
 
-	if(ret)
+  if(ret)
     {
       fprintf (stderr,
-               "ioctl() failed.  The socket could not be set as blocking.\n" );
+               _("ioctl() failed.  The socket could not be set as blocking.\n") );
       DEBUGP (("Winsock error: %d\n", WSAGetLastError ()));
       abort ();
     }
-	return;
+  return;
 }

src/mswindows.h

@@ -57,13 +57,6 @@ as that of the covered work.  */
 /* Declares getpid(). */
 #include <process.h>
 
-#ifndef S_ISDIR
-# define S_ISDIR(m) (((m) & (_S_IFMT)) == (_S_IFDIR))
-#endif
-#ifndef S_ISLNK
-# define S_ISLNK(a) 0
-#endif
-
 /* We have strcasecmp and strncasecmp, just under different names.  */
 #ifndef HAVE_STRCASECMP
 # define strcasecmp stricmp
@@ -90,18 +83,6 @@ typedef __int64 wgint;
 # define strtoll_type __int64
 #endif
 
-/* Windows has no symlink, therefore no lstat.  Without symlinks lstat
-   is equivalent to stat anyway.  */
-#define lstat stat
-
-/* Define LFS aliases for stat and fstat. */
-#ifdef stat_alias
-# define stat(f, b) stat_alias (f, b)
-#endif
-#ifdef fstat_alias
-# define fstat(f, b) fstat_alias (f, b)
-#endif
-
 #define PATH_SEPARATOR '\\'
 
 /* Additional declarations needed for IPv6: */

src/netrc.c

@@ -44,10 +44,16 @@ as that of the covered work.  */
 
 #define NETRC_FILE_NAME ".netrc"
 
-acc_t *netrc_list;
+static acc_t *netrc_list;
 
 static acc_t *parse_netrc (const char *);
 
+void
+netrc_cleanup(void)
+{
+  free_netrc (netrc_list);
+}
+
 /* Return the correct user and password, given the host, user (as
    given in the URL), and password (as given in the URL).  May return
    NULL.
@@ -155,57 +161,13 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 
 #ifdef STANDALONE
 
-#include <assert.h>
-
 /* Normally, these functions would be defined by your package.  */
 # define xmalloc malloc
-# define xfree free
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
 # define xstrdup strdup
 
 # define xrealloc realloc
 
-/* Read a line from FP.  The function reallocs the storage as needed
-   to accomodate for any length of the line.  Reallocs are done
-   storage exponentially, doubling the storage after each overflow to
-   minimize the number of calls to realloc() and fgets().  The newline
-   character at the end of line is retained.
-
-   After end-of-file is encountered without anything being read, NULL
-   is returned.  NULL is also returned on error.  To distinguish
-   between these two cases, use the stdio function ferror().  */
-
-char *
-read_whole_line (FILE *fp)
-{
-  int length = 0;
-  int bufsize = 81;
-  char *line = xmalloc (bufsize);
-
-  while (fgets (line + length, bufsize - length, fp))
-    {
-      length += strlen (line + length);
-      assert (length > 0);
-      if (line[length - 1] == '\n')
-        break;
-      /* fgets() guarantees to read the whole line, or to use up the
-         space we've given it.  We can double the buffer
-         unconditionally.  */
-      bufsize <<= 1;
-      line = xrealloc (line, bufsize);
-    }
-  if (length == 0 || ferror (fp))
-    {
-      xfree (line);
-      return NULL;
-    }
-  if (length + 1 < bufsize)
-    /* Relieve the memory from our exponential greediness.  We say
-       `length + 1' because the terminating \0 is not included in
-       LENGTH.  We don't need to zero-terminate the string ourselves,
-       though, because fgets() does that.  */
-    line = xrealloc (line, length + 1);
-  return line;
-}
 #endif /* STANDALONE */
 
 /* Maybe add NEWENTRY to the account information list, LIST.  NEWENTRY is
@@ -221,9 +183,9 @@ maybe_add_to_list (acc_t **newentry, acc_t **list)
   if (a && ! a->acc)
     {
       /* Free any allocated space.  */
-      xfree_null (a->host);
-      xfree_null (a->acc);
-      xfree_null (a->passwd);
+      xfree (a->host);
+      xfree (a->acc);
+      xfree (a->passwd);
     }
   else
     {
@@ -264,10 +226,11 @@ static acc_t *
 parse_netrc (const char *path)
 {
   FILE *fp;
-  char *line, *p, *tok;
+  char *line = NULL, *p, *tok;
   const char *premature_token;
   acc_t *current, *retval;
   int ln, qmark;
+  size_t bufsize = 0;
 
   /* The latest token we've seen in the file.  */
   enum
@@ -290,7 +253,7 @@ parse_netrc (const char *path)
   premature_token = NULL;
 
   /* While there are lines in the file...  */
-  while ((line = read_whole_line (fp)) != NULL)
+  while (getline (&line, &bufsize, fp) > 0)
     {
       ln ++;
 
@@ -423,10 +386,9 @@ parse_netrc (const char *path)
                          exec_name, path, ln, tok);
             }
         }
-
-      xfree (line);
     }
 
+  xfree (line);
   fclose (fp);
 
   /* Finalize the last machine entry we found.  */
@@ -462,9 +424,9 @@ free_netrc(acc_t *l)
   while (l)
     {
       t = l->next;
-      xfree_null (l->acc);
-      xfree_null (l->passwd);
-      xfree_null (l->host);
+      xfree (l->acc);
+      xfree (l->passwd);
+      xfree (l->host);
       xfree (l);
       l = t;
     }
@@ -484,18 +446,26 @@ main (int argc, char **argv)
   if (argc < 2 || argc > 3)
     {
       fprintf (stderr, _("Usage: %s NETRC [HOSTNAME]\n"), argv[0]);
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   program_name = argv[0];
   file = argv[1];
   target = argv[2];
 
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   if (stat (file, &sb))
     {
       fprintf (stderr, _("%s: cannot stat %s: %s\n"), argv[0], file,
                strerror (errno));
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   head = parse_netrc (file);
@@ -534,14 +504,14 @@ main (int argc, char **argv)
 
       /* Exit if we found the target.  */
       if (target)
-        exit (0);
+        exit (WGET_EXIT_SUCCESS);
       a = a->next;
     }
 
   /* Exit with failure if we had a target, success otherwise.  */
   if (target)
-    exit (1);
+    exit (WGET_EXIT_GENERIC_ERROR);
 
-  exit (0);
+  exit (WGET_EXIT_SUCCESS);
 }
 #endif /* STANDALONE */

src/netrc.h

@@ -33,14 +33,15 @@ as that of the covered work.  */
 
 typedef struct _acc_t
 {
-  char *host;			/* NULL if this is the default machine
-				   entry.  */
+  char *host;           /* NULL if this is the default machine
+                           entry.  */
   char *acc;
-  char *passwd;			/* NULL if there is no password.  */
+  char *passwd;         /* NULL if there is no password.  */
   struct _acc_t *next;
 } acc_t;
 
 void search_netrc (const char *, const char **, const char **, int);
 void free_netrc (acc_t *l);
+void netrc_cleanup(void);
 
 #endif /* NETRC_H */

src/openssl.c

@@ -1,6 +1,6 @@
 /* SSL support via OpenSSL library.
    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011 Free Software Foundation, Inc.
+   2009, 2010, 2011, 2012 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.
@@ -40,6 +40,11 @@ as that of the covered work.  */
 #include <openssl/x509v3.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
+#include <openssl/bio.h>
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+#include <openssl/conf.h>
+#include <openssl/engine.h>
+#endif
 
 #include "utils.h"
 #include "connect.h"
@@ -153,15 +158,27 @@ key_type_to_ssl_type (enum keyfile_type type)
     }
 }
 
+/* SSL has been initialized */
+static int ssl_true_initialized = 0;
+
 /* Create an SSL Context and set default paths etc.  Called the first
    time an HTTP download is attempted.
 
    Returns true on success, false otherwise.  */
 
 bool
-ssl_init ()
+ssl_init (void)
 {
-  SSL_METHOD *meth;
+  SSL_METHOD const *meth;
+  long ssl_options = 0;
+
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+  if (ssl_true_initialized == 0)
+    {
+      OPENSSL_config (NULL);
+      ssl_true_initialized = 1;
+    }
+#endif
 
   if (ssl_ctx)
     /* The SSL has already been initialized. */
@@ -176,6 +193,12 @@ ssl_init ()
       goto error;
     }
 
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+  OPENSSL_load_builtin_modules();
+  ENGINE_load_builtin_engines();
+  CONF_modules_load_file(NULL, NULL,
+      CONF_MFLAGS_DEFAULT_SECTION|CONF_MFLAGS_IGNORE_MISSING_FILE);
+#endif
   SSL_library_init ();
   SSL_load_error_strings ();
   SSLeay_add_all_algorithms ();
@@ -183,31 +206,81 @@ ssl_init ()
 
   switch (opt.secure_protocol)
     {
-    case secure_protocol_auto:
-      meth = SSLv23_client_method ();
-      break;
 #ifndef OPENSSL_NO_SSL2
     case secure_protocol_sslv2:
       meth = SSLv2_client_method ();
       break;
 #endif
+
+#ifndef OPENSSL_NO_SSL3
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
+#endif
+
+    case secure_protocol_auto:
+    case secure_protocol_pfs:
+      meth = SSLv23_client_method ();
+      ssl_options |= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
+      break;
     case secure_protocol_tlsv1:
       meth = TLSv1_client_method ();
       break;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10001000
+    case secure_protocol_tlsv1_1:
+      meth = TLSv1_1_client_method ();
+      break;
+
+    case secure_protocol_tlsv1_2:
+      meth = TLSv1_2_client_method ();
+      break;
+#else
+    case secure_protocol_tlsv1_1:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.1\n"));
+      goto error;
+
+    case secure_protocol_tlsv1_2:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.2\n"));
+      goto error;
+#endif
+
     default:
+      logprintf (LOG_NOTQUIET, _("OpenSSL: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
 
-  ssl_ctx = SSL_CTX_new (meth);
+  /* The type cast below accommodates older OpenSSL versions (0.9.8)
+     where SSL_CTX_new() is declared without a "const" argument. */
+  ssl_ctx = SSL_CTX_new ((SSL_METHOD *)meth);
   if (!ssl_ctx)
     goto error;
 
+  if (ssl_options)
+    SSL_CTX_set_options (ssl_ctx, ssl_options);
+
+  /* OpenSSL ciphers: https://www.openssl.org/docs/apps/ciphers.html
+   * Since we want a good protection, we also use HIGH (that excludes MD4 ciphers and some more)
+   */
+  if (opt.secure_protocol == secure_protocol_pfs)
+    SSL_CTX_set_cipher_list (ssl_ctx, "HIGH:MEDIUM:!RC4:!SRP:!PSK:!RSA:!aNULL@STRENGTH");
+
   SSL_CTX_set_default_verify_paths (ssl_ctx);
   SSL_CTX_load_verify_locations (ssl_ctx, opt.ca_cert, opt.ca_directory);
 
+  if (opt.crl_file)
+    {
+      X509_STORE *store = SSL_CTX_get_cert_store (ssl_ctx);
+      X509_LOOKUP *lookup;
+
+      if (!(lookup = X509_STORE_add_lookup (store, X509_LOOKUP_file ()))
+          || (!X509_load_crl_file (lookup, opt.crl_file, X509_FILETYPE_PEM)))
+        goto error;
+
+      X509_STORE_set_flags (store, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL);
+    }
+
   /* SSL_VERIFY_NONE instructs OpenSSL not to abort SSL_connect if the
      certificate is invalid.  We verify the certificate separately in
      ssl_check_certificate, which provides much better diagnostics
@@ -249,28 +322,54 @@ ssl_init ()
   return false;
 }
 
-struct openssl_transport_context {
+struct openssl_transport_context
+{
   SSL *conn;                    /* SSL connection handle */
   char *last_error;             /* last error printed with openssl_errstr */
 };
 
-static int
-openssl_read (int fd, char *buf, int bufsize, void *arg)
+struct openssl_read_args
 {
-  int ret;
-  struct openssl_transport_context *ctx = arg;
+  int fd;
+  struct openssl_transport_context *ctx;
+  char *buf;
+  int bufsize;
+  int retval;
+};
+
+static void openssl_read_callback(void *arg)
+{
+  struct openssl_read_args *args = (struct openssl_read_args *) arg;
+  struct openssl_transport_context *ctx = args->ctx;
   SSL *conn = ctx->conn;
+  char *buf = args->buf;
+  int bufsize = args->bufsize;
+  int ret;
+
   do
     ret = SSL_read (conn, buf, bufsize);
-  while (ret == -1
-         && SSL_get_error (conn, ret) == SSL_ERROR_SYSCALL
+  while (ret == -1 && SSL_get_error (conn, ret) == SSL_ERROR_SYSCALL
          && errno == EINTR);
-
-  return ret;
+  args->retval = ret;
 }
 
 static int
-openssl_write (int fd, char *buf, int bufsize, void *arg)
+openssl_read (int fd, char *buf, int bufsize, void *arg)
+{
+  struct openssl_read_args args;
+  args.fd = fd;
+  args.buf = buf;
+  args.bufsize = bufsize;
+  args.ctx = (struct openssl_transport_context*) arg;
+
+  if (run_with_timeout(opt.read_timeout, openssl_read_callback, &args)) {
+    return -1;
+  }
+  return args.retval;
+}
+
+static int
+openssl_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg)
 {
   int ret = 0;
   struct openssl_transport_context *ctx = arg;
@@ -312,7 +411,7 @@ openssl_peek (int fd, char *buf, int bufsize, void *arg)
 }
 
 static const char *
-openssl_errstr (int fd, void *arg)
+openssl_errstr (int fd _GL_UNUSED, void *arg)
 {
   struct openssl_transport_context *ctx = arg;
   unsigned long errcode;
@@ -324,7 +423,7 @@ openssl_errstr (int fd, void *arg)
     return NULL;
 
   /* Get rid of previous contents of ctx->last_error, if any.  */
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);
 
   /* Iterate over OpenSSL's error stack and accumulate errors in the
      last_error buffer, separated by "; ".  This is better than using
@@ -368,7 +467,7 @@ openssl_close (int fd, void *arg)
 
   SSL_shutdown (conn);
   SSL_free (conn);
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);
   xfree (ctx);
 
   close (fd);
@@ -384,6 +483,19 @@ static struct transport_implementation openssl_transport = {
   openssl_peek, openssl_errstr, openssl_close
 };
 
+struct scwt_context
+{
+  SSL *ssl;
+  int result;
+};
+
+static void
+ssl_connect_with_timeout_callback(void *arg)
+{
+  struct scwt_context *ctx = (struct scwt_context *)arg;
+  ctx->result = SSL_connect(ctx->ssl);
+}
+
 /* Perform the SSL handshake on file descriptor FD, which is assumed
    to be connected to an SSL server.  The SSL handle provided by
    OpenSSL is registered with the file descriptor FD using
@@ -393,9 +505,10 @@ static struct transport_implementation openssl_transport = {
    Returns true on success, false on failure.  */
 
 bool
-ssl_connect_wget (int fd)
+ssl_connect_wget (int fd, const char *hostname)
 {
   SSL *conn;
+  struct scwt_context scwt_ctx;
   struct openssl_transport_context *ctx;
 
   DEBUGP (("Initiating SSL handshake.\n"));
@@ -404,13 +517,33 @@ ssl_connect_wget (int fd)
   conn = SSL_new (ssl_ctx);
   if (!conn)
     goto error;
+#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
+  /* If the SSL library was build with support for ServerNameIndication
+     then use it whenever we have a hostname.  If not, don't, ever. */
+  if (! is_valid_ip_address (hostname))
+    {
+      if (! SSL_set_tlsext_host_name (conn, hostname))
+        {
+          DEBUGP (("Failed to set TLS server-name indication."));
+          goto error;
+        }
+    }
+#endif
+
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
 #endif
   if (!SSL_set_fd (conn, FD_TO_SOCKET (fd)))
     goto error;
   SSL_set_connect_state (conn);
-  if (SSL_connect (conn) <= 0 || conn->state != SSL_ST_OK)
+
+  scwt_ctx.ssl = conn;
+  if (run_with_timeout(opt.read_timeout, ssl_connect_with_timeout_callback,
+                       &scwt_ctx)) {
+    DEBUGP (("SSL handshake timed out.\n"));
+    goto timeout;
+  }
+  if (scwt_ctx.result <= 0 || conn->state != SSL_ST_OK)
     goto error;
 
   ctx = xnew0 (struct openssl_transport_context);
@@ -426,6 +559,7 @@ ssl_connect_wget (int fd)
  error:
   DEBUGP (("SSL handshake failed.\n"));
   print_errors ();
+ timeout:
   if (conn)
     SSL_free (conn);
   return false;
@@ -474,6 +608,27 @@ pattern_match (const char *pattern, const char *string)
   return *n == '\0';
 }
 
+static char *_get_rfc2253_formatted (X509_NAME *name)
+{
+  int len;
+  char *out = NULL;
+  BIO* b;
+
+  if ((b = BIO_new (BIO_s_mem ())))
+    {
+      if (X509_NAME_print_ex (b, name, 0, XN_FLAG_RFC2253) >= 0
+          && (len = BIO_number_written (b)) > 0)
+        {
+          out = xmalloc (len + 1);
+          BIO_read (b, out, len);
+          out[len] = 0;
+        }
+      BIO_free (b);
+    }
+
+  return out ? out : xstrdup("");
+}
+
 /* Verify the validity of the certificate presented by the server.
    Also check that the "common name" of the server, as presented by
    its certificate, corresponds to HOST.  (HOST typically comes from
@@ -517,23 +672,25 @@ ssl_check_certificate (int fd, const char *host)
 
   IF_DEBUG
     {
-      char *subject = X509_NAME_oneline (X509_get_subject_name (cert), 0, 0);
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *subject = _get_rfc2253_formatted (X509_get_subject_name (cert));
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
       DEBUGP (("certificate:\n  subject: %s\n  issuer:  %s\n",
                quotearg_n_style (0, escape_quoting_style, subject),
                quotearg_n_style (1, escape_quoting_style, issuer)));
-      OPENSSL_free (subject);
-      OPENSSL_free (issuer);
+      xfree (subject);
+      xfree (issuer);
     }
 
   vresult = SSL_get_verify_result (conn);
   if (vresult != X509_V_OK)
     {
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
       logprintf (LOG_NOTQUIET,
                  _("%s: cannot verify %s's certificate, issued by %s:\n"),
                  severity, quotearg_n_style (0, escape_quoting_style, host),
                  quote_n (1, issuer));
+      xfree(issuer);
+
       /* Try to print more user-friendly (and translated) messages for
          the frequent verification errors.  */
       switch (vresult)
@@ -619,7 +776,7 @@ ssl_check_certificate (int fd, const char *host)
                       /* Compare and check for NULL attack in ASN1_STRING */
                       if (pattern_match ((char *)name_in_utf8, host) &&
                             (strlen ((char *)name_in_utf8) ==
-                                ASN1_STRING_length (name->d.dNSName)))
+                                (size_t) ASN1_STRING_length (name->d.dNSName)))
                         {
                           OPENSSL_free (name_in_utf8);
                           break;
@@ -629,7 +786,7 @@ ssl_check_certificate (int fd, const char *host)
                 }
             }
         }
-      sk_GENERAL_NAME_free (subjectAltNames);
+      sk_GENERAL_NAME_pop_free(subjectAltNames, GENERAL_NAME_free);
       if (host_in_octet_string)
         ASN1_OCTET_STRING_free(host_in_octet_string);
 
@@ -642,7 +799,7 @@ ssl_check_certificate (int fd, const char *host)
           success = false;
         }
     }
-  
+
   if (alt_name_checked == false)
     {
       /* Test commomName */
@@ -683,7 +840,7 @@ ssl_check_certificate (int fd, const char *host)
 
           xentry = X509_NAME_get_entry(xname,i);
           sdata = X509_NAME_ENTRY_get_data(xentry);
-          if (strlen (common_name) != ASN1_STRING_length (sdata))
+          if (strlen (common_name) != (size_t) ASN1_STRING_length (sdata))
             {
               logprintf (LOG_NOTQUIET, _("\
     %s: certificate common name is invalid (contains a NUL character).\n\

src/options.h

@@ -31,213 +31,245 @@ as that of the covered work.  */
 
 struct options
 {
-  int verbose;			/* Are we verbose?  (First set to -1,
+  int verbose;                  /* Are we verbose?  (First set to -1,
                                    hence not boolean.) */
-  bool quiet;			/* Are we quiet? */
-  int ntry;			/* Number of tries per URL */
-  bool retry_connrefused;	/* Treat CONNREFUSED as non-fatal. */
-  bool background;		/* Whether we should work in background. */
-  bool ignore_length;		/* Do we heed content-length at all?  */
-  bool recursive;		/* Are we recursive? */
-  bool spanhost;		/* Do we span across hosts in
-				   recursion? */
+  bool quiet;                   /* Are we quiet? */
+  int ntry;                     /* Number of tries per URL */
+  bool retry_connrefused;       /* Treat CONNREFUSED as non-fatal. */
+  bool background;              /* Whether we should work in background. */
+  bool ignore_length;           /* Do we heed content-length at all?  */
+  bool recursive;               /* Are we recursive? */
+  bool spanhost;                /* Do we span across hosts in
+                                   recursion? */
   int  max_redirect;            /* Maximum number of times we'll allow
                                    a page to redirect. */
-  bool relative_only;		/* Follow only relative links. */
-  bool no_parent;		/* Restrict access to the parent
-				   directory.  */
-  int reclevel;			/* Maximum level of recursion */
-  bool dirstruct;		/* Do we build the directory structure
-				  as we go along? */
-  bool no_dirstruct;		/* Do we hate dirstruct? */
-  int cut_dirs;			/* Number of directory components to cut. */
-  bool add_hostdir;		/* Do we add hostname directory? */
-  bool protocol_directories;	/* Whether to prepend "http"/"ftp" to dirs. */
-  bool noclobber;		/* Disables clobbering of existing
-				   data. */
-  bool unlink;			/* remove file before clobbering */
-  char *dir_prefix;		/* The top of directory tree */
-  char *lfilename;		/* Log filename */
-  char *input_filename;		/* Input filename */
-  char *choose_config;		/* Specified config file */
-  bool force_html;		/* Is the input file an HTML file? */
+  bool relative_only;           /* Follow only relative links. */
+  bool no_parent;               /* Restrict access to the parent
+                                   directory.  */
+  int reclevel;                 /* Maximum level of recursion */
+  bool dirstruct;               /* Do we build the directory structure
+                                   as we go along? */
+  bool no_dirstruct;            /* Do we hate dirstruct? */
+  int cut_dirs;                 /* Number of directory components to cut. */
+  bool add_hostdir;             /* Do we add hostname directory? */
+  bool protocol_directories;    /* Whether to prepend "http"/"ftp" to dirs. */
+  bool noclobber;               /* Disables clobbering of existing data. */
+  bool unlink;                  /* remove file before clobbering */
+  char *dir_prefix;             /* The top of directory tree */
+  char *lfilename;              /* Log filename */
+  char *input_filename;         /* Input filename */
+  char *choose_config;          /* Specified config file */
+  bool noconfig;                /* Ignore all config files? */
+  bool force_html;              /* Is the input file an HTML file? */
 
   char *default_page;           /* Alternative default page (index file) */
 
-  bool spider;			/* Is Wget in spider mode? */
+  bool spider;                  /* Is Wget in spider mode? */
 
-  char **accepts;		/* List of patterns to accept. */
-  char **rejects;		/* List of patterns to reject. */
-  char **excludes;		/* List of excluded FTP directories. */
-  char **includes;		/* List of FTP directories to
-				   follow. */
-  bool ignore_case;		/* Whether to ignore case when
-				   matching dirs and files */
+  char **accepts;               /* List of patterns to accept. */
+  char **rejects;               /* List of patterns to reject. */
+  const char **excludes;        /* List of excluded FTP directories. */
+  const char **includes;        /* List of FTP directories to
+                                   follow. */
+  bool ignore_case;             /* Whether to ignore case when
+                                   matching dirs and files */
 
-  char **domains;		/* See host.c */
+  char *acceptregex_s;          /* Patterns to accept (a regex string). */
+  char *rejectregex_s;          /* Patterns to reject (a regex string). */
+  void *acceptregex;            /* Patterns to accept (a regex struct). */
+  void *rejectregex;            /* Patterns to reject (a regex struct). */
+  enum {
+#ifdef HAVE_LIBPCRE
+    regex_type_pcre,
+#endif
+    regex_type_posix
+  } regex_type;                 /* The regex library. */
+  void *(*regex_compile_fun)(const char *);             /* Function to compile a regex. */
+  bool (*regex_match_fun)(const void *, const char *);  /* Function to match a string to a regex. */
+
+  char **domains;               /* See host.c */
   char **exclude_domains;
-  bool dns_cache;		/* whether we cache DNS lookups. */
+  bool dns_cache;               /* whether we cache DNS lookups. */
 
   char **follow_tags;           /* List of HTML tags to recursively follow. */
   char **ignore_tags;           /* List of HTML tags to ignore if recursing. */
 
-  bool follow_ftp;		/* Are FTP URL-s followed in recursive
-				   retrieving? */
-  bool retr_symlinks;		/* Whether we retrieve symlinks in
-				   FTP. */
-  char *output_document;	/* The output file to which the
-				   documents will be printed.  */
+  bool follow_ftp;              /* Are FTP URL-s followed in recursive
+                                   retrieving? */
+  bool retr_symlinks;           /* Whether we retrieve symlinks in
+                                   FTP. */
+  char *output_document;        /* The output file to which the
+                                   documents will be printed.  */
+  char *warc_filename;          /* WARC output filename */
+  char *warc_tempdir;           /* WARC temp dir */
+  char *warc_cdx_dedup_filename;/* CDX file to be used for deduplication. */
+  wgint warc_maxsize;           /* WARC max archive size */
+  bool warc_compression_enabled;/* For GZIP compression. */
+  bool warc_digests_enabled;    /* For SHA1 digests. */
+  bool warc_cdx_enabled;        /* Create CDX files? */
+  bool warc_keep_log;           /* Store the log file in a WARC record. */
+  char **warc_user_headers;     /* User-defined WARC header(s). */
 
-  char *user;			/* Generic username */
-  char *passwd;			/* Generic password */
+  char *user;                   /* Generic username */
+  char *passwd;                 /* Generic password */
   bool ask_passwd;              /* Ask for password? */
 
-  bool always_rest;		/* Always use REST. */
-  char *ftp_user;		/* FTP username */
-  char *ftp_passwd;		/* FTP password */
-  bool netrc;			/* Whether to read .netrc. */
-  bool ftp_glob;		/* FTP globbing */
-  bool ftp_pasv;			/* Passive FTP. */
+  bool always_rest;             /* Always use REST. */
+  wgint start_pos;              /* Start position of a download. */
+  char *ftp_user;               /* FTP username */
+  char *ftp_passwd;             /* FTP password */
+  bool netrc;                   /* Whether to read .netrc. */
+  bool ftp_glob;                /* FTP globbing */
+  bool ftp_pasv;                /* Passive FTP. */
 
-  char *http_user;		/* HTTP username. */
-  char *http_passwd;		/* HTTP password. */
-  char **user_headers;		/* User-defined header(s). */
-  bool http_keep_alive;		/* whether we use keep-alive */
+  char *http_user;              /* HTTP username. */
+  char *http_passwd;            /* HTTP password. */
+  char **user_headers;          /* User-defined header(s). */
+  bool http_keep_alive;         /* whether we use keep-alive */
 
-  bool use_proxy;		/* Do we use proxy? */
-  bool allow_cache;		/* Do we allow server-side caching? */
+  bool use_proxy;               /* Do we use proxy? */
+  bool allow_cache;             /* Do we allow server-side caching? */
   char *http_proxy, *ftp_proxy, *https_proxy;
   char **no_proxy;
   char *base_href;
-  char *progress_type;		/* progress indicator type. */
+  char *progress_type;          /* progress indicator type. */
+  bool show_progress;           /* Show only the progress bar */
+  bool noscroll;                /* Don't scroll the filename in the progressbar */
   char *proxy_user; /*oli*/
   char *proxy_passwd;
 
-  double read_timeout;		/* The read/write timeout. */
-  double dns_timeout;		/* The DNS timeout. */
-  double connect_timeout;	/* The connect timeout. */
+  double read_timeout;          /* The read/write timeout. */
+  double dns_timeout;           /* The DNS timeout. */
+  double connect_timeout;       /* The connect timeout. */
 
-  bool random_wait;		/* vary from 0 .. wait secs by random()? */
-  double wait;			/* The wait period between retrievals. */
-  double waitretry;		/* The wait period between retries. - HEH */
-  bool use_robots;		/* Do we heed robots.txt? */
+  bool random_wait;             /* vary from 0 .. wait secs by random()? */
+  double wait;                  /* The wait period between retrievals. */
+  double waitretry;             /* The wait period between retries. - HEH */
+  bool use_robots;              /* Do we heed robots.txt? */
 
-  wgint limit_rate;		/* Limit the download rate to this
-				   many bps. */
-  SUM_SIZE_INT quota;		/* Maximum file size to download and
-				   store. */
+  wgint limit_rate;             /* Limit the download rate to this
+                                   many bps. */
+  SUM_SIZE_INT quota;           /* Maximum file size to download and
+                                   store. */
 
-  bool server_response;		/* Do we print server response? */
-  bool save_headers;		/* Do we save headers together with
-				   file? */
+  bool server_response;         /* Do we print server response? */
+  bool save_headers;            /* Do we save headers together with
+                                   file? */
+  bool content_on_error;        /* Do we output the content when the HTTP
+                                   status code indicates a server error */
 
-#ifdef ENABLE_DEBUG
-  bool debug;			/* Debugging on/off */
-#endif
+  bool debug;                   /* Debugging on/off */
 
 #ifdef USE_WATT32
   bool wdebug;                  /* Watt-32 tcp/ip debugging on/off */
 #endif
 
-  bool timestamping;		/* Whether to use time-stamping. */
+  bool timestamping;            /* Whether to use time-stamping. */
 
-  bool backup_converted;	/* Do we save pre-converted files as *.orig? */
-  bool backups;			/* Are numeric backups made? */
+  bool backup_converted;        /* Do we save pre-converted files as *.orig? */
+  int backups;                  /* Are numeric backups made? */
 
-  char *useragent;		/* User-Agent string, which can be set
-				   to something other than Wget. */
-  char *referer;		/* Naughty Referer, which can be
-				   set to something other than
-				   NULL. */
-  bool convert_links;		/* Will the links be converted
-				   locally? */
-  bool remove_listing;		/* Do we remove .listing files
-				   generated by FTP? */
-  bool htmlify;			/* Do we HTML-ify the OS-dependent
-				   listings? */
+  char *useragent;              /* User-Agent string, which can be set
+                                   to something other than Wget. */
+  char *referer;                /* Naughty Referer, which can be
+                                   set to something other than
+                                   NULL. */
+  bool convert_links;           /* Will the links be converted
+                                   locally? */
+  bool remove_listing;          /* Do we remove .listing files
+                                   generated by FTP? */
+  bool htmlify;                 /* Do we HTML-ify the OS-dependent
+                                   listings? */
 
   char *dot_style;
-  wgint dot_bytes;		/* How many bytes in a printing
-				   dot. */
-  int dots_in_line;		/* How many dots in one line. */
-  int dot_spacing;		/* How many dots between spacings. */
+  wgint dot_bytes;              /* How many bytes in a printing
+                                   dot. */
+  int dots_in_line;             /* How many dots in one line. */
+  int dot_spacing;              /* How many dots between spacings. */
 
-  bool delete_after;		/* Whether the files will be deleted
-				   after download. */
+  bool delete_after;            /* Whether the files will be deleted
+                                   after download. */
 
-  bool adjust_extension;		/* Use ".html" extension on all text/html? */
+  bool adjust_extension;        /* Use ".html" extension on all text/html? */
 
-  bool page_requisites;		/* Whether we need to download all files
-				   necessary to display a page properly. */
-  char *bind_address;		/* What local IP address to bind to. */
+  bool page_requisites;         /* Whether we need to download all files
+                                   necessary to display a page properly. */
+  char *bind_address;           /* What local IP address to bind to. */
 
 #ifdef HAVE_SSL
   enum {
     secure_protocol_auto,
     secure_protocol_sslv2,
     secure_protocol_sslv3,
-    secure_protocol_tlsv1
-  } secure_protocol;		/* type of secure protocol to use. */
-  bool check_cert;		/* whether to validate the server's cert */
-  char *cert_file;		/* external client certificate to use. */
-  char *private_key;		/* private key file (if not internal). */
+    secure_protocol_tlsv1,
+    secure_protocol_tlsv1_1,
+    secure_protocol_tlsv1_2,
+    secure_protocol_pfs
+  } secure_protocol;            /* type of secure protocol to use. */
+  bool check_cert;              /* whether to validate the server's cert */
+  char *cert_file;              /* external client certificate to use. */
+  char *private_key;            /* private key file (if not internal). */
   enum keyfile_type {
     keyfile_pem,
     keyfile_asn1
-  } cert_type;			/* type of client certificate file */
+  } cert_type;                  /* type of client certificate file */
   enum keyfile_type
-    private_key_type;		/* type of private key file */
+    private_key_type;           /* type of private key file */
 
-  char *ca_directory;		/* CA directory (hash files) */
-  char *ca_cert;		/* CA certificate file to use */
+  char *ca_directory;           /* CA directory (hash files) */
+  char *ca_cert;                /* CA certificate file to use */
+  char *crl_file;               /* file with CRLs */
 
-
-  char *random_file;		/* file with random data to seed the PRNG */
-  char *egd_file;		/* file name of the egd daemon socket */
+  char *random_file;            /* file with random data to seed the PRNG */
+  char *egd_file;               /* file name of the egd daemon socket */
+  bool https_only;              /* whether to follow HTTPS only */
 #endif /* HAVE_SSL */
 
-  bool cookies;			/* whether cookies are used. */
-  char *cookies_input;		/* file we're loading the cookies from. */
-  char *cookies_output;		/* file we're saving the cookies to. */
-  bool keep_session_cookies;	/* whether session cookies should be
-				   saved and loaded. */
+  bool cookies;                 /* whether cookies are used. */
+  char *cookies_input;          /* file we're loading the cookies from. */
+  char *cookies_output;         /* file we're saving the cookies to. */
+  bool keep_session_cookies;    /* whether session cookies should be
+                                   saved and loaded. */
 
-  char *post_data;		/* POST query string */
-  char *post_file_name;		/* File to post */
+  char *post_data;              /* POST query string */
+  char *post_file_name;         /* File to post */
+  char *method;                 /* HTTP Method to use in Header */
+  char *body_data;              /* HTTP Method Data String */
+  char *body_file;              /* HTTP Method File */
 
   enum {
     restrict_unix,
     restrict_windows
-  } restrict_files_os;		/* file name restriction ruleset. */
-  bool restrict_files_ctrl;	/* non-zero if control chars in URLs
-				   are restricted from appearing in
-				   generated file names. */
+  } restrict_files_os;          /* file name restriction ruleset. */
+  bool restrict_files_ctrl;     /* non-zero if control chars in URLs
+                                   are restricted from appearing in
+                                   generated file names. */
   bool restrict_files_nonascii; /* non-zero if bytes with values greater
                                    than 127 are restricted. */
   enum {
     restrict_no_case_restriction,
     restrict_lowercase,
     restrict_uppercase
-  } restrict_files_case;	/* file name case restriction. */
+  } restrict_files_case;        /* file name case restriction. */
 
-  bool strict_comments;		/* whether strict SGML comments are
-				   enforced.  */
+  bool strict_comments;         /* whether strict SGML comments are
+                                   enforced.  */
 
   bool preserve_perm;           /* whether remote permissions are used
-				  or that what is set by umask. */
+                                   or that what is set by umask. */
 
 #ifdef ENABLE_IPV6
-  bool ipv4_only;		/* IPv4 connections have been requested. */
-  bool ipv6_only;		/* IPv4 connections have been requested. */
+  bool ipv4_only;               /* IPv4 connections have been requested. */
+  bool ipv6_only;               /* IPv4 connections have been requested. */
 #endif
   enum {
     prefer_ipv4,
     prefer_ipv6,
     prefer_none
-  } prefer_family;		/* preferred address family when more
-				   than one type is available */
+  } prefer_family;              /* preferred address family when more
+                                   than one type is available */
 
-  bool content_disposition;	/* Honor HTTP Content-Disposition header. */
+  bool content_disposition;     /* Honor HTTP Content-Disposition header. */
   bool auth_without_challenge;  /* Issue Basic authentication creds without
                                    waiting for a challenge. */
 
@@ -250,11 +282,12 @@ struct options
   int ftp_stmlf;                /* Force Stream_LF format for binary FTP. */
 #endif /* def __VMS */
 
-  bool useservertimestamps;  	/* Update downloaded files' timestamps to
-				   match those on server? */
+  bool useservertimestamps;     /* Update downloaded files' timestamps to
+                                   match those on server? */
 
-  bool show_all_dns_entries; /* Show all the DNS entries when resolving a
-                                name. */
+  bool show_all_dns_entries;    /* Show all the DNS entries when resolving a
+                                   name. */
+  bool report_bps;              /*Output bandwidth in bits format*/
 };
 
 extern struct options opt;

src/progress.c

@@ -36,38 +36,41 @@ as that of the covered work.  */
 #include <assert.h>
 #include <unistd.h>
 #include <signal.h>
-#ifdef HAVE_WCHAR_H
-# include <wchar.h>
-#endif
+#include <wchar.h>
+#include <mbiter.h>
 
 #include "progress.h"
 #include "utils.h"
 #include "retr.h"
+#include "c-strcase.h"
 
 struct progress_implementation {
   const char *name;
   bool interactive;
-  void *(*create) (wgint, wgint);
+  void *(*create) (const char *, wgint, wgint);
   void (*update) (void *, wgint, double);
+  void (*draw) (void *);
   void (*finish) (void *, double);
-  void (*set_params) (const char *);
+  void (*set_params) (char *);
 };
 
 /* Necessary forward declarations. */
 
-static void *dot_create (wgint, wgint);
+static void *dot_create (const char *, wgint, wgint);
 static void dot_update (void *, wgint, double);
 static void dot_finish (void *, double);
-static void dot_set_params (const char *);
+static void dot_draw (void *);
+static void dot_set_params (char *);
 
-static void *bar_create (wgint, wgint);
+static void *bar_create (const char *, wgint, wgint);
 static void bar_update (void *, wgint, double);
+static void bar_draw (void *);
 static void bar_finish (void *, double);
-static void bar_set_params (const char *);
+static void bar_set_params (char *);
 
 static struct progress_implementation implementations[] = {
-  { "dot", 0, dot_create, dot_update, dot_finish, dot_set_params },
-  { "bar", 1, bar_create, bar_update, bar_finish, bar_set_params }
+  { "dot", 0, dot_create, dot_update, dot_draw, dot_finish, dot_set_params },
+  { "bar", 1, bar_create, bar_update, bar_draw, bar_finish, bar_set_params }
 };
 static struct progress_implementation *current_impl;
 static int current_impl_locked;
@@ -109,7 +112,7 @@ set_progress_implementation (const char *name)
 {
   size_t i, namelen;
   struct progress_implementation *pi = implementations;
-  const char *colon;
+  char *colon;
 
   if (!name)
     name = DEFAULT_PROGRESS_IMPLEMENTATION;
@@ -151,7 +154,7 @@ progress_schedule_redirect (void)
    advance.  */
 
 void *
-progress_create (wgint initial, wgint total)
+progress_create (const char *f_download, wgint initial, wgint total)
 {
   /* Check if the log status has changed under our feet. */
   if (output_redirected)
@@ -161,7 +164,7 @@ progress_create (wgint initial, wgint total)
       output_redirected = 0;
     }
 
-  return current_impl->create (initial, total);
+  return current_impl->create (f_download, initial, total);
 }
 
 /* Return true if the progress gauge is "interactive", i.e. if it can
@@ -170,7 +173,7 @@ progress_create (wgint initial, wgint total)
    and current update.  */
 
 bool
-progress_interactive_p (void *progress)
+progress_interactive_p (void *progress _GL_UNUSED)
 {
   return current_impl->interactive;
 }
@@ -182,6 +185,7 @@ void
 progress_update (void *progress, wgint howmuch, double dltime)
 {
   current_impl->update (progress, howmuch, dltime);
+  current_impl->draw (progress);
 }
 
 /* Tell the progress gauge to clean up.  Calling this will free the
@@ -192,7 +196,7 @@ progress_finish (void *progress, double dltime)
 {
   current_impl->finish (progress, dltime);
 }
-
+
 /* Dot-printing. */
 
 struct dot_progress {
@@ -204,6 +208,7 @@ struct dot_progress {
   int accumulated;              /* number of bytes accumulated after
                                    the last printed dot */
 
+  double dltime;                /* download time so far */
   int rows;                     /* number of rows printed so far */
   int dots;                     /* number of dots printed in this row */
 
@@ -213,7 +218,7 @@ struct dot_progress {
 /* Dot-progress backend for progress_create. */
 
 static void *
-dot_create (wgint initial, wgint total)
+dot_create (const char *f_download _GL_UNUSED, wgint initial, wgint total)
 {
   struct dot_progress *dp = xnew0 (struct dot_progress);
   dp->initial_length = initial;
@@ -237,18 +242,18 @@ dot_create (wgint initial, wgint total)
           /* Align the [ skipping ... ] line with the dots.  To do
              that, insert the number of spaces equal to the number of
              digits in the skipped amount in K.  */
-          logprintf (LOG_VERBOSE, _("\n%*s[ skipping %sK ]"),
+          logprintf (LOG_PROGRESS, _("\n%*s[ skipping %sK ]"),
                      2 + skipped_k_len, "",
                      number_to_static_string (skipped_k));
         }
 
-      logprintf (LOG_VERBOSE, "\n%6sK",
+      logprintf (LOG_PROGRESS, "\n%6sK",
                  number_to_static_string (skipped / 1024));
       for (; remainder >= dot_bytes; remainder -= dot_bytes)
         {
           if (dp->dots % opt.dot_spacing == 0)
-            logputs (LOG_VERBOSE, " ");
-          logputs (LOG_VERBOSE, ",");
+            logputs (LOG_PROGRESS, " ");
+          logputs (LOG_PROGRESS, ",");
           ++dp->dots;
         }
       assert (dp->dots < opt.dots_in_line);
@@ -289,7 +294,7 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
          been retrieved.  12.8% will round to 12% because the 13% mark
          has not yet been reached.  100% is only shown when done.  */
       int percentage = 100.0 * bytes_displayed / dp->total_length;
-      logprintf (LOG_VERBOSE, "%3d%%", percentage);
+      logprintf (LOG_PROGRESS, "%3d%%", percentage);
     }
 
   {
@@ -306,7 +311,7 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
     if (dp->rows == dp->initial_length / ROW_BYTES)
       bytes_this_row -= dp->initial_length % ROW_BYTES;
     rate = calc_rate (bytes_this_row, dltime - dp->last_timer_value, &units);
-    logprintf (LOG_VERBOSE, " %4.*f%c",
+    logprintf (LOG_PROGRESS, " %4.*f%c",
                rate >= 99.95 ? 0 : rate >= 9.995 ? 1 : 2,
                rate, names[units]);
     dp->last_timer_value = dltime;
@@ -323,7 +328,7 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
           wgint bytes_sofar = bytes_displayed - dp->initial_length;
           double eta = dltime * bytes_remaining / bytes_sofar;
           if (eta < INT_MAX - 1)
-            logprintf (LOG_VERBOSE, " %s",
+            logprintf (LOG_PROGRESS, " %s",
                        eta_to_human_short ((int) (eta + 0.5), true));
         }
     }
@@ -331,10 +336,10 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
     {
       /* When done, print the total download time */
       if (dltime >= 10)
-        logprintf (LOG_VERBOSE, "=%s",
+        logprintf (LOG_PROGRESS, "=%s",
                    eta_to_human_short ((int) (dltime + 0.5), true));
       else
-        logprintf (LOG_VERBOSE, "=%ss", print_decimal (dltime));
+        logprintf (LOG_PROGRESS, "=%ss", print_decimal (dltime));
     }
 }
 
@@ -342,6 +347,14 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
 
 static void
 dot_update (void *progress, wgint howmuch, double dltime)
+{
+  struct dot_progress *dp = progress;
+  dp->accumulated += howmuch;
+  dp->dltime = dltime;
+}
+
+static void
+dot_draw (void *progress)
 {
   struct dot_progress *dp = progress;
   int dot_bytes = opt.dot_bytes;
@@ -349,16 +362,15 @@ dot_update (void *progress, wgint howmuch, double dltime)
 
   log_set_flush (false);
 
-  dp->accumulated += howmuch;
   for (; dp->accumulated >= dot_bytes; dp->accumulated -= dot_bytes)
     {
       if (dp->dots == 0)
-        logprintf (LOG_VERBOSE, "\n%6sK",
+        logprintf (LOG_PROGRESS, "\n%6sK",
                    number_to_static_string (dp->rows * ROW_BYTES / 1024));
 
       if (dp->dots % opt.dot_spacing == 0)
-        logputs (LOG_VERBOSE, " ");
-      logputs (LOG_VERBOSE, ".");
+        logputs (LOG_PROGRESS, " ");
+      logputs (LOG_PROGRESS, ".");
 
       ++dp->dots;
       if (dp->dots >= opt.dots_in_line)
@@ -366,7 +378,7 @@ dot_update (void *progress, wgint howmuch, double dltime)
           ++dp->rows;
           dp->dots = 0;
 
-          print_row_stats (dp, dltime, false);
+          print_row_stats (dp, dp->dltime, false);
         }
     }
 
@@ -385,13 +397,13 @@ dot_finish (void *progress, double dltime)
   log_set_flush (false);
 
   if (dp->dots == 0)
-    logprintf (LOG_VERBOSE, "\n%6sK",
+    logprintf (LOG_PROGRESS, "\n%6sK",
                number_to_static_string (dp->rows * ROW_BYTES / 1024));
   for (i = dp->dots; i < opt.dots_in_line; i++)
     {
       if (i % opt.dot_spacing == 0)
-        logputs (LOG_VERBOSE, " ");
-      logputs (LOG_VERBOSE, " ");
+        logputs (LOG_PROGRESS, " ");
+      logputs (LOG_PROGRESS, " ");
     }
 
   print_row_stats (dp, dltime, true);
@@ -407,7 +419,7 @@ dot_finish (void *progress, double dltime)
    giga.  */
 
 static void
-dot_set_params (const char *params)
+dot_set_params (char *params)
 {
   if (!params || !*params)
     params = opt.dot_style;
@@ -416,7 +428,7 @@ dot_set_params (const char *params)
     return;
 
   /* We use this to set the retrieval style.  */
-  if (!strcasecmp (params, "default"))
+  if (!c_strcasecmp (params, "default"))
     {
       /* Default style: 1K dots, 10 dots in a cluster, 50 dots in a
          line.  */
@@ -424,7 +436,7 @@ dot_set_params (const char *params)
       opt.dot_spacing = 10;
       opt.dots_in_line = 50;
     }
-  else if (!strcasecmp (params, "binary"))
+  else if (!c_strcasecmp (params, "binary"))
     {
       /* "Binary" retrieval: 8K dots, 16 dots in a cluster, 48 dots
          (384K) in a line.  */
@@ -432,7 +444,7 @@ dot_set_params (const char *params)
       opt.dot_spacing = 16;
       opt.dots_in_line = 48;
     }
-  else if (!strcasecmp (params, "mega"))
+  else if (!c_strcasecmp (params, "mega"))
     {
       /* "Mega" retrieval, for retrieving very long files; each dot is
          64K, 8 dots in a cluster, 6 clusters (3M) in a line.  */
@@ -440,7 +452,7 @@ dot_set_params (const char *params)
       opt.dot_spacing = 8;
       opt.dots_in_line = 48;
     }
-  else if (!strcasecmp (params, "giga"))
+  else if (!c_strcasecmp (params, "giga"))
     {
       /* "Giga" retrieval, for retrieving very very *very* long files;
          each dot is 1M, 8 dots in a cluster, 4 clusters (32M) in a
@@ -454,7 +466,7 @@ dot_set_params (const char *params)
              _("Invalid dot style specification %s; leaving unchanged.\n"),
              quote (params));
 }
-
+
 /* "Thermometer" (bar) progress. */
 
 /* Assumed screen width if we can't find the real value.  */
@@ -495,6 +507,7 @@ static volatile sig_atomic_t received_sigwinch;
 #define ETA_REFRESH_INTERVAL 0.99
 
 struct bar_progress {
+  const char *f_download;       /* Filename of the downloaded file */
   wgint initial_length;         /* how many bytes have been downloaded
                                    previously. */
   wgint total_length;           /* expected total byte count when the
@@ -505,6 +518,7 @@ struct bar_progress {
                                    measured since the beginning of
                                    download. */
 
+  double dltime;                /* download time so far */
   int width;                    /* screen width we're using at the
                                    time the progress gauge was
                                    created.  this is different from
@@ -551,7 +565,7 @@ static void create_image (struct bar_progress *, double, bool);
 static void display_image (char *);
 
 static void *
-bar_create (wgint initial, wgint total)
+bar_create (const char *f_download, wgint initial, wgint total)
 {
   struct bar_progress *bp = xnew0 (struct bar_progress);
 
@@ -562,6 +576,7 @@ bar_create (wgint initial, wgint total)
 
   bp->initial_length = initial;
   bp->total_length   = total;
+  bp->f_download     = f_download;
 
   /* Initialize screen_width if this hasn't been done or if it might
      have changed, as indicated by receiving SIGWINCH.  */
@@ -595,8 +610,8 @@ static void
 bar_update (void *progress, wgint howmuch, double dltime)
 {
   struct bar_progress *bp = progress;
-  bool force_screen_update = false;
 
+  bp->dltime = dltime;
   bp->count += howmuch;
   if (bp->total_length > 0
       && bp->count + bp->initial_length > bp->total_length)
@@ -608,6 +623,13 @@ bar_update (void *progress, wgint howmuch, double dltime)
     bp->total_length = bp->initial_length + bp->count;
 
   update_speed_ring (bp, howmuch, dltime);
+}
+
+static void
+bar_draw (void *progress)
+{
+  bool force_screen_update = false;
+  struct bar_progress *bp = progress;
 
   /* If SIGWINCH (the window size change signal) been received,
      determine the new screen size and update the screen.  */
@@ -628,13 +650,13 @@ bar_update (void *progress, wgint howmuch, double dltime)
       received_sigwinch = 0;
     }
 
-  if (dltime - bp->last_screen_update < REFRESH_INTERVAL && !force_screen_update)
+  if (bp->dltime - bp->last_screen_update < REFRESH_INTERVAL && !force_screen_update)
     /* Don't update more often than five times per second. */
     return;
 
-  create_image (bp, dltime, false);
+  create_image (bp, bp->dltime, false);
   display_image (bp->buffer);
-  bp->last_screen_update = dltime;
+  bp->last_screen_update = bp->dltime;
 }
 
 static void
@@ -650,7 +672,8 @@ bar_finish (void *progress, double dltime)
   create_image (bp, dltime, true);
   display_image (bp->buffer);
 
-  logputs (LOG_VERBOSE, "\n\n");
+  logputs (LOG_VERBOSE, "\n");
+  logputs (LOG_PROGRESS, "\n");
 
   xfree (bp->buffer);
   xfree (bp);
@@ -766,7 +789,7 @@ update_speed_ring (struct bar_progress *bp, wgint howmuch, double dltime)
 }
 
 #if USE_NLS_PROGRESS_BAR
-int
+static int
 count_cols (const char *mbs)
 {
   wchar_t wc;
@@ -791,16 +814,47 @@ count_cols (const char *mbs)
     }
   return cols;
 }
+
+static int
+cols_to_bytes (const char *mbs, const int cols, int *ncols)
+{
+  int p_cols = 0, bytes = 0;
+  mbchar_t mbc;
+  mbi_iterator_t iter;
+  mbi_init (iter, mbs, strlen(mbs));
+  while (p_cols < cols && mbi_avail (iter))
+    {
+      mbc = mbi_cur (iter);
+      p_cols += mb_width (mbc);
+      /* The multibyte character has exceeded the total number of columns we
+       * have available. The remaining bytes will be padded with a space. */
+      if (p_cols > cols)
+        {
+          p_cols -= mb_width (mbc);
+          break;
+        }
+      bytes += mb_len (mbc);
+      mbi_advance (iter);
+    }
+  *ncols = p_cols;
+  return bytes;
+}
 #else
-# define count_cols(mbs) ((int)(strlen(mbs)))
+static int count_cols (const char *mbs) { return (int) strlen(mbs); }
+static int
+cols_to_bytes (const char *mbs _GL_UNUSED, const int cols, int *ncols)
+{
+  *ncols = cols;
+  return cols;
+}
 #endif
 
-const char *
+static const char *
 get_eta (int *bcd)
 {
   /* TRANSLATORS: "ETA" is English-centric, but this must
      be short, ideally 3 chars.  Abbreviate if necessary.  */
-  static const char eta_str[] = N_("  eta %s");
+  static const char eta_str[] = N_("   eta %s");
   static const char *eta_trans;
   static int bytes_cols_diff;
   if (eta_trans == NULL)
@@ -842,26 +896,20 @@ get_eta (int *bcd)
    are confused when they see strchr (s, '\0') in the code.  */
 #define move_to_end(s) s = strchr (s, '\0');
 
-#ifndef MAX
-# define MAX(a, b) ((a) >= (b) ? (a) : (b))
-#endif
-
 static void
 create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {
+  const int MAX_FILENAME_COLS = bp->width / 4;
   char *p = bp->buffer;
   wgint size = bp->initial_length + bp->count;
 
-  const char *size_grouped = with_thousand_seps (size);
-  int size_grouped_len = count_cols (size_grouped);
-  /* Difference between num cols and num bytes: */
-  int size_grouped_diff = strlen (size_grouped) - size_grouped_len;
   int size_grouped_pad; /* Used to pad the field width for size_grouped. */
 
   struct bar_progress_hist *hist = &bp->hist;
+  int orig_filename_cols = count_cols (bp->f_download);
 
   /* The progress bar should look like this:
-     xx% [=======>             ] nn,nnn 12.34K/s  eta 36m 51s
+     file xx% [=======>             ] nnn.nnK 12.34KB/s  eta 36m 51s
 
      Calculate the geometry.  The idea is to assign as much room as
      possible to the progress bar.  The other idea is to never let
@@ -870,24 +918,70 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      It would be especially bad for the progress bar to be resized
      randomly.
 
+     "file "           - Downloaded filename      - MAX_FILENAME_COLS chars + 1
      "xx% " or "100%"  - percentage               - 4 chars
      "[]"              - progress bar decorations - 2 chars
-     " nnn,nnn,nnn"    - downloaded bytes         - 12 chars or very rarely more
-     " 12.5K/s"        - download rate             - 8 chars
+     " nnn.nnK"        - downloaded bytes         - 7 chars + 1
+     " 12.5KB/s"       - download rate            - 8 chars + 1
      "  eta 36m 51s"   - ETA                      - 14 chars
 
      "=====>..."       - progress bar             - the rest
   */
-  int dlbytes_size = 1 + MAX (size_grouped_len, 11);
-  int progress_size = bp->width - (4 + 2 + dlbytes_size + 8 + 14);
+
+#define PROGRESS_FILENAME_LEN  MAX_FILENAME_COLS + 1
+#define PROGRESS_PERCENT_LEN   4
+#define PROGRESS_DECORAT_LEN   2
+#define PROGRESS_FILESIZE_LEN  7 + 1
+#define PROGRESS_DWNLOAD_RATE  8 + 1
+#define PROGRESS_ETA_LEN       14
+
+  int progress_size = bp->width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
+                                   PROGRESS_DECORAT_LEN + PROGRESS_FILESIZE_LEN +
+                                   PROGRESS_DWNLOAD_RATE + PROGRESS_ETA_LEN);
 
   /* The difference between the number of bytes used,
      and the number of columns used. */
   int bytes_cols_diff = 0;
+  int cols_diff;
+  const char *down_size;
 
   if (progress_size < 5)
     progress_size = 0;
 
+  if (orig_filename_cols <= MAX_FILENAME_COLS)
+    {
+      int padding = MAX_FILENAME_COLS - orig_filename_cols;
+      sprintf (p, "%s ", bp->f_download);
+      p += orig_filename_cols + 1;
+      for (;padding;padding--)
+        *p++ = ' ';
+    }
+  else
+    {
+      int offset_cols;
+      int bytes_in_filename, offset_bytes, col;
+      int *cols_ret = &col;
+      int padding;
+
+#define MIN_SCROLL_TEXT 5
+      if ((orig_filename_cols > MAX_FILENAME_COLS + MIN_SCROLL_TEXT) &&
+          !opt.noscroll &&
+          !done)
+        offset_cols = ((int) bp->tick) % (orig_filename_cols - MAX_FILENAME_COLS + 1);
+      else
+        offset_cols = 0;
+      offset_bytes = cols_to_bytes (bp->f_download, offset_cols, cols_ret);
+      bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes,
+                                         MAX_FILENAME_COLS,
+                                         cols_ret);
+      memcpy (p, bp->f_download + offset_bytes, bytes_in_filename);
+      p += bytes_in_filename;
+      padding = MAX_FILENAME_COLS - *cols_ret;
+      for (;padding;padding--)
+          *p++ = ' ';
+      *p++ = ' ';
+    }
+
   /* "xx% " */
   if (bp->total_length > 0)
     {
@@ -895,7 +989,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       assert (percentage <= 100);
 
       if (percentage < 100)
-        sprintf (p, "%2d%% ", percentage);
+        sprintf (p, "%3d%%", percentage);
       else
         strcpy (p, "100%");
       p += 4;
@@ -961,26 +1055,34 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
         }
       *p++ = ']';
 
-      ++bp->tick;
     }
+ ++bp->tick;
 
-  /* " 234,567,890" */
-  sprintf (p, " %s", size_grouped);
+  /* " 234.56M" */
+  down_size = human_readable (size, 1000, 2);
+  cols_diff = 7 - count_cols (down_size);
+  while (cols_diff > 0)
+  {
+    *p++=' ';
+    cols_diff--;
+  }
+  sprintf (p, " %s", down_size);
   move_to_end (p);
-  /* Pad with spaces to 11 chars for the size_grouped field;
+  /* Pad with spaces to 7 chars for the size_grouped field;
    * couldn't use the field width specifier in sprintf, because
    * it counts in bytes, not characters. */
-  for (size_grouped_pad = 11 - size_grouped_len;
+  for (size_grouped_pad = PROGRESS_FILESIZE_LEN - 7;
        size_grouped_pad > 0;
        --size_grouped_pad)
     {
       *p++ = ' ';
     }
 
-  /* " 12.52K/s" */
+  /* " 12.52Kb/s or 12.52KB/s" */
   if (hist->total_time > 0 && hist->total_bytes)
     {
-      static const char *short_units[] = { "B/s", "K/s", "M/s", "G/s" };
+      static const char *short_units[] = { " B/s", "KB/s", "MB/s", "GB/s" };
+      static const char *short_units_bits[] = { " b/s", "Kb/s", "Mb/s", "Gb/s" };
       int units = 0;
       /* Calculate the download speed using the history ring and
          recent data that hasn't made it to the ring yet.  */
@@ -988,11 +1090,11 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       double dltime = hist->total_time + (dl_total_time - bp->recent_start);
       double dlspeed = calc_rate (dlquant, dltime, &units);
       sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
-               dlspeed, short_units[units]);
+               dlspeed,  !opt.report_bps ? short_units[units] : short_units_bits[units]);
       move_to_end (p);
     }
   else
-    APPEND_LITERAL (" --.-K/s");
+    APPEND_LITERAL (" --.-KB/s");
 
   if (!done)
     {
@@ -1057,9 +1159,18 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       move_to_end (p);
     }
 
-  while (p - bp->buffer - bytes_cols_diff - size_grouped_diff < bp->width)
+  while (p - bp->buffer - bytes_cols_diff < bp->width)
     *p++ = ' ';
   *p = '\0';
+
+  /* 2014-11-14  Darshit Shah  <darnir@gmail.com>
+   * Assert that the length of the progress bar is lesser than the size of the
+   * screen with which we are dealing. This assertion *MUST* always be removed
+   * from the release code since we do not want Wget to crash and burn when the
+   * assertion fails. Instead Wget should continue downloading and display a
+   * horrible and irritating progress bar that spams the screen with newlines.
+   */
+  assert (count_cols (bp->buffer) <= bp->width + 1);
 }
 
 /* Print the contents of the buffer as a one-line ASCII "image" so
@@ -1069,19 +1180,27 @@ static void
 display_image (char *buf)
 {
   bool old = log_set_save_context (false);
-  logputs (LOG_VERBOSE, "\r");
-  logputs (LOG_VERBOSE, buf);
+  logputs (LOG_PROGRESS, "\r");
+  logputs (LOG_PROGRESS, buf);
   log_set_save_context (old);
 }
 
 static void
-bar_set_params (const char *params)
+bar_set_params (char *params)
 {
   char *term = getenv ("TERM");
 
-  if (params
-      && 0 == strcmp (params, "force"))
-    current_impl_locked = 1;
+  if (params)
+    {
+      char *param = strtok (params, ":");
+      do
+        {
+          if (0 == strcmp (param, "force"))
+            current_impl_locked = 1;
+          else if (0 == strcmp (param, "noscroll"))
+            opt.noscroll = true;
+        } while ((param = strtok (NULL, ":")) != NULL);
+    }
 
   if ((opt.lfilename
 #ifdef HAVE_ISATTY
@@ -1111,7 +1230,7 @@ bar_set_params (const char *params)
 
 #ifdef SIGWINCH
 void
-progress_handle_sigwinch (int sig)
+progress_handle_sigwinch (int sig _GL_UNUSED)
 {
   received_sigwinch = 1;
   signal (SIGWINCH, progress_handle_sigwinch);

src/progress.h

@@ -35,7 +35,7 @@ bool valid_progress_implementation_p (const char *);
 void set_progress_implementation (const char *);
 void progress_schedule_redirect (void);
 
-void *progress_create (wgint, wgint);
+void *progress_create (const char *, wgint, wgint);
 bool progress_interactive_p (void *);
 void progress_update (void *, wgint, double);
 void progress_finish (void *, double);

src/ptimer.c

@@ -59,9 +59,7 @@ as that of the covered work.  */
 #include <errno.h>
 #include <unistd.h>
 #include <time.h>
-#ifdef HAVE_SYS_TIME_H
-# include <sys/time.h>
-#endif
+#include <sys/time.h>
 
 /* Cygwin currently (as of 2005-04-08, Cygwin 1.5.14) lacks clock_getres,
    but still defines _POSIX_TIMERS!  Because of that we simply use the
@@ -293,7 +291,7 @@ windows_resolution (void)
     return 10;                  /* according to MSDN */
 }
 #endif  /* PTIMER_WINDOWS */
-
+
 /* The code below this point is independent of timer implementation. */
 
 struct ptimer {

src/ptimer.h

@@ -31,7 +31,7 @@ as that of the covered work.  */
 #ifndef PTIMER_H
 #define PTIMER_H
 
-struct ptimer;			/* forward declaration; all struct
+struct ptimer;                  /* forward declaration; all struct
                                    members are private */
 
 struct ptimer *ptimer_new (void);

src/recur.c

@@ -1,6 +1,6 @@
 /* Handling of recursive HTTP retrieving.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -50,7 +50,7 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-url.h"
 #include "spider.h"
-
+
 /* Functions for maintaining the URL queue.  */
 
 struct queue_element {
@@ -159,7 +159,28 @@ url_dequeue (struct url_queue *queue, struct iri **i,
   xfree (qel);
   return true;
 }
-
+
+static void blacklist_add (struct hash_table *blacklist, const char *url)
+{
+  char *url_unescaped = xstrdup (url);
+
+  url_unescape (url_unescaped);
+  string_set_add (blacklist, url_unescaped);
+  xfree (url_unescaped);
+}
+
+static int blacklist_contains (struct hash_table *blacklist, const char *url)
+{
+  char *url_unescaped = xstrdup(url);
+  int ret;
+
+  url_unescape (url_unescaped);
+  ret = string_set_contains (blacklist, url_unescaped);
+  xfree (url_unescaped);
+
+  return ret;
+}
+
 static bool download_child_p (const struct urlpos *, struct url *, int,
                               struct url *, struct hash_table *, struct iri *);
 static bool descend_redirect_p (const char *, struct url *, int,
@@ -220,7 +241,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
      just URL so we enqueue the canonical form of the URL.  */
   url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true,
                false);
-  string_set_add (blacklist, start_url_parsed->url);
+  blacklist_add (blacklist, start_url_parsed->url);
 
   while (1)
     {
@@ -253,25 +274,22 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
          the second time.  */
       if (dl_url_file_map && hash_table_contains (dl_url_file_map, url))
         {
+          bool is_css_bool;
+
           file = xstrdup (hash_table_get (dl_url_file_map, url));
 
           DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n",
                    url, file));
 
-          /* this sucks, needs to be combined! */
-          if (html_allowed
-              && downloaded_html_set
-              && string_set_contains (downloaded_html_set, file))
+          if ((is_css_bool = (css_allowed
+                  && downloaded_css_set
+                  && string_set_contains (downloaded_css_set, file)))
+              || (html_allowed
+                && downloaded_html_set
+                && string_set_contains (downloaded_html_set, file)))
             {
               descend = true;
-              is_css = false;
-            }
-          if (css_allowed
-              && downloaded_css_set
-              && string_set_contains (downloaded_css_set, file))
-            {
-              descend = true;
-              is_css = true;
+              is_css = is_css_bool;
             }
         }
       else
@@ -314,7 +332,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                   else
                     /* Make sure that the old pre-redirect form gets
                        blacklisted. */
-                    string_set_add (blacklist, url);
+                    blacklist_add (blacklist, url);
                 }
 
               xfree (url);
@@ -325,7 +343,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
               xfree (url);
               url = xstrdup (url_parsed->url);
             }
-          url_free(url_parsed);
+          url_free (url_parsed);
         }
 
       if (opt.spider)
@@ -407,7 +425,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                       /* We blacklist the URL we have enqueued, because we
                          don't want to enqueue (and hence download) the
                          same URL twice.  */
-                      string_set_add (blacklist, child->url->url);
+                      blacklist_add (blacklist, child->url->url);
                     }
                 }
 
@@ -443,8 +461,8 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
         }
 
       xfree (url);
-      xfree_null (referer);
-      xfree_null (file);
+      xfree (referer);
+      xfree (file);
       iri_free (i);
     }
 
@@ -460,7 +478,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
       {
         iri_free (d6);
         xfree (d1);
-        xfree_null (d2);
+        xfree (d2);
       }
   }
   url_queue_delete (queue);
@@ -494,7 +512,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
 
   DEBUGP (("Deciding whether to enqueue \"%s\".\n", url));
 
-  if (string_set_contains (blacklist, url))
+  if (blacklist_contains (blacklist, url))
     {
       if (opt.spider)
         {
@@ -508,15 +526,16 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
     }
 
   /* Several things to check for:
-     1. if scheme is not http, and we don't load it
-     2. check for relative links (if relative_only is set)
-     3. check for domain
-     4. check for no-parent
-     5. check for excludes && includes
-     6. check for suffix
-     7. check for same host (if spanhost is unset), with possible
+     1. if scheme is not https and https_only requested
+     2. if scheme is not http, and we don't load it
+     3. check for relative links (if relative_only is set)
+     4. check for domain
+     5. check for no-parent
+     6. check for excludes && includes
+     7. check for suffix
+     8. check for same host (if spanhost is unset), with possible
      gethostbyname baggage
-     8. check for robots.txt
+     9. check for robots.txt
 
      Addendum: If the URL is FTP, and it is to be loaded, only the
      domain and suffix settings are "stronger".
@@ -528,6 +547,14 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
      More time- and memory- consuming tests should be put later on
      the list.  */
 
+#ifdef HAVE_SSL
+  if (opt.https_only && u->scheme != SCHEME_HTTPS)
+    {
+      DEBUGP (("Not following non-HTTPS links.\n"));
+      goto out;
+    }
+#endif
+
   /* Determine whether URL under consideration has a HTTP-like scheme. */
   u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
 
@@ -586,6 +613,11 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           goto out;
         }
     }
+  if (!accept_url (url))
+    {
+      DEBUGP (("%s is excluded/not-included through regex.\n", url));
+      goto out;
+    }
 
   /* 6. Check for acceptance/rejection rules.  We ignore these rules
      for directories (no file name to match) and for non-leaf HTMLs,
@@ -636,7 +668,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
                  files after downloading or we're just running a spider. */
               if (opt.delete_after || opt.spider)
                 {
-                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  logprintf (LOG_VERBOSE, _("Removing %s.\n"), rfile);
                   if (unlink (rfile))
                       logprintf (LOG_NOTQUIET, "unlink: %s\n",
                                  strerror (errno));
@@ -659,7 +691,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       if (!res_match_path (specs, u->path))
         {
           DEBUGP (("Not following %s because robots.txt forbids it.\n", url));
-          string_set_add (blacklist, url);
+          blacklist_add (blacklist, url);
           goto out;
         }
     }
@@ -701,12 +733,14 @@ descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
   success = download_child_p (upos, orig_parsed, depth,
                               start_url_parsed, blacklist, iri);
 
+  if (success)
+    blacklist_add (blacklist, upos->url->url);
+  else
+    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
+
   url_free (new_parsed);
   xfree (upos);
 
-  if (!success)
-    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
-
   return success;
 }
 

src/res.c

@@ -81,6 +81,7 @@ as that of the covered work.  */
 #include "url.h"
 #include "retr.h"
 #include "res.h"
+#include "c-strcase.h"
 
 #ifdef TESTING
 #include "test.h"
@@ -97,7 +98,7 @@ struct robot_specs {
   int size;
   struct path_info *paths;
 };
-
+
 /* Parsing the robot spec. */
 
 /* Check whether AGENT (a string of length LENGTH) equals "wget" or
@@ -403,10 +404,10 @@ free_specs (struct robot_specs *specs)
   int i;
   for (i = 0; i < specs->count; i++)
     xfree (specs->paths[i].path);
-  xfree_null (specs->paths);
+  xfree (specs->paths);
   xfree (specs);
 }
-
+
 /* Matching of a path according to the specs. */
 
 /* If C is '%' and (ptr[1], ptr[2]) form a hexadecimal number, and if
@@ -471,7 +472,7 @@ res_match_path (const struct robot_specs *specs, const char *path)
       }
   return true;
 }
-
+
 /* Registering the specs. */
 
 static struct hash_table *registered_specs;
@@ -521,7 +522,7 @@ res_get_specs (const char *host, int port)
     return NULL;
   return hash_table_get (registered_specs, hp);
 }
-
+
 /* Loading the robots file.  */
 
 #define RES_SPECS_LOCATION "/robots.txt"
@@ -578,11 +579,10 @@ res_retrieve_file (const char *url, char **file, struct iri *iri)
          allocated the file name, deallocate is here so that the
          caller doesn't have to worry about it.  */
       xfree (*file);
-      *file = NULL;
     }
   return err == RETROK;
 }
-
+
 bool
 is_robots_txt_url (const char *url)
 {
@@ -593,7 +593,7 @@ is_robots_txt_url (const char *url)
 
   return ret;
 }
-
+
 void
 res_cleanup (void)
 {
@@ -611,15 +611,15 @@ res_cleanup (void)
       registered_specs = NULL;
     }
 }
-
+
 #ifdef TESTING
 
 const char *
-test_is_robots_txt_url()
+test_is_robots_txt_url(void)
 {
-  int i;
-  struct {
-    char *url;
+  unsigned i;
+  static const struct {
+    const char *url;
     bool expected_result;
   } test_array[] = {
     { "http://www.yoyodyne.com/robots.txt", true },
@@ -627,7 +627,7 @@ test_is_robots_txt_url()
     { "http://www.yoyodyne.com/somepath/robots.txt", false },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       mu_assert ("test_is_robots_txt_url: wrong result",
                  is_robots_txt_url (test_array[i].url) == test_array[i].expected_result);
@@ -641,4 +641,3 @@ test_is_robots_txt_url()
 /*
  * vim: et ts=2 sw=2
  */
-

src/retr.c

@@ -1,6 +1,6 @@
 /* File retrieval.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -37,6 +37,9 @@ as that of the covered work.  */
 #include <errno.h>
 #include <string.h>
 #include <assert.h>
+#ifdef VMS
+# include <unixio.h>            /* For delete(). */
+#endif
 
 #include "exits.h"
 #include "utils.h"
@@ -67,7 +70,7 @@ FILE *output_stream;
 /* Whether output_document is a regular file we can manipulate,
    i.e. not `-' or a device file. */
 bool output_stream_regular;
-
+
 static struct {
   wgint chunk_bytes;
   double chunk_start;
@@ -133,19 +136,18 @@ limit_bandwidth (wgint bytes, struct ptimer *timer)
   limit_data.chunk_start = ptimer_read (timer);
 }
 
-#ifndef MIN
-# define MIN(i, j) ((i) <= (j) ? (i) : (j))
-#endif
-
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount
-   of data written.  */
+   of data written.  If OUT2 is not NULL, also write BUF to OUT2.
+   In case of error writing to OUT, -1 is returned.  In case of error
+   writing to OUT2, -2 is returned.  Return 1 if the whole BUF was
+   skipped.  */
 
 static int
-write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
-            wgint *written)
+write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
+            wgint *skip, wgint *written)
 {
-  if (!out)
+  if (out == NULL && out2 == NULL)
     return 1;
   if (*skip > bufsize)
     {
@@ -161,7 +163,10 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
         return 1;
     }
 
-  fwrite (buf, 1, bufsize, out);
+  if (out != NULL)
+    fwrite (buf, 1, bufsize, out);
+  if (out2 != NULL)
+    fwrite (buf, 1, bufsize, out2);
   *written += bufsize;
 
   /* Immediately flush the downloaded data.  This should not hinder
@@ -178,9 +183,17 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
      actual justification.  (Also, why 16K?  Anyone test other values?)
   */
 #ifndef __VMS
-  fflush (out);
+  if (out != NULL)
+    fflush (out);
+  if (out2 != NULL)
+    fflush (out2);
 #endif /* ndef __VMS */
-  return !ferror (out);
+  if (out != NULL && ferror (out))
+    return -1;
+  else if (out2 != NULL && ferror (out2))
+    return -2;
+  else
+    return 0;
 }
 
 /* Read the contents of file descriptor FD until it the connection
@@ -198,13 +211,21 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
    the amount of data written to disk.  The time it took to download
    the data is stored to ELAPSED.
 
+   If OUT2 is non-NULL, the contents is also written to OUT2.
+   OUT2 will get an exact copy of the response: if this is a chunked
+   response, everything -- including the chunk headers -- is written
+   to OUT2.  (OUT will only get the unchunked response.)
+
    The function exits and returns the amount of data read.  In case of
    error while reading data, -1 is returned.  In case of error while
-   writing data, -2 is returned.  */
+   writing data to OUT, -2 is returned.  In case of error while writing
+   data to OUT2, -3 is returned.  */
 
 int
-fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
-              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
+fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread, wgint startpos,
+
+              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
+              FILE *out2)
 {
   int ret = 0;
 #undef max
@@ -238,13 +259,18 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
   if (flags & rb_skip_startpos)
     skip = startpos;
 
-  if (opt.verbose)
+  if (opt.show_progress)
     {
+      const char *filename_progress;
       /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
          argument to progress_create because the indicator doesn't
          (yet) know about "skipping" data.  */
       wgint start = skip ? 0 : startpos;
-      progress = progress_create (start, start + toread);
+      if (opt.dir_prefix)
+        filename_progress = downloaded_filename + strlen (opt.dir_prefix) + 1;
+      else
+        filename_progress = downloaded_filename;
+      progress = progress_create (filename_progress, start, start + toread);
       progress_interactive = progress_interactive_p (progress);
     }
 
@@ -287,13 +313,24 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
                   ret = -1;
                   break;
                 }
+              else if (out2 != NULL)
+                fwrite (line, 1, strlen (line), out2);
 
               remaining_chunk_size = strtol (line, &endl, 16);
+              xfree (line);
+
               if (remaining_chunk_size == 0)
                 {
                   ret = 0;
-                  if (fd_read_line (fd) == NULL)
+                  line = fd_read_line (fd);
+                  if (line == NULL)
                     ret = -1;
+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
                   break;
                 }
             }
@@ -342,21 +379,33 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
 
       if (ret > 0)
         {
+          int write_res;
+
           sum_read += ret;
-          if (!write_data (out, dlbuf, ret, &skip, &sum_written))
+          write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
+          if (write_res < 0)
             {
-              ret = -2;
+              ret = (write_res == -3) ? -3 : -2;
               goto out;
             }
           if (chunked)
             {
               remaining_chunk_size -= ret;
               if (remaining_chunk_size == 0)
-                if (fd_read_line (fd) == NULL)
-                  {
-                    ret = -1;
-                    break;
-                  }
+                {
+                  char *line = fd_read_line (fd);
+                  if (line == NULL)
+                    {
+                      ret = -1;
+                      break;
+                    }
+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
+                }
             }
         }
 
@@ -366,7 +415,7 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
       if (progress)
         progress_update (progress, ret, ptimer_read (timer));
 #ifdef WINDOWS
-      if (toread > 0 && !opt.quiet)
+      if (toread > 0 && opt.show_progress)
         ws_percenttitle (100.0 *
                          (startpos + sum_read) / (startpos + toread));
 #endif
@@ -388,11 +437,11 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
   if (qtywritten)
     *qtywritten += sum_written;
 
-  free (dlbuf);
+  xfree (dlbuf);
 
   return ret;
 }
-
+
 /* Read a hunk of data from FD, up until a terminator.  The hunk is
    limited by whatever the TERMINATOR callback chooses as its
    terminator.  For example, if terminator stops at newline, the hunk
@@ -497,7 +546,7 @@ fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
       rdlen = fd_read (fd, hunk + tail, remain, 0);
       if (rdlen < 0)
         {
-          xfree_null (hunk);
+          xfree (hunk);
           return NULL;
         }
       tail += rdlen;
@@ -542,7 +591,7 @@ fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
 }
 
 static const char *
-line_terminator (const char *start, const char *peeked, int peeklen)
+line_terminator (const char *start _GL_UNUSED, const char *peeked, int peeklen)
 {
   const char *p = memchr (peeked, '\n', peeklen);
   if (p)
@@ -569,7 +618,7 @@ fd_read_line (int fd)
 {
   return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
 }
-
+
 /* Return a printed representation of the download rate, along with
    the units appropriate for the download speed.  */
 
@@ -578,6 +627,7 @@ retr_rate (wgint bytes, double secs)
 {
   static char res[20];
   static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
+  static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" };
   int units;
 
   double dlrate = calc_rate (bytes, secs, &units);
@@ -585,7 +635,7 @@ retr_rate (wgint bytes, double secs)
      e.g. "1022", "247", "12.5", "2.38".  */
   sprintf (res, "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
-           dlrate, rate_names[units]);
+           dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);
 
   return res;
 }
@@ -602,6 +652,11 @@ double
 calc_rate (wgint bytes, double secs, int *units)
 {
   double dlrate;
+  double bibyte = 1000.0;
+
+  if (!opt.report_bps)
+    bibyte = 1024.0;
+
 
   assert (secs >= 0);
   assert (bytes >= 0);
@@ -613,35 +668,39 @@ calc_rate (wgint bytes, double secs, int *units)
        0 and the timer's resolution, assume half the resolution.  */
     secs = ptimer_resolution () / 2.0;
 
-  dlrate = bytes / secs;
-  if (dlrate < 1024.0)
+  dlrate = convert_to_bits (bytes) / secs;
+  if (dlrate < bibyte)
     *units = 0;
-  else if (dlrate < 1024.0 * 1024.0)
-    *units = 1, dlrate /= 1024.0;
-  else if (dlrate < 1024.0 * 1024.0 * 1024.0)
-    *units = 2, dlrate /= (1024.0 * 1024.0);
+  else if (dlrate < (bibyte * bibyte))
+    *units = 1, dlrate /= bibyte;
+  else if (dlrate < (bibyte * bibyte * bibyte))
+    *units = 2, dlrate /= (bibyte * bibyte);
+
   else
     /* Maybe someone will need this, one day. */
-    *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0);
+    *units = 3, dlrate /= (bibyte * bibyte * bibyte);
 
   return dlrate;
 }
-
 
-#define SUSPEND_POST_DATA do {                  \
-  post_data_suspended = true;                   \
-  saved_post_data = opt.post_data;              \
-  saved_post_file_name = opt.post_file_name;    \
-  opt.post_data = NULL;                         \
-  opt.post_file_name = NULL;                    \
+
+#define SUSPEND_METHOD do {                     \
+  method_suspended = true;                      \
+  saved_body_data = opt.body_data;              \
+  saved_body_file_name = opt.body_file;         \
+  saved_method = opt.method;                    \
+  opt.body_data = NULL;                         \
+  opt.body_file = NULL;                         \
+  opt.method = NULL;                            \
 } while (0)
 
-#define RESTORE_POST_DATA do {                          \
-  if (post_data_suspended)                              \
+#define RESTORE_METHOD do {                             \
+  if (method_suspended)                                 \
     {                                                   \
-      opt.post_data = saved_post_data;                  \
-      opt.post_file_name = saved_post_file_name;        \
-      post_data_suspended = false;                      \
+      opt.body_data = saved_body_data;                  \
+      opt.body_file = saved_body_file_name;             \
+      opt.method = saved_method;                        \
+      method_suspended = false;                         \
     }                                                   \
 } while (0)
 
@@ -669,9 +728,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
   char *local_file;
   int redirection_count = 0;
 
-  bool post_data_suspended = false;
-  char *saved_post_data = NULL;
-  char *saved_post_file_name = NULL;
+  bool method_suspended = false;
+  char *saved_body_data = NULL;
+  char *saved_method = NULL;
+  char *saved_body_file_name = NULL;
 
   /* If dt is NULL, use local storage.  */
   if (!dt)
@@ -704,7 +764,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       pi->utf8_encode = false;
 
       /* Parse the proxy URL.  */
-      proxy_url = url_parse (proxy, &up_error_code, NULL, true);
+      proxy_url = url_parse (proxy, &up_error_code, pi, true);
       if (!proxy_url)
         {
           char *error = url_error (proxy, up_error_code);
@@ -712,7 +772,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
                      proxy, error);
           xfree (url);
           xfree (error);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           result = PROXERR;
           goto bail;
         }
@@ -721,10 +781,12 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
           url_free (proxy_url);
           xfree (url);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           result = PROXERR;
           goto bail;
         }
+      iri_free(pi);
+      xfree (proxy);
     }
 
   if (u->scheme == SCHEME_HTTP
@@ -773,8 +835,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
       assert (mynewloc != NULL);
 
-      if (local_file)
-        xfree (local_file);
+      xfree (local_file);
 
       /* The HTTP specs only allow absolute URLs to appear in
          redirects, but a ton of boneheaded webservers and CGIs out
@@ -788,7 +849,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          the content encoding. */
       iri->utf8_encode = opt.enable_iri;
       set_content_encoding (iri, NULL);
-      xfree_null (iri->orig_url);
+      xfree (iri->orig_url);
 
       /* Now, see if this new location makes sense. */
       newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
@@ -804,7 +865,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           xfree (url);
           xfree (mynewloc);
           xfree (error);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           goto bail;
         }
 
@@ -826,7 +887,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
             }
           xfree (url);
           xfree (mynewloc);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           result = WRONGCODE;
           goto bail;
         }
@@ -845,15 +906,19 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          index page; that redirection is clearly a GET.  We "suspend"
          POST data for the duration of the redirections, and restore
          it when we're done.
-	 
-	 RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
-	 specifically to preserve the method of the request.
-	 */
-      if (result != NEWLOCATION_KEEP_POST && !post_data_suspended)
-        SUSPEND_POST_DATA;
+
+         RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
+         specifically to preserve the method of the request.
+     */
+      if (result != NEWLOCATION_KEEP_POST && !method_suspended)
+        SUSPEND_METHOD;
 
       goto redirected;
     }
+  else
+    {
+      xfree(mynewloc);
+    }
 
   /* Try to not encode in UTF-8 if fetching failed */
   if (!(*dt & RETROKF) && iri->utf8_encode)
@@ -867,6 +932,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       if (u)
         {
           DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
+          xfree (url);
           url = xstrdup (u->url);
           iri_fallbacked = 1;
           goto redirected;
@@ -883,16 +949,16 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
         register_redirection (origurl, u->url);
 
       if (*dt & TEXTHTML)
-        register_html (u->url, local_file);
+        register_html (local_file);
 
       if (*dt & TEXTCSS)
-        register_css (u->url, local_file);
+        register_css (local_file);
     }
 
   if (file)
     *file = local_file ? local_file : NULL;
   else
-    xfree_null (local_file);
+    xfree (local_file);
 
   if (orig_parsed != u)
     {
@@ -913,7 +979,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       xfree (url);
     }
 
-  RESTORE_POST_DATA;
+  RESTORE_METHOD;
 
 bail:
   if (register_status)
@@ -947,7 +1013,6 @@ retrieve_from_file (const char *file, bool html, int *count)
   if (url_valid_scheme (url))
     {
       int dt,url_err;
-      uerr_t status;
       struct url *url_parsed = url_parse (url, &url_err, iri, true);
       if (!url_parsed)
         {
@@ -973,12 +1038,11 @@ retrieve_from_file (const char *file, bool html, int *count)
       /* If we have a found a content encoding, use it.
        * ( == is okay, because we're checking for identical object) */
       if (iri->content_encoding != opt.locale)
-	  set_uri_encoding (iri, iri->content_encoding, false);
+          set_uri_encoding (iri, iri->content_encoding, false);
 
       /* Reset UTF-8 encode status */
       iri->utf8_encode = opt.enable_iri;
-      xfree_null (iri->orig_url);
-      iri->orig_url = NULL;
+      xfree (iri->orig_url);
 
       input_file = url_file;
     }
@@ -988,12 +1052,12 @@ retrieve_from_file (const char *file, bool html, int *count)
   url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
               : get_urls_file (input_file));
 
-  xfree_null (url_file);
+  xfree (url_file);
 
   for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
     {
-      char *filename = NULL, *new_file = NULL;
-      int dt;
+      char *filename = NULL, *new_file = NULL, *proxy;
+      int dt = 0;
       struct iri *tmpiri = iri_dup (iri);
       struct url *parsed_url = NULL;
 
@@ -1008,8 +1072,9 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
 
+      proxy = getproxy (cur_url->url);
       if ((opt.recursive || opt.page_requisites)
-          && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
+          && (cur_url->url->scheme != SCHEME_FTP || proxy))
         {
           int old_follow_ftp = opt.follow_ftp;
 
@@ -1027,6 +1092,7 @@ retrieve_from_file (const char *file, bool html, int *count)
                                cur_url->url->url, &filename,
                                &new_file, NULL, &dt, opt.recursive, tmpiri,
                                true);
+      xfree (proxy);
 
       if (parsed_url)
           url_free (parsed_url);
@@ -1041,8 +1107,8 @@ Removing file due to --delete-after in retrieve_from_file():\n"));
           dt &= ~RETROKF;
         }
 
-      xfree_null (new_file);
-      xfree_null (filename);
+      xfree (new_file);
+      xfree (filename);
       iri_free (tmpiri);
     }
 
@@ -1118,7 +1184,7 @@ free_urlpos (struct urlpos *l)
       struct urlpos *next = l->next;
       if (l->url)
         url_free (l->url);
-      xfree_null (l->local_name);
+      xfree (l->local_name);
       xfree (l);
       l = next;
     }
@@ -1128,7 +1194,16 @@ free_urlpos (struct urlpos *l)
 void
 rotate_backups(const char *fname)
 {
-  int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
+#ifdef __VMS
+# define SEP "_"
+# define AVS ";*"                       /* All-version suffix. */
+# define AVSL (sizeof (AVS) - 1)
+#else
+# define SEP "."
+# define AVSL 0
+#endif
+
+  int maxlen = strlen (fname) + sizeof (SEP) + numdigit (opt.backups) + AVSL;
   char *from = (char *)alloca (maxlen);
   char *to = (char *)alloca (maxlen);
   struct_stat sb;
@@ -1140,12 +1215,24 @@ rotate_backups(const char *fname)
 
   for (i = opt.backups; i > 1; i--)
     {
-      sprintf (from, "%s.%d", fname, i - 1);
-      sprintf (to, "%s.%d", fname, i);
+#ifdef VMS
+      /* Delete (all versions of) any existing max-suffix file, to avoid
+       * creating multiple versions of it.  (On VMS, rename() will
+       * create a new version of an existing destination file, not
+       * destroy/overwrite it.)
+       */
+      if (i == opt.backups)
+        {
+          sprintf (to, "%s%s%d%s", fname, SEP, i, AVS);
+          delete (to);
+        }
+#endif
+      sprintf (to, "%s%s%d", fname, SEP, i);
+      sprintf (from, "%s%s%d", fname, SEP, i - 1);
       rename (from, to);
     }
 
-  sprintf (to, "%s.%d", fname, 1);
+  sprintf (to, "%s%s%d", fname, SEP, 1);
   rename(fname, to);
 }
 
@@ -1158,7 +1245,6 @@ getproxy (struct url *u)
 {
   char *proxy = NULL;
   char *rewritten_url;
-  static char rewritten_storage[1024];
 
   if (!opt.use_proxy)
     return NULL;
@@ -1188,13 +1274,9 @@ getproxy (struct url *u)
      getproxy() to return static storage. */
   rewritten_url = rewrite_shorthand_url (proxy);
   if (rewritten_url)
-    {
-      strncpy (rewritten_storage, rewritten_url, sizeof (rewritten_storage));
-      rewritten_storage[sizeof (rewritten_storage) - 1] = '\0';
-      proxy = rewritten_storage;
-    }
+    return rewritten_url;
 
-  return proxy;
+  return strdup(proxy);
 }
 
 /* Returns true if URL would be downloaded through a proxy. */
@@ -1203,9 +1285,13 @@ bool
 url_uses_proxy (struct url * u)
 {
   bool ret;
+  char *proxy;
+
   if (!u)
     return false;
-  ret = getproxy (u) != NULL;
+  proxy = getproxy (u);
+  ret = proxy != NULL;
+  xfree (proxy);
   return ret;
 }
 

src/retr.h

@@ -34,6 +34,8 @@ as that of the covered work.  */
 
 #include "url.h"
 
+extern int numurls;
+
 /* These global vars should be made static to retr.c and exported via
    functions! */
 extern SUM_SIZE_INT total_downloaded_bytes;
@@ -50,7 +52,7 @@ enum {
   rb_chunked_transfer_encoding = 4
 };
 
-int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int);
+int fd_read_body (const char *, int, FILE *, wgint, wgint, wgint *, wgint *, double *, int, FILE *);
 
 typedef const char *(*hunk_terminator_t) (const char *, const char *, int);
 

src/spider.c

@@ -51,7 +51,7 @@ spider_cleanup (void)
   if (nonexisting_urls_set)
     string_set_free (nonexisting_urls_set);
 }
-
+
 /* Remembers broken links.  */
 void
 nonexisting_url (const char *url)
@@ -97,4 +97,3 @@ print_broken_links (void)
 /*
  * vim: et ts=2 sw=2
  */
-

src/spider.h

@@ -34,5 +34,6 @@ as that of the covered work.  */
 #define visited_url(a,b)
 void nonexisting_url (const char *);
 void print_broken_links (void);
+void spider_cleanup (void);
 
 #endif /* SPIDER_H */

src/ssl.h

@@ -1,6 +1,6 @@
 /* SSL support.
    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011 Free Software Foundation, Inc.
+   2009, 2010, 2011, 2012 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.
@@ -33,7 +33,7 @@ as that of the covered work.  */
 #define GEN_SSLFUNC_H
 
 bool ssl_init (void);
-bool ssl_connect_wget (int);
+bool ssl_connect_wget (int, const char *);
 bool ssl_check_certificate (int, const char *);
 
 #endif /* GEN_SSLFUNC_H */

src/sysdep.h

@@ -78,35 +78,15 @@ as that of the covered work.  */
    These have to be after the above namespace tweaks, but before any
    non-preprocessor code.  */
 
-#if HAVE_ALLOCA_H
-# include <alloca.h>
-#elif defined WINDOWS
-# include <malloc.h>
-# ifndef alloca
-#  define alloca _alloca
-# endif
-#elif defined __GNUC__
-# define alloca __builtin_alloca
-#elif defined _AIX
-# define alloca __alloca
-#else
-# include <stddef.h>
-# ifdef  __cplusplus
-extern "C"
-# endif
-void *alloca (size_t);
-#endif
+#include <alloca.h>
 
 /* Must include these, so we can test for the missing stat macros and
    define them as necessary.  */
 #include <sys/types.h>
 #include <sys/stat.h>
 
-#ifdef HAVE_INTTYPES_H
-  /* Compaq C V6.5-303 (dtk) on HP Tru64 UNIX V5.1B (Rev. 2650) needs: */
-# include <stdint.h>
-# include <inttypes.h>
-#endif
+#include <stdint.h>
+#include <inttypes.h>
 
 #ifdef WINDOWS
 /* Windows doesn't have some functions normally found on Unix-like
@@ -121,34 +101,6 @@ void *alloca (size_t);
 /* Provided by gnulib on systems that don't have it: */
 # include <stdbool.h>
 
-/* Needed for compilation under OS/2 and MSDOS */
-#if defined(__EMX__) || defined(MSDOS)
-# ifndef S_ISLNK
-#  define S_ISLNK(m) 0
-# endif
-# ifndef lstat
-#  define lstat stat
-# endif
-#endif /* __EMX__ || MSDOS */
-
-/* Reportedly, stat() macros are broken on some old systems.  Those
-   systems will have to fend for themselves, as I will not introduce
-   new code to handle it.
-
-   However, I will add code for *missing* macros, and the following
-   are reportedly missing from many systems.  */
-#ifndef S_ISLNK
-# define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK)
-#endif
-#ifndef S_ISDIR
-# define S_ISDIR(m) (((m) & (_S_IFMT)) == (_S_IFDIR))
-#endif
-#ifndef S_ISREG
-# define S_ISREG(m) (((m) & _S_IFMT) == _S_IFREG)
-#endif
-
-/* These are needed so we can #define struct_stat to struct _stati64
-   under Windows. */
 #ifndef struct_stat
 # define struct_stat struct stat
 #endif
@@ -156,6 +108,8 @@ void *alloca (size_t);
 # define struct_fstat struct stat
 #endif
 
+#include <intprops.h>
+
 /* For CHAR_BIT, LONG_MAX, etc. */
 #include <limits.h>
 
@@ -163,9 +117,6 @@ void *alloca (size_t);
 # define CHAR_BIT 8
 #endif
 
-/* From gnulib, simplified to assume a signed type. */
-#define TYPE_MAXIMUM(t) ((t) (~ (~ (t) 0 << (sizeof (t) * CHAR_BIT - 1))))
-
 /* These are defined in cmpt.c if missing, so we must declare
    them.  */
 #ifndef HAVE_STRCASECMP
@@ -212,23 +163,7 @@ int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
 # define SYSTEM_FNMATCH
 #endif
 
-#ifdef SYSTEM_FNMATCH
-# include <fnmatch.h>
-#else  /* not SYSTEM_FNMATCH */
-/* Define fnmatch flags.  Undef them first to avoid warnings in case
-   an evil library include chose to include system fnmatch.h.  */
-# undef FNM_PATHNAME
-# undef FNM_NOESCAPE
-# undef FNM_PERIOD
-# undef FNM_NOMATCH
-
-# define FNM_PATHNAME	(1 << 0) /* No wildcard can ever match `/'.  */
-# define FNM_NOESCAPE	(1 << 1) /* Backslashes don't quote special chars.  */
-# define FNM_PERIOD	(1 << 2) /* Leading `.' is matched only explicitly.  */
-# define FNM_NOMATCH	1
-
-int fnmatch (const char *, const char *, int);
-#endif
+#include <fnmatch.h>
 
 /* Provide sig_atomic_t if the system doesn't.  */
 #ifndef HAVE_SIG_ATOMIC_T

src/test.c

@@ -28,7 +28,12 @@ Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
+#include "wget.h"
+
 #include <stdio.h>
+#ifdef ENABLE_NLS
+# include <locale.h>
+#endif
 
 #include "test.h"
 
@@ -36,20 +41,22 @@ as that of the covered work.  */
 #error "TESTING not set!!!"
 #endif
 
-const char *test_parse_content_disposition();
-const char *test_subdir_p();
-const char *test_dir_matches_p();
-const char *test_commands_sorted();
-const char *test_cmd_spec_restrict_file_names();
-const char *test_path_simplify ();
-const char *test_append_uri_pathel();
-const char *test_are_urls_equal();
-const char *test_is_robots_txt_url();
+const char *test_parse_content_disposition(void);
+const char *test_subdir_p(void);
+const char *test_dir_matches_p(void);
+const char *test_commands_sorted(void);
+const char *test_cmd_spec_restrict_file_names(void);
+const char *test_path_simplify (void);
+const char *test_append_uri_pathel(void);
+const char *test_are_urls_equal(void);
+const char *test_is_robots_txt_url(void);
+
+const char *program_argstring = "TEST";
 
 int tests_run;
 
 static const char *
-all_tests()
+all_tests(void)
 {
   mu_run_test (test_parse_content_disposition);
   mu_run_test (test_subdir_p);
@@ -64,13 +71,21 @@ all_tests()
   return NULL;
 }
 
-char *program_name; /* Needed by lib/error.c. */
+const char *program_name; /* Needed by lib/error.c. */
 
 int
-main (int argc, char *argv[])
+main (int argc _GL_UNUSED, const char *argv[])
 {
   const char *result;
 
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   program_name = argv[0];
 
   result = all_tests();
@@ -92,4 +107,3 @@ main (int argc, char *argv[])
 /*
  * vim: et ts=2 sw=2
  */
-

src/test.h

@@ -45,9 +45,18 @@ do { \
 
 extern int tests_run;
 
+const char *test_parse_content_disposition(void);
+const char *test_commands_sorted(void);
+const char *test_cmd_spec_restrict_file_names(void);
+const char *test_is_robots_txt_url(void);
+const char *test_path_simplify (void);
+const char *test_append_uri_pathel(void);
+const char *test_are_urls_equal(void);
+const char *test_subdir_p(void);
+const char *test_dir_matches_p(void);
+
 #endif /* TEST_H */
 
 /*
  * vim: et ts=2 sw=2
  */
-

src/url.c

@@ -41,6 +41,7 @@ as that of the covered work.  */
 #include "utils.h"
 #include "url.h"
 #include "host.h"  /* for is_valid_ipv6_address */
+#include "c-strcase.h"
 
 #ifdef __VMS
 #include "vms.h"
@@ -85,7 +86,7 @@ static struct scheme_data supported_schemes[] =
 /* Forward declarations: */
 
 static bool path_simplify (enum url_scheme, char *);
-
+
 /* Support for escaping and unescaping of URL strings.  */
 
 /* Table of "reserved" and "unsafe" characters.  Those terms are
@@ -169,7 +170,7 @@ static const unsigned char urlchr_table[256] =
    The transformation is done in place.  If you need the original
    string intact, make a copy before calling this function.  */
 
-static void
+void
 url_unescape (char *s)
 {
   char *t = s;                  /* t - tortoise */
@@ -272,7 +273,7 @@ url_escape_allow_passthrough (const char *s)
 {
   return url_escape_1 (s, urlchr_unsafe, true);
 }
-
+
 /* Decide whether the char at position P needs to be encoded.  (It is
    not enough to pass a single char *P because the function may need
    to inspect the surrounding context.)
@@ -418,7 +419,7 @@ reencode_escapes (const char *s)
   assert (p2 - newstr == newlen);
   return newstr;
 }
-
+
 /* Returns the scheme type if the scheme is supported, or
    SCHEME_INVALID if not.  */
 
@@ -574,8 +575,8 @@ rewrite_shorthand_url (const char *url)
         goto http;
 
       /* Turn "foo.bar.com:path" to "ftp://foo.bar.com/path". */
-      ret = aprintf ("ftp://%s", url);
-      ret[6 + (p - url)] = '/';
+      if ((ret = aprintf ("ftp://%s", url)) != NULL)
+        ret[6 + (p - url)] = '/';
     }
   else
     {
@@ -585,7 +586,7 @@ rewrite_shorthand_url (const char *url)
     }
   return ret;
 }
-
+
 static void split_path (const char *, char **, char **);
 
 /* Like strpbrk, with the exception that it returns the pointer to the
@@ -681,7 +682,6 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
   char *user = NULL, *passwd = NULL;
 
   const char *url_encoded = NULL;
-  char *new_url = NULL;
 
   int error_code;
 
@@ -695,26 +695,30 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       goto error;
     }
 
+  url_encoded = url;
+
   if (iri && iri->utf8_encode)
     {
+      char *new_url = NULL;
+
       iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
       if (!iri->utf8_encode)
         new_url = NULL;
       else
-        iri->orig_url = xstrdup (url);
+        {
+          xfree (iri->orig_url);
+          iri->orig_url = xstrdup (url);
+          url_encoded = reencode_escapes (new_url);
+          if (url_encoded != new_url)
+            xfree (new_url);
+          percent_encode = false;
+        }
     }
 
-  /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
   if (percent_encode)
-    url_encoded = reencode_escapes (new_url ? new_url : url);
-  else
-    url_encoded = new_url ? new_url : url;
+    url_encoded = reencode_escapes (url);
 
   p = url_encoded;
-
-  if (new_url && url_encoded != new_url)
-    xfree (new_url);
-
   p += strlen (supported_schemes[scheme].leading_string);
   uname_b = p;
   p = url_skip_credentials (p);
@@ -913,7 +917,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       u->url = url_string (u, URL_AUTH_SHOW);
 
       if (url_encoded != url)
-        xfree ((char *) url_encoded);
+        xfree (url_encoded);
     }
   else
     {
@@ -928,7 +932,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
  error:
   /* Cleanup in case of error: */
   if (url_encoded && url_encoded != url)
-    xfree ((char *) url_encoded);
+    xfree (url_encoded);
 
   /* Transmit the error code to the caller, if the caller wants to
      know.  */
@@ -953,7 +957,7 @@ url_error (const char *url, int error_code)
 
       if ((p = strchr (scheme, ':')))
         *p = '\0';
-      if (!strcasecmp (scheme, "https"))
+      if (!c_strcasecmp (scheme, "https"))
         error = aprintf (_("HTTPS support not compiled in"));
       else
         error = aprintf (_(parse_errors[error_code]), quote (scheme));
@@ -1169,22 +1173,25 @@ url_set_file (struct url *url, const char *newfile)
 void
 url_free (struct url *url)
 {
-  xfree (url->host);
-  xfree (url->path);
-  xfree (url->url);
+  if (url)
+    {
+      xfree (url->host);
+      xfree (url->path);
+      xfree (url->url);
 
-  xfree_null (url->params);
-  xfree_null (url->query);
-  xfree_null (url->fragment);
-  xfree_null (url->user);
-  xfree_null (url->passwd);
+      xfree (url->params);
+      xfree (url->query);
+      xfree (url->fragment);
+      xfree (url->user);
+      xfree (url->passwd);
 
-  xfree (url->dir);
-  xfree (url->file);
+      xfree (url->dir);
+      xfree (url->file);
 
-  xfree (url);
+      xfree (url);
+    }
 }
-
+
 /* Create all the necessary directories for PATH (a file).  Calls
    make_directory internally.  */
 int
@@ -1236,7 +1243,7 @@ mkalldirs (const char *path)
   xfree (t);
   return res;
 }
-
+
 /* Functions for constructing the file name out of URL components.  */
 
 /* A growable string structure, used by url_file_name and friends.
@@ -1245,12 +1252,16 @@ mkalldirs (const char *path)
    The idea is to have a convenient and efficient way to construct a
    string by having various functions append data to it.  Instead of
    passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the
-   functions in questions, we pass the pointer to this struct.  */
+   functions in questions, we pass the pointer to this struct.
+
+   Functions that write to the members in this struct must make sure
+   that base remains null terminated by calling append_null().
+   */
 
 struct growable {
   char *base;
-  int size;
-  int tail;
+  int size;   /* memory allocated */
+  int tail;   /* string length */
 };
 
 /* Ensure that the string can accept APPEND_COUNT more characters past
@@ -1268,28 +1279,45 @@ struct growable {
 /* Move the tail position by APPEND_COUNT characters. */
 #define TAIL_INCR(r, append_count) ((r)->tail += append_count)
 
-/* Append the string STR to DEST.  NOTICE: the string in DEST is not
-   terminated.  */
 
+/* Append NULL to DEST. */
+static void
+append_null (struct growable *dest)
+{
+  GROW (dest, 1);
+  *TAIL (dest) = 0;
+}
+
+/* Append CH to DEST. */
+static void
+append_char (char ch, struct growable *dest)
+{
+  if (ch)
+    {
+      GROW (dest, 1);
+      *TAIL (dest) = ch;
+      TAIL_INCR (dest, 1);
+    }
+
+  append_null (dest);
+}
+
+/* Append the string STR to DEST. */
 static void
 append_string (const char *str, struct growable *dest)
 {
   int l = strlen (str);
-  GROW (dest, l);
-  memcpy (TAIL (dest), str, l);
-  TAIL_INCR (dest, l);
+
+  if (l)
+    {
+      GROW (dest, l);
+      memcpy (TAIL (dest), str, l);
+      TAIL_INCR (dest, l);
+    }
+
+  append_null (dest);
 }
 
-/* Append CH to DEST.  For example, append_char (0, DEST)
-   zero-terminates DEST.  */
-
-static void
-append_char (char ch, struct growable *dest)
-{
-  GROW (dest, 1);
-  *TAIL (dest) = ch;
-  TAIL_INCR (dest, 1);
-}
 
 enum {
   filechr_not_unix    = 1,      /* unusable on Unix, / and \0 */
@@ -1361,6 +1389,7 @@ UWC,  C,  C,  C,   C,  C,  C,  C,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
    query, normally '?'.  Since Windows cannot handle '?' as part of
    file name, we use '@' instead there.  */
 #define FN_QUERY_SEP (opt.restrict_files_os != restrict_windows ? '?' : '@')
+#define FN_QUERY_SEP_STR (opt.restrict_files_os != restrict_windows ? "?" : "@")
 
 /* Quote path element, characters in [b, e), as file name, and append
    the quoted string to DEST.  Each character is quoted as per
@@ -1454,6 +1483,7 @@ append_uri_pathel (const char *b, const char *e, bool escaped,
     }
 
   TAIL_INCR (dest, outlen);
+  append_null (dest);
 }
 
 /* Append to DEST the directory structure that corresponds the
@@ -1494,22 +1524,28 @@ append_dir_structure (const struct url *u, struct growable *dest)
     }
 }
 
-/* Return a unique file name that matches the given URL as good as
+/* Return a unique file name that matches the given URL as well as
    possible.  Does not create directories on the file system.  */
 
 char *
 url_file_name (const struct url *u, char *replaced_filename)
 {
   struct growable fnres;        /* stands for "file name result" */
+  struct growable temp_fnres;
 
-  const char *u_file, *u_query;
-  char *fname, *unique;
-  char *index_filename = "index.html"; /* The default index file is index.html */
+  const char *u_file;
+  char *fname, *unique, *fname_len_check;
+  const char *index_filename = "index.html"; /* The default index file is index.html */
+  size_t max_length;
 
   fnres.base = NULL;
   fnres.size = 0;
   fnres.tail = 0;
 
+  temp_fnres.base = NULL;
+  temp_fnres.size = 0;
+  temp_fnres.tail = 0;
+
   /* If an alternative index file was defined, change index_filename */
   if (opt.default_page)
     index_filename = opt.default_page;
@@ -1555,44 +1591,87 @@ url_file_name (const struct url *u, char *replaced_filename)
 
   if (!replaced_filename)
     {
-      /* Add the file name. */
-      if (fnres.tail)
-	append_char ('/', &fnres);
+      /* Create the filename. */
       u_file = *u->file ? u->file : index_filename;
-      append_uri_pathel (u_file, u_file + strlen (u_file), false, &fnres);
 
-      /* Append "?query" to the file name. */
-      u_query = u->query && *u->query ? u->query : NULL;
-      if (u_query)
-	{
-	  append_char (FN_QUERY_SEP, &fnres);
-	  append_uri_pathel (u_query, u_query + strlen (u_query),
-			     true, &fnres);
-	}
+      /* Append "?query" to the file name, even if empty,
+       * and create fname_len_check. */
+      if (u->query)
+        fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);
+      else
+        fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
     }
   else
     {
-      if (fnres.tail)
-	append_char ('/', &fnres);
       u_file = replaced_filename;
-      append_uri_pathel (u_file, u_file + strlen (u_file), false, &fnres);
+      fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
     }
 
-  /* Zero-terminate the file name. */
-  append_char ('\0', &fnres);
+  append_uri_pathel (fname_len_check,
+    fname_len_check + strlen (fname_len_check), false, &temp_fnres);
+
+  /* Zero-terminate the temporary file name. */
+  append_char ('\0', &temp_fnres);
+
+  /* Check that the length of the file name is acceptable. */
+#ifdef WINDOWS
+  if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
+    {
+      max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2);
+      /* FIXME: In Windows a filename is usually limited to 255 characters.
+      To really be accurate you could call GetVolumeInformation() to get
+      lpMaximumComponentLength
+      */
+      if (max_length > 255)
+        {
+          max_length = 255;
+        }
+    }
+  else
+    {
+      max_length = 0;
+    }
+#else
+  max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER;
+#endif
+  if (max_length > 0 && strlen (temp_fnres.base) > max_length)
+    {
+      logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n",
+          (unsigned long) strlen (temp_fnres.base));
+      logprintf (LOG_NOTQUIET, "Trying to shorten...\n");
+
+      /* Shorten the file name. */
+      temp_fnres.base[max_length] = '\0';
+
+      logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
+    }
+
+  xfree (fname_len_check);
+
+  /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
+   * just append it. */
+  if (fnres.tail)
+    append_char ('/', &fnres);
+  append_string (temp_fnres.base, &fnres);
 
   fname = fnres.base;
 
+  /* Make a final check that the path length is acceptable? */
+  /* TODO: check fnres.base for path length problem */
+
+  xfree (temp_fnres.base);
+
   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
      2) Retrieval with regetting.
      3) Timestamping is used.
      4) Hierarchy is built.
+     5) Backups are specified.
 
      The exception is the case when file does exist and is a
      directory (see `mkalldirs' for explanation).  */
 
-  if ((opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct)
+  if (ALLOW_CLOBBER
       && !(file_exists_p (fname) && !file_non_directory_p (fname)))
     {
       unique = fname;
@@ -1620,7 +1699,7 @@ url_file_name (const struct url *u, char *replaced_filename)
 
   return unique;
 }
-
+
 /* Resolve "." and ".." elements of PATH by destructively modifying
    PATH and return true if PATH has been modified, false otherwise.
 
@@ -1706,7 +1785,7 @@ path_simplify (enum url_scheme scheme, char *path)
 
   return t != h;
 }
-
+
 /* Return the length of URL's path.  Path is considered to be
    terminated by one or more of the ?query or ;params or #fragment,
    depending on the scheme.  */
@@ -1917,7 +1996,7 @@ uri_merge (const char *base, const char *link)
 
   return merge;
 }
-
+
 #define APPEND(p, s) do {                       \
   int len = strlen (s);                         \
   memcpy (p, s, len);                           \
@@ -1961,7 +2040,7 @@ url_string (const struct url *url, enum url_auth_mode auth_mode)
           if (url->passwd)
             {
               if (auth_mode == URL_AUTH_HIDE_PASSWD)
-                quoted_passwd = HIDDEN_PASSWORD;
+                quoted_passwd = (char *) HIDDEN_PASSWORD;
               else
                 quoted_passwd = url_escape_allow_passthrough (url->passwd);
             }
@@ -2034,7 +2113,7 @@ url_string (const struct url *url, enum url_auth_mode auth_mode)
 
   return result;
 }
-
+
 /* Return true if scheme a is similar to scheme b.
 
    Schemes are similar if they are equal.  If SSL is supported, schemes
@@ -2052,7 +2131,7 @@ schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
 #endif
   return false;
 }
-
+
 static int
 getchar_from_escaped_string (const char *str, char *c)
 {
@@ -2113,7 +2192,7 @@ are_urls_equal (const char *u1, const char *u2)
 
   return (*p == 0 && *q == 0 ? true : false);
 }
-
+
 #ifdef TESTING
 /* Debugging and testing support for path_simplify. */
 
@@ -2130,7 +2209,7 @@ ps (char *path)
 #endif
 
 static const char *
-run_test (char *test, char *expected_result, enum url_scheme scheme,
+run_test (const char *test, const char *expected_result, enum url_scheme scheme,
           bool expected_change)
 {
   char *test_copy = xstrdup (test);
@@ -2159,8 +2238,8 @@ run_test (char *test, char *expected_result, enum url_scheme scheme,
 const char *
 test_path_simplify (void)
 {
-  static struct {
-    char *test, *result;
+  static const struct {
+    const char *test, *result;
     enum url_scheme scheme;
     bool should_modify;
   } tests[] = {
@@ -2192,15 +2271,16 @@ test_path_simplify (void)
     { "a/b/../../c",            "c",            SCHEME_HTTP, true },
     { "./a/../b",               "b",            SCHEME_HTTP, true }
   };
-  int i;
+  unsigned i;
 
   for (i = 0; i < countof (tests); i++)
     {
       const char *message;
-      char *test = tests[i].test;
-      char *expected_result = tests[i].result;
+      const char *test = tests[i].test;
+      const char *expected_result = tests[i].result;
       enum url_scheme scheme = tests[i].scheme;
       bool  expected_change = tests[i].should_modify;
+
       message = run_test (test, expected_result, scheme, expected_change);
       if (message) return message;
     }
@@ -2208,19 +2288,19 @@ test_path_simplify (void)
 }
 
 const char *
-test_append_uri_pathel()
+test_append_uri_pathel(void)
 {
-  int i;
-  struct {
-    char *original_url;
-    char *input;
+  unsigned i;
+  static const struct {
+    const char *original_url;
+    const char *input;
     bool escaped;
-    char *expected_result;
+    const char *expected_result;
   } test_array[] = {
     { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       struct growable dest;
       const char *p = test_array[i].input;
@@ -2229,7 +2309,6 @@ test_append_uri_pathel()
 
       append_string (test_array[i].original_url, &dest);
       append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);
-      append_char ('\0', &dest);
 
       mu_assert ("test_append_uri_pathel: wrong result",
                  strcmp (dest.base, test_array[i].expected_result) == 0);
@@ -2238,13 +2317,13 @@ test_append_uri_pathel()
   return NULL;
 }
 
-const char*
-test_are_urls_equal()
+const char *
+test_are_urls_equal(void)
 {
-  int i;
-  struct {
-    char *url1;
-    char *url2;
+  unsigned i;
+  static const struct {
+    const char *url1;
+    const char *url2;
     bool expected_result;
   } test_array[] = {
     { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/",       true },
@@ -2255,7 +2334,7 @@ test_are_urls_equal()
     { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/",       false },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       mu_assert ("test_are_urls_equal: wrong result",
                  are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);
@@ -2269,4 +2348,3 @@ test_are_urls_equal()
 /*
  * vim: et ts=2 sw=2
  */
-

src/url.h

@@ -37,6 +37,22 @@ as that of the covered work.  */
 #define DEFAULT_FTP_PORT 21
 #define DEFAULT_HTTPS_PORT 443
 
+/* This represents how many characters less than the OS max name length a file
+ * should be.  More precisely, a file name should be at most
+ * (NAME_MAX - CHOMP_BUFFER) characters in length.  This number was arrived at
+ * by adding the lengths of all possible strings that could be appended to a
+ * file name later in the code (e.g. ".orig", ".html", etc.).  This is
+ * hopefully plenty of extra characters, but I am not guaranteeing that a file
+ * name will be of the proper length by the time the code wants to open a
+ * file descriptor. */
+#define CHOMP_BUFFER 19
+
+/* The flags that allow clobbering the file (opening with "wb").
+   Defined here to avoid repetition later.  #### This will require
+   rework.  */
+#define ALLOW_CLOBBER (opt.noclobber || opt.always_rest || opt.timestamping \
+                  || opt.dirstruct || opt.output_document || opt.backups > 0)
+
 /* Specifies how, or whether, user auth information should be included
  * in URLs regenerated from URL parse structures. */
 enum url_auth_mode {
@@ -60,11 +76,11 @@ enum url_scheme {
 /* Structure containing info on a URL.  */
 struct url
 {
-  char *url;			/* Original URL */
-  enum url_scheme scheme;	/* URL scheme */
+  char *url;                /* Original URL */
+  enum url_scheme scheme;   /* URL scheme */
 
-  char *host;			/* Extracted hostname */
-  int port;			/* Port number */
+  char *host;               /* Extracted hostname */
+  int port;                 /* Port number */
 
   /* URL components (URL-quoted). */
   char *path;
@@ -85,6 +101,7 @@ struct url
 
 char *url_escape (const char *);
 char *url_escape_unsafe_and_reserved (const char *);
+void url_unescape (char *);
 
 struct url *url_parse (const char *, int *, struct iri *iri, bool percent_encode);
 char *url_error (const char *, int);

src/utils.c

@@ -42,29 +42,40 @@ as that of the covered work.  */
 #ifdef HAVE_PROCESS_H
 # include <process.h>  /* getpid() */
 #endif
-#ifdef HAVE_UTIME_H
-# include <utime.h>
-#endif
 #include <errno.h>
 #include <fcntl.h>
 #include <assert.h>
 #include <stdarg.h>
 #include <locale.h>
 
+#if HAVE_UTIME
+# include <sys/types.h>
+# ifdef HAVE_UTIME_H
+#  include <utime.h>
+# endif
+
+# ifdef HAVE_SYS_UTIME_H
+#  include <sys/utime.h>
+# endif
+#endif
+
+#include <sys/time.h>
+
 #include <sys/stat.h>
 
 /* For TIOCGWINSZ and friends: */
-#ifdef HAVE_SYS_IOCTL_H
-# include <sys/ioctl.h>
-#endif
-#ifdef HAVE_TERMIOS_H
-# include <termios.h>
-#endif
+#include <sys/ioctl.h>
+#include <termios.h>
 
 /* Needed for Unix version of run_with_timeout. */
 #include <signal.h>
 #include <setjmp.h>
 
+#include <regex.h>
+#ifdef HAVE_LIBPCRE
+# include <pcre.h>
+#endif
+
 #ifndef HAVE_SIGSETJMP
 /* If sigsetjmp is a macro, configure won't pick it up. */
 # ifdef sigsetjmp
@@ -87,7 +98,10 @@ as that of the covered work.  */
 #include "test.h"
 #endif
 
-static void
+#include "exits.h"
+#include "c-strcase.h"
+
+static void _Noreturn
 memfatal (const char *context, long attempted_size)
 {
   /* Make sure we don't try to store part of the log line, and thus
@@ -110,7 +124,7 @@ memfatal (const char *context, long attempted_size)
                  exec_name, context, attempted_size);
     }
 
-  exit (1);
+  exit (WGET_EXIT_GENERIC_ERROR);
 }
 
 /* Character property table for (re-)escaping VMS ODS5 extended file
@@ -254,7 +268,7 @@ sepstring (const char *s)
   res[i + 1] = NULL;
   return res;
 }
-
+
 /* Like sprintf, but prints into a string of sufficient size freshly
    allocated with malloc, which is returned.  If unable to print due
    to invalid format, returns NULL.  Inability to allocate needed
@@ -336,6 +350,32 @@ aprintf (const char *fmt, ...)
 #endif /* not HAVE_VASPRINTF */
 }
 
+#ifndef HAVE_STRLCPY
+/* strlcpy() is a BSD function that sometimes is really handy.
+ * It is the same as snprintf(dst,dstsize,"%s",src), but much faster. */
+
+size_t
+strlcpy (char *dst, const char *src, size_t size)
+{
+  const char *old = src;
+
+  /* Copy as many bytes as will fit */
+  if (size)
+    {
+      while (--size)
+        {
+          if (!(*dst++ = *src++))
+            return src - old - 1;
+        }
+
+      *dst = 0;
+    }
+
+  while (*src++);
+  return src - old - 1;
+}
+#endif
+
 /* Concatenate the NULL-terminated list of string arguments into
    freshly allocated space.  */
 
@@ -343,47 +383,30 @@ char *
 concat_strings (const char *str0, ...)
 {
   va_list args;
-  int saved_lengths[5];         /* inspired by Apache's apr_pstrcat */
-  char *ret, *p;
+  const char *arg;
+  size_t length = 0, pos = 0;
+  char *s;
 
-  const char *next_str;
-  int total_length = 0;
-  size_t argcount;
+  if (!str0)
+    return NULL;
 
-  /* Calculate the length of and allocate the resulting string. */
-
-  argcount = 0;
+  /* calculate the length of the resulting string */
   va_start (args, str0);
-  for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *))
-    {
-      int len = strlen (next_str);
-      if (argcount < countof (saved_lengths))
-        saved_lengths[argcount++] = len;
-      total_length += len;
-    }
+  for (arg = str0; arg; arg = va_arg (args, const char *))
+    length += strlen(arg);
   va_end (args);
-  p = ret = xmalloc (total_length + 1);
 
-  /* Copy the strings into the allocated space. */
+  s = xmalloc (length + 1);
 
-  argcount = 0;
+  /* concatenate strings */
   va_start (args, str0);
-  for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *))
-    {
-      int len;
-      if (argcount < countof (saved_lengths))
-        len = saved_lengths[argcount++];
-      else
-        len = strlen (next_str);
-      memcpy (p, next_str, len);
-      p += len;
-    }
+  for (arg = str0; arg; arg = va_arg (args, const char *))
+    pos += strlcpy(s + pos, arg, length - pos + 1);
   va_end (args);
-  *p = '\0';
 
-  return ret;
+  return s;
 }
-
+
 /* Format the provided time according to the specified format.  The
    format is a string with format elements supported by strftime.  */
 
@@ -417,7 +440,7 @@ datetime_str (time_t t)
 {
   return fmttime(t, "%Y-%m-%d %H:%M:%S");
 }
-
+
 /* The Windows versions of the following two functions are defined in
    mswindows.c. On MSDOS this function should never be called. */
 
@@ -458,7 +481,7 @@ fork_to_background (void)
     {
       /* parent, error */
       perror ("fork");
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
   else if (pid != 0)
     {
@@ -466,20 +489,23 @@ fork_to_background (void)
       printf (_("Continuing in background, pid %d.\n"), (int) pid);
       if (logfile_changed)
         printf (_("Output will be written to %s.\n"), quote (opt.lfilename));
-      exit (0);                 /* #### should we use _exit()? */
+      exit (WGET_EXIT_SUCCESS);                 /* #### should we use _exit()? */
     }
 
   /* child: give up the privileges and keep running. */
   setsid ();
-  freopen ("/dev/null", "r", stdin);
-  freopen ("/dev/null", "w", stdout);
-  freopen ("/dev/null", "w", stderr);
+  if (freopen ("/dev/null", "r", stdin) == NULL)
+    DEBUGP (("Failed to redirect stdin to /dev/null.\n"));
+  if (freopen ("/dev/null", "w", stdout) == NULL)
+    DEBUGP (("Failed to redirect stdout to /dev/null.\n"));
+  if (freopen ("/dev/null", "w", stderr) == NULL)
+    DEBUGP (("Failed to redirect stderr to /dev/null.\n"));
 }
 #endif /* !WINDOWS && !MSDOS */
 
 #endif /* def __VMS [else] */
 
-
+
 /* "Touch" FILE, i.e. make its mtime ("modified time") equal the time
    specified with TM.  The atime ("access time") is set to the current
    time.  */
@@ -487,6 +513,20 @@ fork_to_background (void)
 void
 touch (const char *file, time_t tm)
 {
+#if HAVE_UTIME
+# ifdef HAVE_STRUCT_UTIMBUF
+  struct utimbuf times;
+# else
+  struct {
+    time_t actime;
+    time_t modtime;
+  } times;
+# endif
+  times.modtime = tm;
+  times.actime = time (NULL);
+  if (utime (file, &times) == -1)
+    logprintf (LOG_NOTQUIET, "utime(%s): %s\n", file, strerror (errno));
+#else
   struct timespec timespecs[2];
   int fd;
 
@@ -506,6 +546,7 @@ touch (const char *file, time_t tm)
     logprintf (LOG_NOTQUIET, "futimens(%s): %s\n", file, strerror (errno));
 
   close (fd);
+#endif
 }
 
 /* Checks if FILE is a symbolic link, and removes it if it is.  Does
@@ -675,7 +716,7 @@ unique_create (const char *name, bool binary, char **opened_name)
       xfree (uname);
       uname = unique_name (name, false);
     }
-  if (opened_name && fp != NULL)
+  if (opened_name)
     {
       if (fp)
         *opened_name = uname;
@@ -746,8 +787,7 @@ fopen_excl (const char *fname, int binary)
       open_id = 13;
       fd = open( fname,                 /* File name. */
        flags,                           /* Flags. */
-       0777,                            /* Mode for default protection.
-*/
+       0777,                            /* Mode for default protection. */
        "rfm=stmlf",                     /* Stream_LF. */
        OPEN_OPT_ARGS);                  /* Access callback. */
     }
@@ -775,7 +815,7 @@ fopen_excl (const char *fname, int binary)
   return fopen (fname, binary ? "wb" : "w");
 #endif /* not O_EXCL */
 }
-
+
 /* Create DIRECTORY.  If some of the pathname components of DIRECTORY
    are missing, create them first.  In case any mkdir() call fails,
    return its error status.  Returns 0 on successful completion.
@@ -841,7 +881,7 @@ file_merge (const char *base, const char *file)
 
   return result;
 }
-
+
 /* Like fnmatch, but performs a case-insensitive match.  */
 
 int
@@ -873,15 +913,14 @@ static bool in_acclist (const char *const *, const char *, bool);
 bool
 acceptable (const char *s)
 {
-  int l = strlen (s);
+  const char *p;
 
   if (opt.output_document && strcmp (s, opt.output_document) == 0)
     return true;
 
-  while (l && s[l] != '/')
-    --l;
-  if (s[l] == '/')
-    s += (l + 1);
+  if ((p = strrchr (s, '/')))
+    s = p + 1;
+
   if (opt.accepts)
     {
       if (opt.rejects)
@@ -892,6 +931,20 @@ acceptable (const char *s)
     }
   else if (opt.rejects)
     return !in_acclist ((const char *const *)opt.rejects, s, true);
+
+  return true;
+}
+
+/* Determine whether an URL is acceptable to be followed, according to
+   regex patterns to accept/reject.  */
+bool
+accept_url (const char *s)
+{
+  if (opt.acceptregex && !opt.regex_match_fun (opt.acceptregex, s))
+    return false;
+  if (opt.rejectregex && opt.regex_match_fun (opt.rejectregex, s))
+    return false;
+
   return true;
 }
 
@@ -917,16 +970,16 @@ subdir_p (const char *d1, const char *d2)
    first element that matches DIR, through wildcards or front comparison (as
    appropriate).  */
 static bool
-dir_matches_p (char **dirlist, const char *dir)
+dir_matches_p (const char **dirlist, const char *dir)
 {
-  char **x;
+  const char **x;
   int (*matcher) (const char *, const char *, int)
     = opt.ignore_case ? fnmatch_nocase : fnmatch;
 
   for (x = dirlist; *x; x++)
     {
       /* Remove leading '/' */
-      char *p = *x + (**x == '/');
+      const char *p = *x + (**x == '/');
       if (has_wildcards_p (p))
         {
           if (matcher (p, dir, FNM_PATHNAME) == 0)
@@ -978,29 +1031,15 @@ accdir (const char *directory)
 bool
 match_tail (const char *string, const char *tail, bool fold_case)
 {
-  int i, j;
+  int pos = strlen (string) - strlen (tail);
 
-  /* We want this to be fast, so we code two loops, one with
-     case-folding, one without. */
+  if (pos < 0)
+    return false;  /* tail is longer than string.  */
 
   if (!fold_case)
-    {
-      for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--)
-        if (string[i] != tail[j])
-          break;
-    }
+    return !strcmp (string + pos, tail);
   else
-    {
-      for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--)
-        if (c_tolower (string[i]) != c_tolower (tail[j]))
-          break;
-    }
-
-  /* If the tail was exhausted, the match was succesful.  */
-  if (j == -1)
-    return true;
-  else
-    return false;
+    return !strcasecmp (string + pos, tail);
 }
 
 /* Checks whether string S matches each element of ACCEPTS.  A list
@@ -1049,15 +1088,12 @@ in_acclist (const char *const *accepts, const char *s, bool backward)
 char *
 suffix (const char *str)
 {
-  int i;
+  char *p;
 
-  for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--)
-    ;
+  if ((p = strrchr (str, '.')) && !strchr (p + 1, '/'))
+    return p + 1;
 
-  if (str[i++] == '.')
-    return (char *)str + i;
-  else
-    return NULL;
+  return NULL;
 }
 
 /* Return true if S contains globbing wildcards (`*', `?', `[' or
@@ -1066,10 +1102,7 @@ suffix (const char *str)
 bool
 has_wildcards_p (const char *s)
 {
-  for (; *s; s++)
-    if (*s == '*' || *s == '?' || *s == '[' || *s == ']')
-      return true;
-  return false;
+  return !!strpbrk (s, "*?[]");
 }
 
 /* Return true if FNAME ends with a typical HTML suffix.  The
@@ -1089,65 +1122,15 @@ has_html_suffix_p (const char *fname)
 
   if ((suf = suffix (fname)) == NULL)
     return false;
-  if (!strcasecmp (suf, "html"))
+  if (!c_strcasecmp (suf, "html"))
     return true;
-  if (!strcasecmp (suf, "htm"))
+  if (!c_strcasecmp (suf, "htm"))
     return true;
-  if (suf[0] && !strcasecmp (suf + 1, "html"))
+  if (suf[0] && !c_strcasecmp (suf + 1, "html"))
     return true;
   return false;
 }
 
-/* Read a line from FP and return the pointer to freshly allocated
-   storage.  The storage space is obtained through malloc() and should
-   be freed with free() when it is no longer needed.
-
-   The length of the line is not limited, except by available memory.
-   The newline character at the end of line is retained.  The line is
-   terminated with a zero character.
-
-   After end-of-file is encountered without anything being read, NULL
-   is returned.  NULL is also returned on error.  To distinguish
-   between these two cases, use the stdio function ferror().  */
-
-char *
-read_whole_line (FILE *fp)
-{
-  int length = 0;
-  int bufsize = 82;
-  char *line = xmalloc (bufsize);
-
-  while (fgets (line + length, bufsize - length, fp))
-    {
-      length += strlen (line + length);
-      if (length == 0)
-        /* Possible for example when reading from a binary file where
-           a line begins with \0.  */
-        continue;
-
-      if (line[length - 1] == '\n')
-        break;
-
-      /* fgets() guarantees to read the whole line, or to use up the
-         space we've given it.  We can double the buffer
-         unconditionally.  */
-      bufsize <<= 1;
-      line = xrealloc (line, bufsize);
-    }
-  if (length == 0 || ferror (fp))
-    {
-      xfree (line);
-      return NULL;
-    }
-  if (length + 1 < bufsize)
-    /* Relieve the memory from our exponential greediness.  We say
-       `length + 1' because the terminating \0 is not included in
-       LENGTH.  We don't need to zero-terminate the string ourselves,
-       though, because fgets() does that.  */
-    line = xrealloc (line, length + 1);
-  return line;
-}
-
 /* Read FILE into memory.  A pointer to `struct file_memory' are
    returned; use struct element `content' to access file contents, and
    the element `length' to know the file length.  `content' is *not*
@@ -1289,7 +1272,7 @@ wget_read_file_free (struct file_memory *fm)
     }
   xfree (fm);
 }
-
+
 /* Free the pointers in a NULL-terminated vector of pointers, then
    free the pointer itself.  */
 void
@@ -1358,7 +1341,7 @@ vec_append (char **vec, const char *str)
   vec[cnt] = NULL;
   return vec;
 }
-
+
 /* Sometimes it's useful to create "sets" of strings, i.e. special
    hash tables where you want to store strings as keys and merely
    query for their existence.  Here is a set of utility routines that
@@ -1423,7 +1406,7 @@ free_keys_and_values (struct hash_table *ht)
       xfree (iter.value);
     }
 }
-
+
 /* Get digit grouping data for thousand separors by calling
    localeconv().  The data includes separator string and grouping info
    and is cached after the first call to the function.
@@ -1550,7 +1533,7 @@ with_thousand_seps (wgint n)
    some detail.  */
 
 char *
-human_readable (HR_NUMTYPE n)
+human_readable (HR_NUMTYPE n, const int acc, const int decimals)
 {
   /* These suffixes are compatible with those of GNU `ls -lh'. */
   static char powers[] =
@@ -1583,10 +1566,10 @@ human_readable (HR_NUMTYPE n)
       if ((n / 1024) < 1024 || i == countof (powers) - 1)
         {
           double val = n / 1024.0;
-          /* Print values smaller than 10 with one decimal digits, and
-             others without any decimals.  */
+          /* Print values smaller than the accuracy level (acc) with (decimal)
+           * decimal digits, and others without any decimals.  */
           snprintf (buf, sizeof (buf), "%.*f%c",
-                    val < 10 ? 1 : 0, val, powers[i]);
+                    val < acc ? decimals : 0, val, powers[i]);
           return buf;
         }
       n /= 1024;
@@ -1803,7 +1786,18 @@ number_to_static_string (wgint number)
   ringpos = (ringpos + 1) % RING_SIZE;
   return buf;
 }
-
+
+/* Converts the byte to bits format if --report-bps option is enabled
+ */
+wgint
+convert_to_bits (wgint num)
+{
+  if (opt.report_bps)
+    return num * 8;
+  return num;
+}
+
+
 /* Determine the width of the terminal we're running on.  If that's
    not possible, return 0.  */
 
@@ -1833,7 +1827,7 @@ determine_screen_width (void)
   return 0;
 #endif /* neither TIOCGWINSZ nor WINDOWS */
 }
-
+
 /* Whether the rnd system (either rand or [dl]rand48) has been
    seeded.  */
 static int rnd_seeded;
@@ -1853,7 +1847,14 @@ static int rnd_seeded;
 int
 random_number (int max)
 {
-#ifdef HAVE_DRAND48
+#ifdef HAVE_RANDOM
+  if (!rnd_seeded)
+    {
+      srandom ((long) time (NULL) ^ (long) getpid ());
+      rnd_seeded = 1;
+    }
+  return random () % max;
+#elif defined HAVE_DRAND48
   if (!rnd_seeded)
     {
       srand48 ((long) time (NULL) ^ (long) getpid ());
@@ -1888,7 +1889,9 @@ random_number (int max)
 double
 random_float (void)
 {
-#ifdef HAVE_DRAND48
+#ifdef HAVE_RANDOM
+    return ((double) random_number (RAND_MAX)) / RAND_MAX;
+#elif defined HAVE_DRAND48
   if (!rnd_seeded)
     {
       srand48 ((long) time (NULL) ^ (long) getpid ());
@@ -1902,7 +1905,7 @@ random_float (void)
           + random_number (10000) / (10000.0 * 10000.0 * 10000.0 * 10000.0));
 #endif /* not HAVE_DRAND48 */
 }
-
+
 /* Implementation of run_with_timeout, a generic timeout-forcing
    routine for systems with Unix-like signal handling.  */
 
@@ -1912,8 +1915,8 @@ random_float (void)
 
 static sigjmp_buf run_with_timeout_env;
 
-static void
-abort_run_with_timeout (int sig)
+static void _Noreturn
+abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
   siglongjmp (run_with_timeout_env, -1);
@@ -1923,8 +1926,8 @@ abort_run_with_timeout (int sig)
 
 static jmp_buf run_with_timeout_env;
 
-static void
-abort_run_with_timeout (int sig)
+static void _Noreturn
+abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
   /* We don't have siglongjmp to preserve the set of blocked signals;
@@ -2061,7 +2064,7 @@ run_with_timeout (double timeout, void (*fun) (void *), void *arg)
 }
 #endif /* not WINDOWS */
 #endif /* not USE_SIGNAL_TIMEOUT */
-
+
 #ifndef WINDOWS
 
 /* Sleep the specified amount of seconds.  On machines without
@@ -2121,8 +2124,8 @@ xsleep (double seconds)
    This implementation does not emit newlines after 76 characters of
    base64 data.  */
 
-int
-base64_encode (const void *data, int length, char *dest)
+size_t
+base64_encode (const void *data, size_t length, char *dest)
 {
   /* Conversion table.  */
   static const char tbl[64] = {
@@ -2184,12 +2187,12 @@ base64_encode (const void *data, int length, char *dest)
 
    Since DEST is assumed to contain binary data, it is not
    NUL-terminated.  The function returns the length of the data
-   written to TO.  -1 is returned in case of error caused by malformed
+   written to "TO".  -1 is returned in case of error caused by malformed
    base64 input.
 
    This function originates from Free Recode.  */
 
-int
+ssize_t
 base64_decode (const char *base64, void *dest)
 {
   /* Table of base64 values for first 128 characters.  Note that this
@@ -2276,9 +2279,93 @@ base64_decode (const char *base64, void *dest)
   return q - (char *) dest;
 }
 
+#ifdef HAVE_LIBPCRE
+/* Compiles the PCRE regex. */
+void *
+compile_pcre_regex (const char *str)
+{
+  const char *errbuf;
+  int erroffset;
+  pcre *regex = pcre_compile (str, 0, &errbuf, &erroffset, 0);
+  if (! regex)
+    {
+      fprintf (stderr, _("Invalid regular expression %s, %s\n"),
+               quote (str), errbuf);
+      return false;
+    }
+  return regex;
+}
+#endif
+
+/* Compiles the POSIX regex. */
+void *
+compile_posix_regex (const char *str)
+{
+  regex_t *regex = xmalloc (sizeof (regex_t));
+  int errcode = regcomp ((regex_t *) regex, str, REG_EXTENDED | REG_NOSUB);
+  if (errcode != 0)
+    {
+      size_t errbuf_size = regerror (errcode, (regex_t *) regex, NULL, 0);
+      char *errbuf = xmalloc (errbuf_size);
+      regerror (errcode, (regex_t *) regex, errbuf, errbuf_size);
+      fprintf (stderr, _("Invalid regular expression %s, %s\n"),
+               quote (str), errbuf);
+      xfree (errbuf);
+      xfree (regex);
+      return NULL;
+    }
+
+  return regex;
+}
+
+#ifdef HAVE_LIBPCRE
+#define OVECCOUNT 30
+/* Matches a PCRE regex.  */
+bool
+match_pcre_regex (const void *regex, const char *str)
+{
+  size_t l = strlen (str);
+  int ovector[OVECCOUNT];
+
+  int rc = pcre_exec ((pcre *) regex, 0, str, (int) l, 0, 0, ovector, OVECCOUNT);
+  if (rc == PCRE_ERROR_NOMATCH)
+    return false;
+  else if (rc < 0)
+    {
+      logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"),
+                 quote (str), rc);
+      return false;
+    }
+  else
+    return true;
+}
+#undef OVECCOUNT
+#endif
+
+/* Matches a POSIX regex.  */
+bool
+match_posix_regex (const void *regex, const char *str)
+{
+  int rc = regexec ((regex_t *) regex, str, 0, NULL, 0);
+  if (rc == REG_NOMATCH)
+    return false;
+  else if (rc == 0)
+    return true;
+  else
+    {
+      size_t errbuf_size = regerror (rc, opt.acceptregex, NULL, 0);
+      char *errbuf = xmalloc (errbuf_size);
+      regerror (rc, opt.acceptregex, errbuf, errbuf_size);
+      logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"),
+                 quote (str), rc);
+      xfree (errbuf);
+      return false;
+    }
+}
+
 #undef IS_ASCII
 #undef NEXT_CHAR
-
+
 /* Simple merge sort for use by stable_sort.  Implementation courtesy
    Zeljko Vrba with additional debugging by Nenad Barbutov.  */
 
@@ -2324,7 +2411,7 @@ stable_sort (void *base, size_t nmemb, size_t size,
       mergesort_internal (base, temp, size, 0, nmemb - 1, cmpfun);
     }
 }
-
+
 /* Print a decimal number.  If it is equal to or larger than ten, the
    number is rounded.  Otherwise it is printed with one significant
    digit without trailing zeros and with no more than three fractional
@@ -2360,21 +2447,75 @@ print_decimal (double number)
   return buf;
 }
 
+/* Get the maximum name length for the given path. */
+/* Return 0 if length is unknown. */
+long
+get_max_length (const char *path, int length, int name)
+{
+  long ret;
+  char *p, *d;
+
+  /* Make a copy of the path that we can modify. */
+  p = path ? strdupdelim (path, path + length) : strdup ("");
+
+  for (;;)
+    {
+      errno = 0;
+      /* For an empty path query the current directory. */
+#if HAVE_PATHCONF
+      ret = pathconf (*p ? p : ".", name);
+      if (!(ret < 0 && errno == ENOENT))
+        break;
+#else
+      ret = PATH_MAX;
+#endif
+
+      /* The path does not exist yet, but may be created. */
+      /* Already at current or root directory, give up. */
+      if (!*p || strcmp (p, "/") == 0)
+        break;
+
+      /* Remove one directory level and try again. */
+      d = strrchr (p, '/');
+      if (d == p)
+        p[1] = '\0';  /* check root directory */
+      else if (d)
+        *d = '\0';  /* remove last directory part */
+      else
+        *p = '\0';  /* check current directory */
+    }
+
+  xfree (p);
+
+  if (ret < 0)
+    {
+      /* pathconf() has a message for us. */
+      if (errno != 0)
+          perror ("pathconf");
+
+      /* If (errno == 0) then there is no max length.
+         Even on error return 0 so the caller can continue. */
+      return 0;
+    }
+
+  return ret;
+}
+
 #ifdef TESTING
 
 const char *
-test_subdir_p()
+test_subdir_p(void)
 {
-  int i;
-  struct {
-    char *d1;
-    char *d2;
+  static const struct {
+    const char *d1;
+    const char *d2;
     bool result;
   } test_array[] = {
     { "/somedir", "/somedir", true },
     { "/somedir", "/somedir/d2", true },
     { "/somedir/d1", "/somedir", false },
   };
+  unsigned i;
 
   for (i = 0; i < countof(test_array); ++i)
     {
@@ -2388,12 +2529,11 @@ test_subdir_p()
 }
 
 const char *
-test_dir_matches_p()
+test_dir_matches_p(void)
 {
-  int i;
-  struct {
-    char *dirlist[3];
-    char *dir;
+  static struct {
+    const char *dirlist[3];
+    const char *dir;
     bool result;
   } test_array[] = {
     { { "/somedir", "/someotherdir", NULL }, "somedir", true },
@@ -2412,6 +2552,7 @@ test_dir_matches_p()
     { { "/Tmp/has", NULL, NULL }, "/Tmp/has space", false },
     { { "/Tmp/has", NULL, NULL }, "/Tmp/has,comma", false },
   };
+  unsigned i;
 
   for (i = 0; i < countof(test_array); ++i)
     {
@@ -2425,4 +2566,3 @@ test_dir_matches_p()
 }
 
 #endif /* TESTING */
-

src/utils.h

@@ -47,12 +47,7 @@ as that of the covered work.  */
 
 #define alloca_array(type, size) ((type *) alloca ((size) * sizeof (type)))
 
-#define xfree free
-/* Free P if it is non-NULL.  C requires free() to behaves this way by
-   default, but Wget's code is historically careful not to pass NULL
-   to free.  This allows us to assert p!=NULL in xfree to check
-   additional errors.  (But we currently don't do that!)  */
-#define xfree_null(p) if (!(p)) ; else xfree (p)
+#define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
 
 struct hash_table;
 
@@ -90,6 +85,7 @@ char *file_merge (const char *, const char *);
 
 int fnmatch_nocase (const char *, const char *, int);
 bool acceptable (const char *);
+bool accept_url (const char *);
 bool accdir (const char *s);
 char *suffix (const char *s);
 bool match_tail (const char *, const char *, bool);
@@ -97,7 +93,6 @@ bool has_wildcards_p (const char *);
 
 bool has_html_suffix_p (const char *);
 
-char *read_whole_line (FILE *);
 struct file_memory *wget_read_file (const char *);
 void wget_read_file_free (struct file_memory *);
 
@@ -121,12 +116,13 @@ const char *with_thousand_seps (wgint);
 #else
 # define HR_NUMTYPE double
 #endif
-char *human_readable (HR_NUMTYPE);
+char *human_readable (HR_NUMTYPE, const int, const int);
 
 
 int numdigit (wgint);
 char *number_to_string (char *, wgint);
 char *number_to_static_string (wgint);
+wgint convert_to_bits (wgint);
 
 int determine_screen_width (void);
 int random_number (int);
@@ -138,13 +134,27 @@ void xsleep (double);
 /* How many bytes it will take to store LEN bytes in base64.  */
 #define BASE64_LENGTH(len) (4 * (((len) + 2) / 3))
 
-int base64_encode (const void *, int, char *);
-int base64_decode (const char *, void *);
+size_t base64_encode (const void *, size_t, char *);
+ssize_t base64_decode (const char *, void *);
+
+#ifdef HAVE_LIBPCRE
+void *compile_pcre_regex (const char *);
+bool match_pcre_regex (const void *, const char *);
+#endif
+
+void *compile_posix_regex (const char *);
+bool match_posix_regex (const void *, const char *);
 
 void stable_sort (void *, size_t, size_t, int (*) (const void *, const void *));
 
 const char *print_decimal (double);
 
+long get_max_length (const char *path, int length, int name);
+
+#ifndef HAVE_STRLCPY
+size_t strlcpy (char *dst, const char *src, size_t size);
+#endif
+
 extern unsigned char char_prop[];
 
 #endif /* UTILS_H */

src/version.h

@@ -0,0 +1,36 @@
+/* Extern declarations for printing version information
+   Copyright (C) 2013 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+/* Extern declarations for strings in version.c */
+extern const char *version_string;
+extern const char *compilation_string;
+extern const char *link_string;
+
+/* Extern declaration for string in build_info.c */
+extern const char *compiled_features[];

src/vms.c

@@ -45,7 +45,7 @@
  *
  *   Simplifies a fancy URL-derived file name into an ODS2- or
  *   ODS5-compatible file name.
- *    
+ *
  *----------------------------------------------------------------------
  *
  *       utime()
@@ -299,7 +299,7 @@ int dmy_lib$initialize = (int) lib$initialize;
 /*--------------------------------------------------------------------*/
 
 /* vms_arch()
-  
+
       Returns (run-time) VMS architecture string.
 */
 
@@ -556,12 +556,12 @@ char *vms_getpass( const char *prompt)
         fclose( sdc);
     }
     return ret;
-}    
+}
 
 /*--------------------------------------------------------------------*/
 
 /* vms_vers()
-  
+
       Returns (run-time) VMS version string.
 */
 
@@ -841,12 +841,11 @@ else
 
 /* Action routine for decc$to_vms(), in utime(). */
 
-char vms_path[ NAMX$C_MAXRSS+ 1];
+char *vms_path;
 
 int set_vms_name( char *name, int type)
 {
-   strncpy( vms_path, name, NAMX$C_MAXRSS);
-   vms_path[ NAMX$C_MAXRSS] = '\0';
+   vms_path = strdup(name);
    return 1;
 }
 
@@ -1337,7 +1336,7 @@ localtime_r (t, tp)
   return tp;
 }
 
-#endif	/* __CRTL_VER < 70000000 */
+#endif  /* __CRTL_VER < 70000000 */
 
 /*--------------------------------------------------------------------*/
 
@@ -1350,4 +1349,3 @@ const char *link_string = NULL;
 const char *version_string = VERSION;
 
 /*--------------------------------------------------------------------*/
-

src/warc.h

@@ -0,0 +1,27 @@
+/* Declarations of WARC helper methods. */
+#ifndef WARC_H
+#define WARC_H
+
+#include "host.h"
+
+void warc_init (void);
+void warc_close (void);
+void warc_uuid_str (char *id_str);
+
+char * warc_timestamp (char *timestamp, size_t timestamp_size);
+
+FILE * warc_tempfile (void);
+
+bool warc_write_request_record (char *url, char *timestamp_str,
+  char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset);
+bool warc_write_response_record (char *url, char *timestamp_str,
+  char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset,
+  char *mime_type, int response_code, char *redirect_location);
+bool warc_write_resource_record (char *resource_uuid, const char *url,
+  const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip,
+  const char *content_type, FILE *body, off_t payload_offset);
+bool warc_write_metadata_record (char *record_uuid, const char *url,
+  const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip,
+  const char *content_type, FILE *body, off_t payload_offset);
+
+#endif /* WARC_H */

src/wget.h

@@ -58,8 +58,13 @@ as that of the covered work.  */
 
 /* `gettext (FOO)' is long to write, so we use `_(FOO)'.  If NLS is
    unavailable, _(STRING) simply returns STRING.  */
-#include "gettext.h"
-#define _(string)   gettext (string)
+#if ENABLE_NLS
+#	include <libintl.h>
+#	define _(STRING) gettext(STRING)
+#else
+#	define _(STRING) STRING
+#	define ngettext(STRING1,STRING2,N) STRING2
+#endif
 
 /* A pseudo function call that serves as a marker for the automated
    extraction of messages, but does not call gettext().  The run-time
@@ -72,12 +77,6 @@ as that of the covered work.  */
    variables.  -- explanation partly taken from GNU make.  */
 #define N_(string) string
 
-#if ! ENABLE_NLS
-# undef HAVE_WCHAR_H
-# undef HAVE_WCWIDTH
-# undef HAVE_MBTOWC
-#endif /* not ENABLE_NLS */
-
 #if HAVE_WCWIDTH && HAVE_MBTOWC
 # define USE_NLS_PROGRESS_BAR 1
 #else
@@ -254,25 +253,25 @@ typedef double SUM_SIZE_INT;
 /* Copy the data delimited with BEG and END to alloca-allocated
    storage, and zero-terminate it.  Arguments are evaluated only once,
    in the order BEG, END, PLACE.  */
-#define BOUNDED_TO_ALLOCA(beg, end, place) do {	\
-  const char *BTA_beg = (beg);			\
-  int BTA_len = (end) - BTA_beg;		\
-  char **BTA_dest = &(place);			\
-  *BTA_dest = alloca (BTA_len + 1);		\
-  memcpy (*BTA_dest, BTA_beg, BTA_len);		\
-  (*BTA_dest)[BTA_len] = '\0';			\
+#define BOUNDED_TO_ALLOCA(beg, end, place) do { \
+  const char *BTA_beg = (beg);                  \
+  int BTA_len = (end) - BTA_beg;                \
+  char **BTA_dest = &(place);                   \
+  *BTA_dest = alloca (BTA_len + 1);             \
+  memcpy (*BTA_dest, BTA_beg, BTA_len);         \
+  (*BTA_dest)[BTA_len] = '\0';                  \
 } while (0)
 
 /* Return non-zero if string bounded between BEG and END is equal to
    STRING_LITERAL.  The comparison is case-sensitive.  */
-#define BOUNDED_EQUAL(beg, end, string_literal)				\
-  ((end) - (beg) == sizeof (string_literal) - 1				\
+#define BOUNDED_EQUAL(beg, end, string_literal)             \
+  ((end) - (beg) == sizeof (string_literal) - 1             \
    && !memcmp (beg, string_literal, sizeof (string_literal) - 1))
 
 /* The same as above, except the comparison is case-insensitive. */
-#define BOUNDED_EQUAL_NO_CASE(beg, end, string_literal)			\
-  ((end) - (beg) == sizeof (string_literal) - 1				\
-   && !strncasecmp (beg, string_literal, sizeof (string_literal) - 1))
+#define BOUNDED_EQUAL_NO_CASE(beg, end, string_literal)         \
+  ((end) - (beg) == sizeof (string_literal) - 1                 \
+   && !c_strncasecmp (beg, string_literal, sizeof (string_literal) - 1))
 
 /* Like ptr=strdup(str), but allocates the space for PTR on the stack.
    This cannot be an expression because this is not portable:
@@ -280,11 +279,11 @@ typedef double SUM_SIZE_INT;
    The problem is that some compilers can't handle alloca() being an
    argument to a function.  */
 
-#define STRDUP_ALLOCA(ptr, str) do {			\
-  char **SA_dest = &(ptr);				\
-  const char *SA_src = (str);				\
-  *SA_dest = (char *)alloca (strlen (SA_src) + 1);	\
-  strcpy (*SA_dest, SA_src);				\
+#define STRDUP_ALLOCA(ptr, str) do {                \
+  char **SA_dest = &(ptr);                          \
+  const char *SA_src = (str);                       \
+  *SA_dest = (char *)alloca (strlen (SA_src) + 1);  \
+  strcpy (*SA_dest, SA_src);                        \
 } while (0)
 
 /* Generally useful if you want to avoid arbitrary size limits but
@@ -295,17 +294,17 @@ typedef double SUM_SIZE_INT;
    NEEDED_SIZE objects.  The reallocing is done by doubling, which
    ensures constant amortized time per element.  */
 
-#define DO_REALLOC(basevar, sizevar, needed_size, type)	do {		\
-  long DR_needed_size = (needed_size);					\
-  long DR_newsize = 0;							\
-  while ((sizevar) < (DR_needed_size)) {				\
-    DR_newsize = sizevar << 1;						\
-    if (DR_newsize < 16)						\
-      DR_newsize = 16;							\
-    (sizevar) = DR_newsize;						\
-  }									\
-  if (DR_newsize)							\
-    basevar = xrealloc (basevar, DR_newsize * sizeof (type));		\
+#define DO_REALLOC(basevar, sizevar, needed_size, type) do {    \
+  long DR_needed_size = (needed_size);                          \
+  long DR_newsize = 0;                                          \
+  while ((sizevar) < (DR_needed_size)) {                        \
+    DR_newsize = sizevar << 1;                                  \
+    if (DR_newsize < 16)                                        \
+      DR_newsize = 16;                                          \
+    (sizevar) = DR_newsize;                                     \
+  }                                                             \
+  if (DR_newsize)                                               \
+    basevar = xrealloc (basevar, DR_newsize * sizeof (type));   \
 } while (0)
 
 /* Used to print pointers (usually for debugging).  Print pointers
@@ -314,19 +313,30 @@ typedef double SUM_SIZE_INT;
    0-pad the address.)  */
 #define PTR_FORMAT(p) (int) (2 * sizeof (void *)), (unsigned long) (p)
 
+/* Find the maximum buffer length needed to print an integer of type `x'
+   in base 10. 24082 / 10000 = 8*log_{10}(2).  */
+#define MAX_INT_TO_STRING_LEN(x) ((sizeof(x) * 24082 / 10000) + 2)
+
+/* Find the minimum or maximum of two provided values */
+# define MIN(i, j) ((i) <= (j) ? (i) : (j))
+# define MAX(i, j) ((i) >= (j) ? (i) : (j))
+
+
 extern const char *exec_name;
-
+extern const char *program_name;
+extern const char *program_argstring;
+
 /* Document type ("dt") flags */
 enum
 {
-  TEXTHTML             = 0x0001,	/* document is of type text/html
+  TEXTHTML             = 0x0001,        /* document is of type text/html
                                            or application/xhtml+xml */
-  RETROKF              = 0x0002,	/* retrieval was OK */
-  HEAD_ONLY            = 0x0004,	/* only send the HEAD request */
-  SEND_NOCACHE         = 0x0008,	/* send Pragma: no-cache directive */
-  ACCEPTRANGES         = 0x0010,	/* Accept-ranges header was found */
+  RETROKF              = 0x0002,        /* retrieval was OK */
+  HEAD_ONLY            = 0x0004,        /* only send the HEAD request */
+  SEND_NOCACHE         = 0x0008,        /* send Pragma: no-cache directive */
+  ACCEPTRANGES         = 0x0010,        /* Accept-ranges header was found */
   ADDED_HTML_EXTENSION = 0x0020,        /* added ".html" extension due to -E */
-  TEXTCSS              = 0x0040	        /* document is of type text/css */
+  TEXTCSS              = 0x0040         /* document is of type text/css */
 };
 
 /* Universal error type -- used almost everywhere.  Error reporting of
@@ -334,33 +344,26 @@ enum
    simplified.  */
 typedef enum
 {
-  /*  0  */
   NOCONERROR, HOSTERR, CONSOCKERR, CONERROR, CONSSLERR,
-  CONIMPOSSIBLE, NEWLOCATION, NOTENOUGHMEM /* ! */,
-  CONPORTERR /* ! */, CONCLOSED /* ! */,
-  /* 10  */
+  CONIMPOSSIBLE, NEWLOCATION,
   FTPOK, FTPLOGINC, FTPLOGREFUSED, FTPPORTERR, FTPSYSERR,
-  FTPNSFOD, FTPRETROK /* ! */, FTPUNKNOWNTYPE, FTPRERR, FTPREXC /* ! */,
-  /* 20  */
+  FTPNSFOD, FTPUNKNOWNTYPE, FTPRERR,
   FTPSRVERR, FTPRETRINT, FTPRESTFAIL, URLERROR, FOPENERR,
-  FOPEN_EXCL_ERR, FWRITEERR, HOK /* ! */, HLEXC /* ! */, HEOF,
-  /* 30  */
-  HERR, RETROK, RECLEVELEXC, FTPACCDENIED /* ! */, WRONGCODE,
+  FOPEN_EXCL_ERR, FWRITEERR, HEOF, GATEWAYTIMEOUT,
+  HERR, RETROK, RECLEVELEXC, WRONGCODE,
   FTPINVPASV, FTPNOPASV, CONTNOTSUPPORTED, RETRUNNEEDED, RETRFINISHED,
-  /* 40  */
-  READERR, TRYLIMEXC, URLBADPATTERN /* ! */, FILEBADFILE /* ! */, RANGEERR,
-  RETRBADPATTERN, RETNOTSUP /* ! */, ROBOTSOK /* ! */, NOROBOTS /* ! */,
-  PROXERR,
-  /* 50  */
+  READERR, TRYLIMEXC, FILEBADFILE, RANGEERR,
+  RETRBADPATTERN, PROXERR,
   AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR,
-  UNLINKERR, NEWLOCATION_KEEP_POST
+  UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, ATTRMISSING, UNKNOWNATTR,
+  WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR
 } uerr_t;
 
 /* 2005-02-19 SMS.
    Select an appropriate "orig" suffix and a separator character for
    adding a unique suffix to a file name.
 
-   A VMS ODS2 file system can not tolerate multiple dots.  An ODS5 file
+   A VMS ODS2 file system can't tolerate multiple dots.  An ODS5 file
    system can, but even there not all dots are equal, and heroic effort
    would be needed to get ".html^.orig" rather than (the less desirable)
    "^.html.orig".  It's more satisfactory always to use "_orig" on VMS

testenv/ChangeLog

@@ -0,0 +1,576 @@
+2014-11-26  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (TESTS): Add Test-redirect-crash.py.
+	* Test-redirect-crash.py: New File.
+
+2014-11-26  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+       * Makefile.am: Removed Test-auth-both.py from XFAIL_TESTS
+
+2014-11-21  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* server/http/http_server.py: Fixed typo Blackisted to Blacklisted
+
+2014-11-19  Darshit Shah  <darnir@gmail.com>
+
+	* Test-504.py: Add new test case to show how Wget handles 504 Gateway
+	Timeouts
+	* Makefile.am: Add Test-504.py to TESTS
+
+2014-11-17  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* server/http/http_server.py: allow case-insensitive auth-type,
+	send BasIc and DIgest to provoke Wget failures with turkish locales
+
+2014-11-15  Darshit Shah  <darnir@gmail.com>
+
+	* certs/README: Remove trailing whitespaces
+
+2014-11-11  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* recreated the certs and the CRL file, also amended certs/README
+
+2014-11-10  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* removed certs/wget-cert.pem
+	* added certs/ca-cert.pem, certs/ca-key.pem, certs/server-cert.pem,
+	  certs/server-key.pem and certs/server-crl.pem
+	* added certs/README for description how to generate the certs and keys
+	* server/http/http_server.py: amended to work with new certs/ files
+	* added Test--https-crl.py to test --crl-file
+	* Makefile.am: added Test--https-crl.py
+	* Test--https.py: use --ca-certificate of --no-check-certificates
+
+2014-10-25  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* test/base_test.py (gen_cmd_line): generate valgrind command line if requested
+	* README: amend description of VALGRIND_TESTS
+	* Makefile.am: set/export VALGRIND_TESTS
+
+2014-10-01  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am: Run the tests in Python's Optimizedmode
+	* conf/__init__.py (gen_hook): Use try..except instead of if..else
+	* misc/color_terminal.py: System and check will not change while a test is
+	run. Do not test for them on every invokation of printer()
+	* server/http/http_server.py: The ssl and re modules are required by
+	specific functions. Load them lazily
+	(HTTPSServer.__init__): Lazy load ssl module here
+	(_handler.parse_range_header): Lazy load re module here
+	(_Handler.get_rule_list): get() can return a default value. Use it
+	(_Handler.guess_type): Same
+	(_Handler.is_authorized): Unused function artefact. Remove
+	(_Handler.reject_headers): Unused function artefact. Remove
+
+2014-10-08  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am: Fix EXTRA_DIST variable for make distcheck
+	* server/http/http_server.py (HTTPServer.__init__):  Fix how CERTFILE is
+	found when running make dist / make distcheck.
+
+2014-09-30  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* test/base_test.py: Add --track-origins=yes to valgrind testing
+
+2014-08-08  Darshit Shah  <darnir@gmail.com>
+
+	* conf/__init__.py: Add extra newline according to PEP8
+	* conf/{authentication,expect_header,expected_files,expected_ret_code,
+	files_crawled,hook_sample,local_files,reject_header,response,send_header,
+	server_files,urls,wget_commands}.py: Add docstrings explaining the conf file
+	and how it should be used
+	* server/http/http_server (InvalidRangeHeader): Clear TODO and eliminate
+	this exception. Use ServerError for all such purposes.
+	(_Handler): Remove reference to InvalidRangeHeader
+	(_handler.parse_range_header): User ServerError instead of InvalidRangeHeader
+	(_Handler.do_GET):  Add docstring
+	(_Handler.do_POST): Add docstring. Also create an empty dict for rules if
+	no rules are supplied. Send the Location header as suggested in RFC 7231
+	(_Handler.do_PUT): Don't pop the server file already. Push it to later in ..
+	(_Handler.send_put): .. Here. If the file exists respond with a 204 No
+	Content message and pop the file for replacement. Do not send the
+	Content-Length, Content-Type headers since PUT requests should not respond
+	with data.
+	(_Handler.parse_auth_header): Fit line within 80 chars
+	(_Handler.check_response): Better visual indent
+	(_Handler.authorize_digest): Better visual indent.
+	(_Handler.expect_headers): Remove unused function
+	(_Handler.guess_type): Fix indentation
+	(HTTPd): Add newline according to PEP8 guidelines
+	(HTTPSd): Fix indentation
+	(StoppableHTTPServer): Add docstring
+	(HTTPSServer): Fix indentation
+	(WgetHTTPRequestHandler): Merge class into _handler.
+	(_Handler): Add docstring
+	(_Handler.parse_range_header): Fix indentation
+	(ServerError): Split exception into separate file ...
+	* exc/server_error.py: ... Here
+	* misc/colour_terminal.py: Add docstring, fix indentation
+	* test/base_test.py: Fix visual indent
+	* test/http_test.py: Fit within 80 char lines
+
+2014-08-04  Darshit Shah  <darnir@gmail.com>
+
+	* conf/server_conf.py: Delete file. Server configuration is now done via the
+	server_conf() method.
+	* server/http/http_server.py (StppableHTTPServer.server_sett): Delete
+	method required by the above hook
+	(HTTPd.server_sett): Same
+
+2014-07-26  Darshit Shah  <darnir@gmail.com>
+
+	* Test-*.py: Remove the '-d' switch from WGET_OPTIONS.
+	* test/base_test (BaseTest.gen_cmd_line): Add --debug and --no-config to the
+	list of switches passed to wget unconditionally.
+
+2014-07-23  Darshit Shah  <darnir@gmail.com>
+
+	* test/base_test.py (BaseTest.gen_cmd_line): Add support for running all
+	tests through valgrind if the relevant environment variable is set
+	* conf/expected_ret_code (ExpectedRetCode.__call__): Valgrind returns error
+	code 45 when it detects a memory leak.
+	* Readme: Update with details about valgrind tests
+
+2014-07-22  Darshit Shah  <darnir@gmail.com>
+
+	* (README): Remove old TODO and document SERVER_WAIT variable
+
+2014-06-22  Darshit Shah  <darnir@gmail.com>
+
+	* (conf.files_crawled): diff is a set object and needs explicit str
+	conversion.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* base_test.py:
+	(CommonMethods): Rename to BaseTest.
+	(BaseTest): Implement __init__ method where the class-wide variables are
+	initialized. Also variable names like `xxx_list` is renamed to its plural
+	form, e.g. `server_list` => `servers`.
+	(BaseTest.init_test_env): Remove name argument due to its unnecessarity.
+	(BaseTest.get_test_dir): Because the path of the test directory is needed
+	in multiple methods, this method is implemented.
+	(BaseTest.get_domain_addr): Rewrite the return statement utilizing str
+	formatting (which is more Pythonic).
+	(BaseTest.get_cmd_line): Rename to gen_cmd_line. Change the variables with
+	capitcal characters to lower ones. Also, the nested for loop is rewritten
+	to a plain loop using the zip function.
+	(BaseTest.__gen_local_filesys): Rename to gen_local_fs_snapshot. Move to
+	ExpectedFiles in conf/expected_files.py and is marked as a static
+	method. Refactor to a less verbose implementation.
+	(BaseTest._check_downloaded_files): Rename to __call__ to agree with the
+	invocation in test case classes. Move to ExpectedFiles in
+	conf/expected_files.py.
+	(BaseTest.get_server_rules): Refactor to a more Pythonic form utilizing
+	dict.items() and is marked static.
+	(BaseTest.stop_server): (new method) an abstract method which should stop
+	the currently using servers.
+	(BaseTest.instantiate_server_by): (new method) an abstract method which
+	should instantiate a server instance according to the given argument.
+	(BaseTest.__enter__): (new method) method which initialize the context
+	manager
+	(BaseTest.__exit__): (new method) method that finilize the context manager
+	and deal with the exceptions during the execution of the with statement,
+	subclasses can override this method for extensibility
+	* http_test.py:
+	(HTTPTest.__init__): Add call to super.__init__. Default values of
+	pre_hook, test_params, post_hook are set to None to avoid a subtle bug of
+	Python. Argument servers is renamed to protocols.
+	(HTTPTest.Server_setup): Move to BaseTest and rename to server_setup.
+	Calls to pre_hook_call, call_test, post_hook_call are removed.
+	(HTTPTest.hook_call, pre_hook_call, call_test, post_hook_call): Move to
+	BaseTest for that both HTTP test cases and FTP test cases may use these
+	methods.
+	(HTTPTest.init_HTTP_Server, init_HTTPS_Server): Merge and rename to
+	instantiate_server_by to implement the abstract method in BaseTest.
+	(HTTPTest.stop_HTTP_Server): Rename to stop_server to implement the
+	abstract method in BaseTest. Also, pull out the part where remaining
+	requests are gathered into a new method request_remaining.
+	(BaseTest.act_retcode): Rename to ret_code because ExpectedRetCode is
+	moved out from BaseTest, so the name act_retcode is actually a bit
+	verbose.
+	* conf/expected_ret_code.py:
+	(ExpectedRetCode.__call__): Rewrite the str into a more readable form.
+	* conf/files_crawled.py:
+	(FilesCrawled.__call__): Refactor this method into a more Pythonic form
+	utilizing the zip function.
+	* conf/local_files.py:
+	(LocalFiles__call__): Rewrite this method with the recommended with
+	statement.
+	* conf/server_conf.py:
+	(ServerConf.__call__): Rewrite this method due to BaseTest.server_list is
+	renamed to BaseTest.servers.
+	* conf/server_files.py:
+	(ServerFiles.__call__): Refactor the nested for loop into a plain one
+	utilizing the zip function.
+	* conf/urls.py:
+	(URLs): Rename url_list to urls.
+	* conf/wget_commands.py:
+	(WgetCommands): Rename command_list to commands, rename test_obj.options
+	to test_obj.wget_options.
+	* Test--https.py, Test-Proto.py, Test-Parallel-Proto.py: Argument servers
+	is changed to protocols due to change in the signature of
+	HTTPTest.__init__.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* test: (new package) package for test case classes
+	* WgetTest.py: Split into test/base_test.py and test/http_test.py.
+	* Test-*.py: Optimize the imports according to changes of WgetTest.py
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* server: (new package) package for the server classes
+	* server.http: (new package) package for HTTP server
+	* server.ftp: (new package) package for FTP server
+	* HTTPServer.py: Move to server/http/http_server.py. Also change the
+	CERTFILE to '../certs/wget-cert.pem'.
+	* FTPServer.py: Move to server/ftp/ftp_server.py.
+	* WgetTest.py: Optimize import respect to the server classes.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* conf: (new package) package for rule classes and hook methods
+	* WgetTest.py:
+	(CommonMethods.Authentication): Move to conf/authentication.py.
+	(CommonMethods.ExpectHeader): Move to conf/expect_header.py.
+	(CommonMethods.RejectHeader): Move to conf/reject_header.py.
+	(CommonMethods.Response): Move to conf/response.py.
+	(CommonMethods.SendHeader): Move to conf/send_header.py.
+	(CommonMethods.ServerFiles): Move to conf/server_files.py.
+	(CommonMethods.LocalFiles): Move to conf/local_files.py.
+	(CommonMethods.ServerConf): Move to conf/server_conf.py.
+	(CommonMethods.WgetCommands): Move to conf/wget_commands.py.
+	(CommonMethods.Urls): Move to conf/urls.py.
+	(CommonMethods.ExpectedRetcode): Move to conf/expected_retcode.py.
+	(CommonMethods.ExpectedFiles): Move to conf/expected_files.py.
+	(CommonMethods.FilesCrawled): Move to conf/files_crawled.py.
+	(CommonMethods.__check_downloaded_files): Rename to
+	_check_downloaded_files, so that the method is callable from outside the
+	class.
+	(CommomMethods.get_server_rules): Modify so that it utilizes the conf
+	package.
+	(HTTPTest): Add a method hook_call(configs, name) to reduce duplications
+	in pre_hook_call, call_test and post_hook_call utilizing the conf package.
+	* conf/hook_sample.py: (new file) sample for hooks
+	* conf/rule_sample.py: (new file) sample for rules
+	* REAMDE: Update sections about customizing rules and hooks.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* exc: (new package) package for miscellaneous exceptions
+	* WgetTest.py: Move TestFailed to exc/test_failed.py.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* Test-Proto.py: Fix a typo (line 71: server to servers).
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* WgetTest.py: Move WgetFile to package misc.
+	* README: Modify documentation respect to WgetFile.
+	* Test-*.py: Optimize imports about WgetFile.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* misc: (new package) package for miscellaneous modules
+	* ColourTerm.py: Move to package misc and rename to colour_terminal.py,
+	add print_color functions to reduce the use of string literals like
+	"BLUE", "RED" etc.
+	* WgetTest.py:
+	(CommonMethods.Server_setup): Change invocation to printer to print_blue.
+	(CommonMethods.FilesCrawled): Change invocation to printer to print_red.
+	(HTTPTest.__init__): Change invocations to printer to print_red and
+	print_green respectively.
+
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+	* Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
+	Also replace all tabs with spaces in file for conformity.
+	* Test--https.py: New test to check if Wget works correctly with HTTPS
+	servers
+	* HTTPServer.py: Import new modules for use in HTTPS Servers
+	(HTTPSServer): New class that generates a SSL-wrapped socket for use in a
+	HTTPS Server.
+	(HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
+	* WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
+	types
+	(CommonMethods.exec_wget): Add the protocol information to the URL before
+	passing it to wget
+	(HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
+	servers defined by their type. E.g. HTTP, HTTPS.
+	(HTTPTest.Server_setup): Reflect change in type of variable servers.
+	However, we maintin the value of self.servers to allow most of the code to
+	remain unchanged.
+	(HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
+	* Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
+	* Test-Proto.py: Same
+
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling
+	the Wget executable.
+
+2013-12-27  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py: Add modeline
+	(CommonMethods.ServerConf): New pre-test hook that sets
+	BaseHTTPRequestHandler class variables in all available servers
+	* HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server
+	to set the class variables
+	(StoppableHTTPServer.server_sett): Set the handler class variables
+
+2013-12-26  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server.
+
+2013-12-25  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the
+	Wget executable is not found at src/wget
+	(HTTPTest.call_test): In case of error during execution, remove all existing
+	servers before quitting
+
+2013-12-15  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be
+	easily reused for other non-HTTP servers.
+	(HTTPTest.__init__): Call Server_setup instead of HTTP_setup
+	(HTTPTest.Server_setup): Split into three more functions, that handle
+	pre-hooks, test execution and post-hooks respectively.
+	(HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split
+	from HTTPTest.Server_setup
+	(HTTPTest.call_test): Execute wget and log exit code. Code split from
+	HTTPTest.Server_setup
+	(HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split
+	from HTTPTest.Server_setup
+
+2013-10-14  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py.
+
+2013-10-06  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (EXTRA_DIST): Distribute test files.
+
+2013-09-16  Darshit Shah  <darnir@gmail.com>
+
+	* README: Update documentation
+
+2013-09-14  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (StoppableHTTPServer): Define object variable
+	request_headers which stores a list of requests received by the server
+	(StoppableHTTPServer.get_req_headers): Return the list of Request
+	Headers stored by the server
+	(_Handler.do_HEAD): Send the Request MEthod string for identification
+	(_Handler.do_GET): Same
+	(_Handler.__log_request): Log the request in Request_Headers list
+	(_Handler.send_head): Make a call to __log_request
+	* Test--spider-r.py: Add new list, Request_List, which contains all
+	the requests that Wget is expected to send. This will allow for
+	fine-grained tests on recursive downloading.
+	* WgetTest.py (CommonMethods.FilesCrawled): New Post-Test Hook, that
+	ensures that all the expected Files on the server were accessed as
+	expected.
+	(HTTPTest.stop_HTTP_server): On stopping server, asks it to respond
+	with list of all requests it received.
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* Test--spider-r.py: Test retrieval in recursive spider mode.
+	* Makefile.am: add new file
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (_Handler.do_HEAD): If requested path is /, respond
+	with /index.html
+	(_Handler.do_HEAD): Smartly guess value of Content-Type Header from
+	file extension
+	(_Handler.guess_type): Use a preset list of extensions and
+	Content-Type strings. If the extension matches one in the list, use
+	that string, else default to "text/plain"
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods._replace_substring): New method that will
+	replace a substring delimited by {{ }} characters by the value of
+	self.<substring> variable
+	(CommonMethods.WgetCommands): Use the _replace_substring () call to
+	replace the substrings in the the command line.
+	(CommonMethods.ServerFiles): Run the _replace_substring () method on
+	the File contents too.
+
+2013-09-11  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods.exec_wget): Expect domain_list instead of
+	domain.
+	(CommonMethods.get_cmd_line): Same. Generate command line by
+	prepending to each file it's respective domain string
+	(CommonMethods.ServerFiles): Generate file_list and server_rules for
+	each Server and set the config details
+	(HTTPTest): New named parameter, servers which signifies number of
+	servers to spawn
+	(HTTPTest.HTTP_setup): This method now takes servers as a new
+	parameter. Instead of storing server and domain, we now store
+	server_list and domain_list. Each server must be initialized through a
+	loop.
+	(HTTPTest.stop_HTTP_server): Stop all servers in a loop.
+	* Test-Parallel-Proto.py: Prototype test file for multiple servers.
+
+2013-09-10  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.stop_HTTP_server): With the threaded servers,
+	we can simply use the socketserver.shutdown() method to close the
+	server instead of sending a QUIT command
+	* HTTPServer.py (StoppabelHTTPServer.serve_forever): Delete method. No
+	need to override this method anymore.
+	(WgetHTTPRequestHandler.do_QUIT): No longer required
+	(HTTPd): Rename self.server to self.server_inst to reduce ambiguity
+	when referenced from WgetTest
+
+2013-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* README (File Structure): Add explanation about various variables
+	used consistently across all tests.
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py: Remove bunch of old code artefacts
+	* WgetTest.py: Same
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (StoppableHTTPServer.server_conf): Change global
+	variable fileSys to an object variable. This is good programming
+	practice and required for parallel-wget support.
+	(StoppableHTTPServer.server_forever): Edit overridden method to remove
+	the global queue variable. No longer required under the new working
+	(WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue
+	(_Handler): Rename class __Handler to _Handler to match Python's
+	encapsulation rules
+	(_Handler.do_POST): fileSys is now an object variable of the server
+	(_Handler.do_PUT): Same
+	(_Handler.send_put): Same
+	(_Handler.send_head): Same
+	(HTTPd): New class that wraps around the server for Threading
+	(create_server): Make new object of HTTPd.
+	(spawn_server): Start the thread created through create_server
+	(ret_fileSys): Removed method. No longer required.
+	* WgetTest.py (HTTPTest.__init__): Don't explicitly set
+	self.act_retcode. Instead toggle tests_passed boolean to set the
+	correct return code.
+	(HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to
+	start a new instance of the server.
+	(HTTPTest.init_HTTP_server): We no longer call the old
+	create_server(), spawn_server() methods. Instead use the new HTTPd
+	class interface to create new instances of the server
+	(HTTPTest.stop_HTTP_server): Don't ask server to return fileSys.
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* Test-Post.py: Test basic functionality for sending HTTP POST
+	requests using the --method command
+	* Makefile.am: Add new test
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods.__check_downloaded_files): Print a
+	unified diff in case there is a mismatch in the file contents
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (WgetHTTPRequestHandler.test_cookies): Comment out the
+	old test_cookies code. This is no longer used and was causing problems
+	with expected cookies. The code will soon be removed anyways
+	* Test-cookie.py: Add new test for basic cookie functionality
+	* Test-cookie-401.py: Ensure cookies are saved during a 401 response
+	* Test-cookie-expires.py: Ensure that the Expires field is correctly
+	handled
+	* Test-cookies-domain-mismatch.py: Ensure that mismatched domains are
+	handled by Wget
+	* Makefile.am: Add the new tests
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* README: New section on pending work. Will keep updating this to keep
+	track of work that remains to be done on this implementation
+
+2013-09-05  Darshit Shah  <darnir@gmail.com>
+
+	* Test-auth-with-content-disposition.py: Add test that ensures Content
+	Disposition works alongwith authentication
+	* Makefile.am: Add new test
+
+2013-09-04  Darshit Shah  <darnir@gmail.com>
+
+	* Test-c-full.py: Test Continue options
+	* Makefile.am: Add Test-c-full.py and Test-O
+
+2013-09-02  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am:	Add new Test
+	* Test-Head.py: New Test to ensure HEAD requests are handled correctly
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* README: Explain that TEST_NAME needs to be unique
+	* Test-auth-no-challenge.py: Edit non-unique TEST_NAME
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPTest.py (ServerError): Define new Exception for handling
+	internal control flow.
+	(StoppableHTTPServer.SendHeader): Simply pass. Do nothing. Adding
+	functionality here seems to crash for no apparent reason.
+	(stoppableHTTPServer.send_cust_headers): Minor optimization. No need
+	for extra variable.
+	(__Handler.Response): Handle explicit Response Code Rules
+	(__Handler.Authentication): Handle Authentication rules
+	(__Handler.handle_auth): Actual worker method for authentication
+	(__Handler.ExpectHeader): Ensure Expected Headers are received
+	(__Handler.RejectHeader): Ensure Blacklisted Headers are NOT received
+	(__Handler.send_HEAD): Dynamically call server rule functions based on
+	the self.rules list. This feature will later be added to POST/PUT, etc
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py: Remove import module defaultdict.
+	(CommonMethods.get_server_rules): server_rules should be a dict, not a
+	defaultdict (list).
+	* HTTPServer.py (WgetHTTPRequestHandler.get_rule_list): If rule does
+	not exist, return None.	Not an emppty list.
+	(WgetHTTPRequestHandler.test_cookies): Rule variable is not a list
+	(__Handler.send_cust_headers): Same
+	(__Handler.custom_response): Same
+	(__Handler.is_authorized): Same
+	(__Handler.expect_headers): Same
+	(__Handler.reject_headers): Same
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* README: (newfile) Simple help / instructions about using the Test
+	Environment.
+	* Makefile.am: (newfile) Makefile for the Test Environment. Uses the
+	Automake Parallel Test Harness
+	* WgetTest.py: (newfile) Base module that executes the Test.
+	* HTTPServer.py: (newfile) Contains the custom HTTP Server for the
+	Test Environment. Creates an instance of http.server in Python3.
+	* FTPServer.py: (newfile) Overrides methods from pyftpdlib for use in
+	the Test Environment. ** Work under progress **.
+	* ColourTerm.py: (newfile) A custom module to output coloured text to
+	the terminal. Known to work on POSIX shells.
+	* Test-Proto.py: (newfile) A prototype Test File. This should be
+	copied when writing a new Test Case.
+	* Test-Content-disposition-2.py: Test Content Disposition clobbering
+	* Test-Content-disposition.py: Test Content Disposition Headers
+	* Test-O.py: Test Output filename command
+	* Test-auth-basic-fail.py: Test returncode on auth failure
+	* Test-auth-basic.py: Test Basic Auth negotiation
+	* Test-auth-both.py: Test handling of Multiple auth providers. This
+	test currently fails.
+	* Test-auth-digest.py: Test Digest Auth Negotiation
+	* Test-auth-no-challenge-url.py: Ensure --auth-no-challenge is handled
+	when auth details are in-URL.
+	* Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
+	* Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
+	response.

testenv/Makefile.am

@@ -0,0 +1,65 @@
+# Makefile for `wget' utility
+# Copyright (C) 2013 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+# Additional permission under GNU GPL version 3 section 7
+
+# If you modify this program, or any covered work, by linking or
+# combining it with the OpenSSL project's OpenSSL library (or a
+# modified version of that library), containing parts covered by the
+# terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+# grants you additional permission to convey the resulting work.
+# Corresponding Source for a non-source form of such a combination
+# shall include the source code for the parts of OpenSSL used as well
+# as that of the covered work.
+
+
+AUTOMAKE_OPTIONS = parallel-tests
+AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; MAKE_CHECK=True; export MAKE_CHECK;\
+ export PYTHONPATH=$$PYTHONPATH:$(srcdir); export VALGRIND_TESTS="@VALGRIND_TESTS@";
+if HAVE_PYTHON3
+  TESTS = Test-auth-basic-fail.py           \
+    Test-auth-basic.py                      \
+    Test-auth-both.py                       \
+    Test-auth-digest.py                     \
+    Test-auth-no-challenge.py               \
+    Test-auth-no-challenge-url.py           \
+    Test-auth-retcode.py                    \
+    Test-auth-with-content-disposition.py   \
+    Test-c-full.py                          \
+    Test-Content-disposition-2.py           \
+    Test-Content-disposition.py             \
+    Test-cookie-401.py                      \
+    Test-cookie-domain-mismatch.py          \
+    Test-cookie-expires.py                  \
+    Test-cookie.py                          \
+    Test-Head.py                            \
+    Test--https.py                          \
+    Test--https-crl.py                      \
+    Test-O.py                               \
+    Test-Post.py                            \
+    Test-504.py                             \
+    Test--spider-r.py                       \
+    Test-redirect-crash.py
+
+  # added test cases expected to fail here and under TESTS
+  XFAIL_TESTS =
+endif
+
+EXTRA_DIST = certs conf exc misc server test README $(TESTS) $(XFAIL_TESTS)
+
+TEST_EXTENSIONS = .py
+PY_LOG_COMPILER = python3
+AM_PY_LOG_FLAGS = -O

testenv/README

@@ -0,0 +1,305 @@
+This document describes the working of the GNU Wget Test Suite.
+
+Install Instructions:
+================================================================================
+
+This Test Suite exploits the Parallel Test Harness available in GNU Autotools.
+Since it uses features from a relatively recent verion of Autotools, the minimum
+required version as been bumped up to 1.11.
+Run the './configure' command to generate the Makefile and then run 'make check'
+to execute the Test Suite. Use the '-j n' option with 'make check' to execute
+n tests simultaneously.
+
+Structure:
+================================================================================
+
+    * server: This package contains custom programmatically configurable servers
+    (both HTTP and FTP) for testing Wget. The HTTP server runs an instance of
+    Python's http.server module. The FTP server is to be implemented.
+
+    * test: This package contains the test case classes for HTTP and FTP. The
+    test case classes includes methods for initializing and cleaning up of the
+    test environment.
+
+    * Test-Proto.py: This is a prototype Test Case file. The file defines all
+    the acceptable elements and their uses. Typically, one must copy this file
+    and edit it for writing Test Cases.
+
+    * exc: This package contains custom exception classes used in this test
+    suite.
+
+    * conf: This package contains the configuration classes for servers to be
+    configured with.
+
+    * misc: This package contains several helper modules used in this test
+    suite.
+        - colour_terminal.py: A custom module for printing coloured output to
+        the terminal. Currently it only supports 4 colours in a *nix
+        environment.
+        - wget_file.py: Module which contains WgetFile, which is a file data
+        container object.
+
+Working:
+================================================================================
+
+The Test Files are valid Python scripts and the default mask for them is 755.
+A singular Test must be invoked in the following manner:
+$ ./python3 <Name of Test File>      OR
+$ ./<Name of Test File>
+The script will then initialize the various elements and pass them to an object
+of the respective Test Class. A directory with the name <Test name>-test will be
+created and the PWD will be changed to this directory. The server is then
+spawned with the required configuration elements. A blocking call to Wget is
+made with the command line arguments specified in the Test Case along with the
+list of URLs that it must download. The server is killed once Wget returns and
+the following checks are used to determine the pass/fail status of the test:
+    * Return Code: The Exit code of Wget is matched against the expected Exit
+    Code as mentioned in the Test Case File.
+    * Downloaded Files: Check whether the expected downloaded files exist on
+    disk.
+    * File Content: Test whether the file contents were correctly downloaded by
+    Wget and not corrupted mid-way.
+    * Excess Files: Check to see whether any unexpected files were downloaded
+    by Wget.
+
+Exit Codes:
+===============================================================================
+
+Following is a list of Exit Status Codes for the tests:
+*   0  Test Successful
+*  66  Errors/Warnings Reported by Thread Sanitizer (If built with -fsanitize)
+*  77  Test Skipped
+*  99  Hard Error
+* 100  Test Failed
+
+Tests are skipped when they are either not supported by the platform, or Wget
+is not compiled with support for that feature. This feature has not yet been
+implemented.
+
+Hard Errors occur when there are problems with the Environment code. Hard
+Error reporting is currently not enabled and all errors are reported as
+failures.
+
+All exceptions should ideally be handled gracefully. If you see any unhandled
+exceptions, please file a bug report at <bug-wget@gnu.org>
+
+Environment Variables:
+================================================================================
+
+* SERVER_WAIT: Set this environment variable with a value for the number of
+  seconds the test should sleep between invoking the server and calling the Wget
+  executable. This is used when one would like to test a different version of
+  the executable or for running the test through external utilities like gdb and
+  valgrind.
+* NO_CLEANUP: Do not remove the temporary files created by the test.
+  This will prevent the ${testname}-test directory from being deleted
+* VALGRIND_TESTS: If this variable is set and contains the valgrind command line,
+  the test suite will execute all the tests via this command.
+  If it is set to "1", valgrind memcheck is enabled with hard coded options.
+  This variable is set by ./configure --enable-valgrind-tests.
+
+
+File Structure:
+================================================================================
+
+The test case files are Python scripts. It is believed that Python is a simple
+yet elegant language and should be easy for everyone to comprehend. This test
+suite is written with the objective of making it easy to write new tests. The
+structure has been kept as intuitive as possible and should not require much
+effort to get accustomed to.
+
+All Test Files MUST begin with the following Three Lines:
+#!/usr/bin/python3
+from sys import exit
+from WgetTest import {HTTPTest|FTPTest}
+from misc.wget_file import WgetFile
+
+It is recommended that a small description of the Test Case is provided next.
+This would be very helpful to future contributors.
+Next, is the const variable, TEST_NAME that defines the name of the Test.
+
+Each File in the Test must be represented as a WgetFile object. The WgetFile
+Class has the following prototype:
+WgetFile (str name, str contents, str timestamp, dict rules)
+None except name is a mandatory paramter, one may pass only those parameters
+that are required by the File object.
+
+The timestamp string should be a valid Unix Timestamp as defined in RFC xxxx.
+The rules object is a dictionary element, with the key as the Rule Name and
+value as the Rule Data. In most cases, the Rule Data is another dictionary.
+
+Various variables used consistently across all tests are:
+    * WGET_OPTIONS: The command line string passed to Wget upon invokation. This
+    string may contain URLs, like in the case where in-URL authentication is
+    used. Variable names passed like {{var_name}} will be replaced by the
+    contents of the variable self.var_name before being passed to Wget
+    * WGET_URLS: This is a list of filenames which will be appended as the URLs
+    to Wget during invokation. This is a list of lists, where WGET_URLS[0]
+    represents the list of Filenames called from Server[0], WGET_URLS[1] is a
+    list of files downloaded from Server[2], etc.
+    * Files: This variable defines the files that exist in the Server's
+    filesystem. The Files variable is a list of lists of WgetFile objects.
+    This means that File[0] is a list of WgetFile objects that lie on Server[0],
+    File[1] a list of files on Server[1] and so on.
+    * Existing_Files: This is a list of files that already exist in the
+    directory from which Wget is invoked.
+    * ExpectedReturnCode: The Exit Code expected to be returned by Wget after
+    the test.
+    * ExpectedDownloadedFiles: A list of files that are expected in the local
+    directory after Wget has finished executing. This does not include the files
+    already existing before Wget was launched and must be mentioned again.
+    * Request_List: An unordered list of Requests that each server must receive.
+    This too is a list of lists and follows the same convention as others above.
+
+Both, the HTTPTest and FTPTest modules have the same prototype:
+{
+    name,
+    pre_hook,
+    test_options,
+    post_hook,
+    protocols
+}
+name should be a string, and is usually passed to the TEST_NAME variable,
+the three hooks should be Python dict objects and protocols should be a list of
+protocols, like [HTTP, HTTPS].
+
+Valid File Rules:
+================================================================================
+
+This section lists the currently supported File Rules and their structure.
+
+    * Authentication: Used when a File must require Authorization for access.
+    The value for this key is the following dictionary:
+    |-->Type        :   Basic|Digest|Both|Both_inline
+    |-->User        :   <Username>
+    --->Pass        :   <Password>
+
+    * ExpectHeader  : The following Headers MUST exist in every Request for the
+    File. The value for this key is a dictionary object where each header is
+    represented as:
+    |-->Header Name :   <Header Data>
+
+    * RejectHeader  : This list of Headers must NEVER occur in a request. It
+    uses the same value format as ExpectHeader.
+
+    * SendHeader    : This list of Headers will be sent in EVERY response to a
+    request for the respective file. It follows the same value format as
+    ExpectHeader.
+
+    * Response      : The HTTP Response Code to send to a request for this File.
+    The value is an Integer that represents a valid HTTP Response Code.
+
+Pre Test Hooks:
+================================================================================
+
+The Pre-Test Hooks are executed just after starting the server and just before
+spawning an instance of the server. These are usually used for setting up the
+Test Environment and Server Rules. The currently supported Pre-Test Hooks are:
+
+    * ServerFiles   : A list of WgetFile objects that must exist on the Server
+    * LocalFiles    : A list of WgetFile objects that exist locally on disk
+    before Wget is executed.
+
+Since pre_test is a dictionary, one may not assume that the hooks will be
+executed in the same order as they are defined.
+
+Test Options:
+================================================================================
+
+The test_options dictionary defines the commands to be used when the Test is
+executed. The currently supported options are:
+
+    * Urls          : A list of the filenames that Wget must attempt to
+    download. The complete URL will be created and passed to Wget
+    automatically. (alias URLs)
+    * WgetCommands  : A string consisting of the various commandline switches
+    sent to Wget upon invokation. Any data placed between {{ }} in this string
+    will be replaced with the contents of self.<data> before being passed to
+    Wget. This is particularly useful for getting the hostname and port for a
+    file. While all Download URL's are passed to Urls, a notable exception is
+    when in-url authentication is used. In such a case, the URL is specified in
+    the WgetCommands string.
+
+Post-Test Hooks:
+================================================================================
+
+These hooks are executed as soon as the call to Wget returns. The post-test
+hooks are usually used to run checks on the data, files downloaded, return code,
+etc. The following hooks are currently supported:
+
+    * ExpectedRetcode : This is an integer value of the ReturnCode with which
+    Wget is expected to exit. (alias ExpectedRetCode)
+    * ExpectedFiles   : This is a list of WgetFile objects of the files that
+    must exist locally on disk in the Test directory.
+    * FilesCrawled    : This requires a list of the Requests that the server is
+    expected to receive. The order is un-important since it will vary on the
+    parallel-wget branch. This hook is used in tests for Recursive mode to
+    ensure that the website is traversed correctly.
+
+Writing New Tests:
+================================================================================
+
+See Test-Proto.py for an example of how to write Test Case files. The
+recommended method for writing new Test Case files is to copy Test-Proto.py and
+modify it to ones needs.
+
+In case you require any functionality that is not currently defined in List of
+Rules defined above, you should implement a new class in the conf package. The
+file name doesn't matter (though it's better to give it an appropriate name).
+The new rule or hook class should be like this:
+============================================
+from conf import rule
+
+
+@rule()
+class MyNewRule:
+    def __init__(self, rule_arg):
+        self.rule_arg = rule_arg
+        # your rule initialization code goes here
+============================================
+from conf import hook
+
+
+@hook()
+class MyNewHook:
+    def __init__(self, hook_arg):
+        self.hook_arg = hook_arg
+        # your hook initialization code goes here
+
+    def __call__(self, test_obj):
+        # your hook code goes here
+============================================
+
+Once a new Test File is created, it must be added to the TESTS variable in
+Makefile.am. This way the Test will be executed on running a 'make check'.
+If a Test is expected to fail on the current master branch, then the Test should
+also be added to the XFAIL_TESTS variable. This will allow expected failures to
+pass through. If a test mentioned in the XFAIL_TESTS variable passes, it gets
+red-flagged as a XPASS. Currently, tests expected to fail under valgrind are not
+explicitly marked as XFAIL. Tests failing under valgrind must always be
+considered a blocking error.
+
+Remember to always name the Test correctly using the TEST_NAME variable. This
+is essential since a directory with the Test Name is created and this can
+cause synchronization problems when the Parallel Test Harness is used.
+One can use the following command on Unix systems to check for TEST_NAME
+clashes:
+$ grep -r -h "TEST_NAME =" | cut -c13- | uniq -c -d
+
+Work Remaining:
+================================================================================
+
+Some amount of work still remains to be done.
+    * Errors in server-side checks need to be handled more explicitly
+    * Support parallel-wget branch
+        * Support to spawn multiple servers is already in place. Need to handle
+        multiple requests to a server simultaneously. Use THreading MixIn.
+    * SSL Tests. Use xyne's HTTPS server implemention
+    * Complete support for FTP Tests
+    * IRI Support. This shouldn't require much effort
+
+Requirements:
+================================================================================
+
+1. Python   >= 3.0
+2. Automake >= 1.11

testenv/Test--https-crl.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+import os
+
+"""
+    This test ensures that Wget can download files from HTTPS Servers
+"""
+TEST_NAME = "HTTPS CRL"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+
+A_File = WgetFile ("File1", File1)
+B_File = WgetFile ("File2", File2)
+
+CAFILE = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs', 'ca-cert.pem'))
+CRLFILE = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs', 'server-crl.pem'))
+WGET_OPTIONS = "--crl-file " + CRLFILE + " --ca-certificate=" + CAFILE
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+
+Servers = [HTTPS]
+
+ExpectedReturnCode = 5
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)