Prepare release 1.15

Previously wget didn't honor the --timeout option if the remote host did
not answer SSL handshake

Signed-off-by: Tomas Hozza <thozza@redhat.com>

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "gnulib"]
+	path = gnulib
+	url = git://git.sv.gnu.org/gnulib.git

ChangeLog

@@ -1,3 +1,146 @@
+2013-12-22  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* gnulib: add git submodule.
+
+2013-09-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* configure.ac: added a summary of build options
+	  fixed some indentations
+	  removed the unconditionally adding of libz with --with-ssl
+	  removed -lgcrypt and -lgpg-error for gnutls
+
+2013-07-23  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* configure.ac: Remove AM_CONDITIONAL HAVE_NETTLE.
+	Reported by: Darshit Shah  <darnir@gmail.com>.
+
+2013-07-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* configure.ac: check for libnettle when GNU TLS is used.
+
+2013-05-17 Bykov Aleksey <gnfalex@rambler.ru>
+
+	* bootstrap: Add `mkostemp'
+
+2012-10-07  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: Check for patchconf.
+
+2012-09-23  Merinov Nikolay  <kim.roader@gmail.com>
+
+	* m4/wget.m4 (WGET_FNMATCH): Add AC_LANG_SOURCE into
+	AC_COMPILE_IFELSE in order to silence autoconf 2.68 warning.
+
+2012-09-20  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap: Update from gnulib.
+
+2012-09-02  Nguyễn Thái Ngọc Duy  <pclouds@gmail.com> (tiny change)
+
+	* po/POTFILES.in: Add more files.
+
+2012-07-08  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap: Update from gnulib.
+	* bootstrap.conf (gnulib_extra_files): Remove $build_aux/missing.
+	* lib/Makefile.am: Delete file.
+
+2012-06-16  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Remove `closeout'.
+	Reported by: Micah Cowan  <micah@cowan.name>.
+
+2012-05-31  Ángel González <keisial@gmail.com>
+
+	* convert.c: fix segfault on wrong urls (bug 36570)
+
+2012-05-13  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Add `git-version-gen'.
+	* build-aux/bzr-version-gen: Remove file.
+	* configure.ac: Invoke `build-aux/git-version-gen' to get the dist
+	version.
+	* Makefile.am (EXTRA_DIST): Distribute build-aux/git-version-gen instead
+	of build-aux/bzr-version-gen.
+
+2012-04-11  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* bootstrap.conf (gnulib_modules): Include module `regex'.
+	* configure.ac: Check for PCRE library.
+
+2012-03-25 Ray Satiro <raysatiro@yahoo.com>
+
+	* configure.ac: Fix build under mingw when OpenSSL is used.
+
+2012-03-20  Ángel González <keisial@gmail.com>
+
+	* bootstrap.conf (gnulib_modules): Add modules `ftello',
+	`mkstemp' and `strtok_r'.
+
+2012-02-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Add module `closeout'.
+
+2012-01-09  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* configure.ac: Always try to use libz, even without SSL.
+
+2011-12-12  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* Makefile.am (EXTRA_DIST): Add build-aux/bzr-version-gen.
+	Reported by: Elan Ruusamäe <glen@pld-linux.org>.
+
+2011-12-11  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* util/trunc.c (main): Call `close' on the fd and check for errors.
+	Reported by: <dga@cs.cmu.edu>.
+
+2011-10-23  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Include module `vsnprintf'.
+
+2011-10-16  Steven Schubiger  <stsc@member.fsf.org>
+
+	* util/paramcheck.pl: Match 1 or more times where applicable.
+	(extract_entries): Return a copy instead of reference.
+
+2011-09-04  Alan Hourihane <alanh@fairlite.co.uk> (tiny change)
+
+	* configure.ac: Check for libz when gnutls is used.
+
+2011-08-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: Under mingw don't check for static OpenSSL
+	libraries if the shared version was already found.
+	Suggested by: Ray Satiro <raysatiro@yahoo.com>.
+
+2011-08-25  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: Check for `utime'.
+
+2011-08-11  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* bootstrap.conf (gnulib_modules): Add `sigprocmask'.
+
+	* configure.ac: Do not hardcode GNU TLS and OpenSSL libraries.
+
+	* bootstrap.conf (gnulib_modules): Include module iconv.
+
+	* configure.ac: Allow --with-libgnutls-prefix and
+	--with-libssl-prefix
+	Suggested by: Karl Berry <karl@freefriends.org>
+
+	* build-aux/bzr-version-gen (TAG): Consider only the last tag.
+
+2011-08-10  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: Print usage string for --with-ssl.
+	Reported by: Karl Berry <karl@freefriends.org>
+
+	* configure.ac: Check for `gnutls_priority_set_direct' when gnutls is
+	used.
+	Reported by: Karl Berry <karl@freefriends.org>
+
 2011-08-09  Giuseppe Scrivano  <gscrivano@southpole.se>
 
 	* build-aux/bzr-version-gen: Fix some portability issues.

Makefile.am

@@ -46,7 +46,7 @@ SUBDIRS = lib src doc po tests util
 EXTRA_DIST = ChangeLog.README MAILING-LIST \
              msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
              msdos/Makefile.WC ABOUT-NLS \
-             build-aux/build_info.pl .version
+             build-aux/build_info.pl build-aux/git-version-gen .version
 
 CLEANFILES = *~ *.bak $(DISTNAME).tar.gz
 

NEWS

@@ -1,15 +1,90 @@
 GNU Wget NEWS -- history of user-visible changes.
 
 Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
 See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
-* Changes in Wget 1.13
+* Changes in Wget 1.15
+
+** Add support for --method.
+
+** Add support for file names longer than MAX_FILE.
+
+** Support FTP listing for the FTP Server on Windows Server 2008 R2.
+
+** Fix a regression when -c and --content-disposition are used together.
+
+** Support shorthand URLs in an input file.
+
+** Fix -c with servers that don't specify a content-length.
+
+** Add support for MD5-SESS
+
+** Do not fail on non fatal GNU TLS alerts during handshake.
+
+** Add support for --https-only.  When used wget will follow only
+   HTTPS links in recursive mode.
+
+** Support Perfect-Forward Secrecy in --secure-protocol.
+
+** Fix a problem with some IRI links that are not followed when contained in a
+   HTML document.
+
+** Support some FTP servers that return an empty list with "LIST -a".
+
+** Specify Host with the HTTP CONNECT method.
+
+** Use the correct HTTP method on a redirection.
+
+* Changes in Wget 1.14
+
+** Add support for content-on-error.  It allows to store the HTTP
+   payload on 4xx or 5xx errors.
+
+** Add support for WARC files.
+
+** Fix a memory leak problem in the GNU TLS backend.
+
+** Autoreconf works again for distributed tarballs.
+
+** Print some diagnostic messages to stderr not to stdout.
+
+** Report stdout close errors.
+
+** Accept the --report-speed option.
+
+** Enable client certificates when GNU TLS is used.
+
+** Add support for TLS Server Name Indication.
+
+** Accept the arguments --accept-reject and --reject-regex.
+
+** The GNU TLS backend honors correctly the timeout value.
+
+** Add support for RFC 2617 Digest Access Authentication.
+
+* Changes in Wget 1.13.4
+
+** Now --version and --help work again.
+
+** Fix a build error on solaris 10 sparc.
+
+** Now --timestamping and --continue work well together.
+
+** Return a network failure when FTP downloads fail and --timestamping
+   is specified.
+
+** Fix a segfault on an incomplete STYLE tag.
+
+* Changes in Wget 1.13.3
 
 ** Support HTTP/1.1
 
+** Now by default the GNU TLS library for secure connections, instead of
+   OpenSSL.
+
 ** Fix some portability issues.
 
 ** Handle properly malformed status line in a HTTP response.

bootstrap

@@ -1,10 +1,10 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2011-04-05.18; # UTC
+scriptversion=2012-07-19.14; # UTC
 
 # Bootstrap this package from checked-out sources.
 
-# Copyright (C) 2003-2011 Free Software Foundation, Inc.
+# Copyright (C) 2003-2012 Free Software Foundation, Inc.
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -36,12 +36,12 @@ nl='
 LC_ALL=C
 export LC_ALL
 
+# Ensure that CDPATH is not set.  Otherwise, the output from cd
+# would cause trouble in at least one use below.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
 local_gl_dir=gl
 
-# Temporary directory names.
-bt='._bootmp'
-bt_regex=`echo "$bt"| sed 's/\./[.]/g'`
-bt2=${bt}2
 me=$0
 
 usage() {
@@ -77,6 +77,33 @@ Running without arguments will suffice in most cases.
 EOF
 }
 
+# warnf_ FORMAT-STRING ARG1...
+warnf_ ()
+{
+  warnf_format_=$1
+  shift
+  nl='
+'
+  case $* in
+    *$nl*) me_=$(printf "$me"|tr "$nl|" '??')
+       printf "$warnf_format_" "$@" | sed "s|^|$me_: |" ;;
+    *) printf "$me: $warnf_format_" "$@" ;;
+  esac >&2
+}
+
+# warn_ WORD1...
+warn_ ()
+{
+  # If IFS does not start with ' ', set it and emit the warning in a subshell.
+  case $IFS in
+    ' '*) warnf_ '%s\n' "$*";;
+    *)    (IFS=' '; warn_ "$@");;
+  esac
+}
+
+# die WORD1...
+die() { warn_ "$@"; exit 1; }
+
 # Configuration.
 
 # Name of the Makefile.am
@@ -88,9 +115,12 @@ gnulib_modules=
 # Any gnulib files needed that are not in modules.
 gnulib_files=
 
-# A function to be called to edit gnulib.mk right after it's created.
+: ${AUTOPOINT=autopoint}
+: ${AUTORECONF=autoreconf}
+
+# A function to be called right after gnulib-tool is run.
 # Override it via your own definition in bootstrap.conf.
-gnulib_mk_hook() { :; }
+bootstrap_post_import_hook() { :; }
 
 # A function to be called after everything else in this script.
 # Override it via your own definition in bootstrap.conf.
@@ -105,6 +135,11 @@ po_download_command_format=\
 "rsync --delete --exclude '*.s1' -Lrtvz \
  'translationproject.org::tp/latest/%s/' '%s'"
 
+# Fallback for downloading .po files (if rsync fails).
+po_download_command_format2=\
+"wget --mirror -nd -q -np -A.po -P '%s' \
+ http://translationproject.org/latest/%s/"
+
 extract_package_name='
   /^AC_INIT(/{
      /.*,.*,.*, */{
@@ -122,7 +157,8 @@ extract_package_name='
      p
   }
 '
-package=`sed -n "$extract_package_name" configure.ac` || exit
+package=$(sed -n "$extract_package_name" configure.ac) \
+  || die 'cannot find package name in configure.ac'
 gnulib_name=lib$package
 
 build_aux=build-aux
@@ -178,6 +214,8 @@ use_git=true
 # otherwise find the first of the NAMES that can be run (i.e.,
 # supports --version).  If found, set ENVVAR to the program name,
 # die otherwise.
+#
+# FIXME: code duplication, see also gnu-web-doc-update.
 find_tool ()
 {
   find_tool_envvar=$1
@@ -195,19 +233,15 @@ find_tool ()
   else
     find_tool_error_prefix="\$$find_tool_envvar: "
   fi
-  if test x"$find_tool_res" = x; then
-    echo >&2 "$me: one of these is required: $find_tool_names"
-    exit 1
-  fi
-  ($find_tool_res --version </dev/null) >/dev/null 2>&1 || {
-    echo >&2 "$me: ${find_tool_error_prefix}cannot run $find_tool_res --version"
-    exit 1
-  }
+  test x"$find_tool_res" != x \
+    || die "one of these is required: $find_tool_names"
+  ($find_tool_res --version </dev/null) >/dev/null 2>&1 \
+    || die "${find_tool_error_prefix}cannot run $find_tool_res --version"
   eval "$find_tool_envvar=\$find_tool_res"
   eval "export $find_tool_envvar"
 }
 
-# Find sha1sum, named gsha1sum on MacPorts, and shasum on MacOS 10.6.
+# Find sha1sum, named gsha1sum on MacPorts, and shasum on Mac OS X 10.6.
 find_tool SHA1SUM sha1sum gsha1sum shasum
 
 # Override the default configuration, if necessary.
@@ -222,7 +256,6 @@ esac
 test -z "${gnulib_extra_files}" && \
   gnulib_extra_files="
         $build_aux/install-sh
-        $build_aux/missing
         $build_aux/mdate-sh
         $build_aux/texinfo.tex
         $build_aux/depcomp
@@ -248,7 +281,7 @@ do
     usage
     exit;;
   --gnulib-srcdir=*)
-    GNULIB_SRCDIR=`expr "X$option" : 'X--gnulib-srcdir=\(.*\)'`;;
+    GNULIB_SRCDIR=${option#--gnulib-srcdir=};;
   --skip-po)
     SKIP_PO=t;;
   --force)
@@ -262,21 +295,15 @@ do
   --no-git)
     use_git=false;;
   *)
-    echo >&2 "$0: $option: unknown option"
-    exit 1;;
+    die "$option: unknown option";;
   esac
 done
 
-if $use_git || test -d "$GNULIB_SRCDIR"; then
-  :
-else
-  echo "$0: Error: --no-git requires --gnulib-srcdir" >&2
-  exit 1
-fi
+$use_git || test -d "$GNULIB_SRCDIR" \
+  || die "Error: --no-git requires --gnulib-srcdir"
 
 if test -n "$checkout_only_file" && test ! -r "$checkout_only_file"; then
-  echo "$0: Bootstrapping from a non-checked-out distribution is risky." >&2
-  exit 1
+  die "Bootstrapping from a non-checked-out distribution is risky."
 fi
 
 # Ensure that lines starting with ! sort last, per gitignore conventions
@@ -290,7 +317,7 @@ sort_patterns() {
     P
     x
     s/^\n//
-  }'
+  }' | sed '/^$/d'
 }
 
 # If $STR is not already on a line by itself in $FILE, insert it,
@@ -299,10 +326,10 @@ insert_sorted_if_absent() {
   file=$1
   str=$2
   test -f $file || touch $file
-  echo "$str" | sort_patterns - $file | cmp - $file > /dev/null \
+  echo "$str" | sort_patterns - $file | cmp -s - $file > /dev/null \
     || { echo "$str" | sort_patterns - $file > $file.bak \
       && mv $file.bak $file; } \
-    || exit 1
+    || die "insert_sorted_if_absent $file $str: failed"
 }
 
 # Adjust $PATTERN for $VC_IGNORE_FILE and insert it with
@@ -312,10 +339,10 @@ insert_vc_ignore() {
   pattern="$2"
   case $vc_ignore_file in
   *.gitignore)
-    # A .gitignore entry that does not start with `/' applies
-    # recursively to subdirectories, so prepend `/' to every
+    # A .gitignore entry that does not start with '/' applies
+    # recursively to subdirectories, so prepend '/' to every
     # .gitignore entry.
-    pattern=`echo "$pattern" | sed s,^,/,`;;
+    pattern=$(echo "$pattern" | sed s,^,/,);;
   esac
   insert_sorted_if_absent "$vc_ignore_file" "$pattern"
 }
@@ -326,11 +353,8 @@ grep '^[	 ]*AC_CONFIG_AUX_DIR(\['"$build_aux"'\])' configure.ac \
     >/dev/null && found_aux_dir=yes
 grep '^[	 ]*AC_CONFIG_AUX_DIR('"$build_aux"')' configure.ac \
     >/dev/null && found_aux_dir=yes
-if test $found_aux_dir = no; then
-  echo "$0: expected line not found in configure.ac. Add the following:" >&2
-  echo "  AC_CONFIG_AUX_DIR([$build_aux])" >&2
-  exit 1
-fi
+test $found_aux_dir = yes \
+  || die "configure.ac lacks 'AC_CONFIG_AUX_DIR([$build_aux])'; add it"
 
 # If $build_aux doesn't exist, create it now, otherwise some bits
 # below will malfunction.  If creating it, also mark it as ignored.
@@ -419,20 +443,51 @@ check_versions() {
       $use_git || continue
     fi
     # Honor $APP variables ($TAR, $AUTOCONF, etc.)
-    appvar=`echo $app | tr '[a-z]-' '[A-Z]_'`
+    appvar=$(echo $app | LC_ALL=C tr '[a-z]-' '[A-Z]_')
     test "$appvar" = TAR && appvar=AMTAR
-    eval "app=\${$appvar-$app}"
-    inst_ver=$(get_version $app)
-    if [ ! "$inst_ver" ]; then
-      echo "$me: Error: '$app' not found" >&2
-      ret=1
-    elif [ ! "$req_ver" = "-" ]; then
-      latest_ver=$(sort_ver $req_ver $inst_ver | cut -d' ' -f2)
-      if [ ! "$latest_ver" = "$inst_ver" ]; then
-        echo "$me: Error: '$app' version == $inst_ver is too old" >&2
-        echo "       '$app' version >= $req_ver is required" >&2
+    case $appvar in
+        GZIP) ;; # Do not use $GZIP:  it contains gzip options.
+        *) eval "app=\${$appvar-$app}" ;;
+    esac
+
+    # Handle the still-experimental Automake-NG programs specially.
+    # They remain named as the mainstream Automake programs ("automake",
+    # and "aclocal") to avoid gratuitous incompatibilities with
+    # pre-existing usages (by, say, autoreconf, or custom autogen.sh
+    # scripts), but correctly identify themselves (as being part of
+    # "GNU automake-ng") when asked their version.
+    case $app in
+      automake-ng|aclocal-ng)
+        app=${app%-ng}
+        ($app --version | grep '(GNU automake-ng)') >/dev/null 2>&1 || {
+          warn_ "Error: '$app' not found or not from Automake-NG"
+          ret=1
+          continue
+        } ;;
+    esac
+    if [ "$req_ver" = "-" ]; then
+      # Merely require app to exist; not all prereq apps are well-behaved
+      # so we have to rely on $? rather than get_version.
+      $app --version >/dev/null 2>&1
+      if [ 126 -le $? ]; then
+        warn_ "Error: '$app' not found"
         ret=1
       fi
+    else
+      # Require app to produce a new enough version string.
+      inst_ver=$(get_version $app)
+      if [ ! "$inst_ver" ]; then
+        warn_ "Error: '$app' not found"
+        ret=1
+      else
+        latest_ver=$(sort_ver $req_ver $inst_ver | cut -d' ' -f2)
+        if [ ! "$latest_ver" = "$inst_ver" ]; then
+          warnf_ '%s\n'                                        \
+              "Error: '$app' version == $inst_ver is too old"  \
+              "       '$app' version >= $req_ver is required"
+          ret=1
+        fi
+      fi
     fi
   done
 
@@ -459,14 +514,37 @@ if test $use_libtool = 1; then
   find_tool LIBTOOLIZE glibtoolize libtoolize
 fi
 
+# gnulib-tool requires at least automake and autoconf.
+# If either is not listed, add it (with minimum version) as a prerequisite.
+case $buildreq in
+  *automake*) ;;
+  *) buildreq="automake 1.9
+$buildreq" ;;
+esac
+case $buildreq in
+  *autoconf*) ;;
+  *) buildreq="autoconf 2.59
+$buildreq" ;;
+esac
+
+# When we can deduce that gnulib-tool will require patch,
+# and when patch is not already listed as a prerequisite, add it, too.
+if test -d "$local_gl_dir" \
+    && ! find "$local_gl_dir" -name '*.diff' -exec false {} +; then
+  case $buildreq in
+    *patch*) ;;
+    *) buildreq="patch -
+$buildreq" ;;
+  esac
+fi
+
 if ! printf "$buildreq" | check_versions; then
   echo >&2
   if test -f README-prereq; then
-    echo "$0: See README-prereq for how to get the prerequisite programs" >&2
+    die "See README-prereq for how to get the prerequisite programs"
   else
-    echo "$0: Please install the prerequisite programs" >&2
+    die "Please install the prerequisite programs"
   fi
-  exit 1
 fi
 
 echo "$0: Bootstrapping from checked-out $package sources..."
@@ -495,7 +573,7 @@ git_modules_config () {
   test -f .gitmodules && git config --file .gitmodules "$@"
 }
 
-gnulib_path=`git_modules_config submodule.gnulib.path`
+gnulib_path=$(git_modules_config submodule.gnulib.path)
 test -z "$gnulib_path" && gnulib_path=gnulib
 
 # Get gnulib files.
@@ -560,7 +638,7 @@ if $bootstrap_sync; then
 fi
 
 gnulib_tool=$GNULIB_SRCDIR/gnulib-tool
-<$gnulib_tool || exit
+<$gnulib_tool || exit $?
 
 # Get translations.
 
@@ -568,7 +646,10 @@ download_po_files() {
   subdir=$1
   domain=$2
   echo "$me: getting translations into $subdir for $domain..."
-  cmd=`printf "$po_download_command_format" "$domain" "$subdir"`
+  cmd=$(printf "$po_download_command_format" "$domain" "$subdir")
+  eval "$cmd" && return
+  # Fallback to HTTP.
+  cmd=$(printf "$po_download_command_format2" "$subdir" "$domain")
   eval "$cmd"
 }
 
@@ -591,7 +672,7 @@ update_po_files() {
     && ls "$ref_po_dir"/*.po 2>/dev/null |
       sed 's|.*/||; s|\.po$||' > "$po_dir/LINGUAS" || return
 
-  langs=`cd $ref_po_dir && echo *.po|sed 's/\.po//g'`
+  langs=$(cd $ref_po_dir && echo *.po | sed 's/\.po//g')
   test "$langs" = '*' && langs=x
   for po in $langs; do
     case $po in x) continue;; esac
@@ -628,18 +709,18 @@ symlink_to_dir()
 
     # If the destination directory doesn't exist, create it.
     # This is required at least for "lib/uniwidth/cjk.h".
-    dst_dir=`dirname "$dst"`
+    dst_dir=$(dirname "$dst")
     if ! test -d "$dst_dir"; then
       mkdir -p "$dst_dir"
 
       # If we've just created a directory like lib/uniwidth,
       # tell version control system(s) it's ignorable.
       # FIXME: for now, this does only one level
-      parent=`dirname "$dst_dir"`
+      parent=$(dirname "$dst_dir")
       for dot_ig in x $vc_ignore; do
         test $dot_ig = x && continue
         ig=$parent/$dot_ig
-        insert_vc_ignore $ig `echo "$dst_dir"|sed 's,.*/,,'`
+        insert_vc_ignore $ig "${dst_dir##*/}"
       done
     fi
 
@@ -656,21 +737,28 @@ symlink_to_dir()
         cp -fp "$src" "$dst"
       }
     else
+      # Leave any existing symlink alone, if it already points to the source,
+      # so that broken build tools that care about symlink times
+      # aren't confused into doing unnecessary builds.  Conversely, if the
+      # existing symlink's time stamp is older than the source, make it afresh,
+      # so that broken tools aren't confused into skipping needed builds.  See
+      # <http://lists.gnu.org/archive/html/bug-gnulib/2011-05/msg00326.html>.
       test -h "$dst" &&
-      src_ls=`ls -diL "$src" 2>/dev/null` && set $src_ls && src_i=$1 &&
-      dst_ls=`ls -diL "$dst" 2>/dev/null` && set $dst_ls && dst_i=$1 &&
-      test "$src_i" = "$dst_i" || {
+      src_ls=$(ls -diL "$src" 2>/dev/null) && set $src_ls && src_i=$1 &&
+      dst_ls=$(ls -diL "$dst" 2>/dev/null) && set $dst_ls && dst_i=$1 &&
+      test "$src_i" = "$dst_i" &&
+      both_ls=$(ls -dt "$src" "$dst") &&
+      test "X$both_ls" = "X$dst$nl$src" || {
         dot_dots=
         case $src in
         /*) ;;
         *)
           case /$dst/ in
           *//* | */../* | */./* | /*/*/*/*/*/)
-             echo >&2 "$me: invalid symlink calculation: $src -> $dst"
-             exit 1;;
-          /*/*/*/*/)	dot_dots=../../../;;
-          /*/*/*/)	dot_dots=../../;;
-          /*/*/)	dot_dots=../;;
+             die "invalid symlink calculation: $src -> $dst";;
+          /*/*/*/*/)    dot_dots=../../../;;
+          /*/*/*/)      dot_dots=../../;;
+          /*/*/)        dot_dots=../;;
           esac;;
         esac
 
@@ -681,164 +769,94 @@ symlink_to_dir()
   }
 }
 
-cp_mark_as_generated()
-{
-  cp_src=$1
-  cp_dst=$2
-
-  if cmp -s "$cp_src" "$GNULIB_SRCDIR/$cp_dst"; then
-    symlink_to_dir "$GNULIB_SRCDIR" "$cp_dst"
-  elif cmp -s "$cp_src" "$local_gl_dir/$cp_dst"; then
-    symlink_to_dir $local_gl_dir "$cp_dst"
-  else
-    case $cp_dst in
-      *.[ch])             c1='/* '; c2=' */';;
-      *.texi)             c1='@c '; c2=     ;;
-      *.m4|*/Make*|Make*) c1='# ' ; c2=     ;;
-      *)                  c1=     ; c2=     ;;
-    esac
-
-    # If the destination directory doesn't exist, create it.
-    # This is required at least for "lib/uniwidth/cjk.h".
-    dst_dir=`dirname "$cp_dst"`
-    test -d "$dst_dir" || mkdir -p "$dst_dir"
-
-    if test -z "$c1"; then
-      cmp -s "$cp_src" "$cp_dst" || {
-        # Copy the file first to get proper permissions if it
-        # doesn't already exist.  Then overwrite the copy.
-        echo "$me: cp -f $cp_src $cp_dst" &&
-        rm -f "$cp_dst" &&
-        cp "$cp_src" "$cp_dst-t" &&
-        sed "s!$bt_regex/!!g" "$cp_src" > "$cp_dst-t" &&
-        mv -f "$cp_dst-t" "$cp_dst"
-      }
-    else
-      # Copy the file first to get proper permissions if it
-      # doesn't already exist.  Then overwrite the copy.
-      cp "$cp_src" "$cp_dst-t" &&
-      (
-        echo "$c1-*- buffer-read-only: t -*- vi: set ro:$c2" &&
-        echo "${c1}DO NOT EDIT! GENERATED AUTOMATICALLY!$c2" &&
-        sed "s!$bt_regex/!!g" "$cp_src"
-      ) > $cp_dst-t &&
-      if cmp -s "$cp_dst-t" "$cp_dst"; then
-        rm -f "$cp_dst-t"
-      else
-        echo "$me: cp $cp_src $cp_dst # with edits" &&
-        mv -f "$cp_dst-t" "$cp_dst"
-      fi
-    fi
-  fi
-}
-
 version_controlled_file() {
-  dir=$1
+  parent=$1
   file=$2
-  found=no
-  if test -d CVS; then
-    grep -F "/$file/" $dir/CVS/Entries 2>/dev/null |
-             grep '^/[^/]*/[0-9]' > /dev/null && found=yes
-  elif test -d .git; then
-    git rm -n "$dir/$file" > /dev/null 2>&1 && found=yes
+  if test -d .git; then
+    git rm -n "$file" > /dev/null 2>&1
   elif test -d .svn; then
-    svn log -r HEAD "$dir/$file" > /dev/null 2>&1 && found=yes
+    svn log -r HEAD "$file" > /dev/null 2>&1
+  elif test -d CVS; then
+    grep -F "/${file##*/}/" "$parent/CVS/Entries" 2>/dev/null |
+             grep '^/[^/]*/[0-9]' > /dev/null
   else
-    echo "$me: no version control for $dir/$file?" >&2
+    warn_ "no version control for $file?"
+    false
   fi
-  test $found = yes
 }
 
-slurp() {
-  for dir in . `(cd $1 && find * -type d -print)`; do
-    copied=
-    sep=
-    for file in `ls -a $1/$dir`; do
-      case $file in
-      .|..) continue;;
-      # FIXME: should all file names starting with "." be ignored?
-      .*) continue;;
-      esac
-      test -d $1/$dir/$file && continue
-      for excluded_file in $excluded_files; do
-        test "$dir/$file" = "$excluded_file" && continue 2
+# NOTE: we have to be careful to run both autopoint and libtoolize
+# before gnulib-tool, since gnulib-tool is likely to provide newer
+# versions of files "installed" by these two programs.
+# Then, *after* gnulib-tool (see below), we have to be careful to
+# run autoreconf in such a way that it does not run either of these
+# two just-pre-run programs.
+
+# Import from gettext.
+with_gettext=yes
+grep '^[	 ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \
+    with_gettext=no
+
+if test $with_gettext = yes || test $use_libtool = 1; then
+
+  tempbase=.bootstrap$$
+  trap "rm -f $tempbase.0 $tempbase.1" 1 2 13 15
+
+  > $tempbase.0 > $tempbase.1 &&
+  find . ! -type d -print | sort > $tempbase.0 || exit
+
+  if test $with_gettext = yes; then
+    # Released autopoint has the tendency to install macros that have been
+    # obsoleted in current gnulib, so run this before gnulib-tool.
+    echo "$0: $AUTOPOINT --force"
+    $AUTOPOINT --force || exit
+  fi
+
+  # Autoreconf runs aclocal before libtoolize, which causes spurious
+  # warnings if the initial aclocal is confused by the libtoolized
+  # (or worse out-of-date) macro directory.
+  # libtoolize 1.9b added the --install option; but we support back
+  # to libtoolize 1.5.22, where the install action was default.
+  if test $use_libtool = 1; then
+    install=
+    case $($LIBTOOLIZE --help) in
+      *--install*) install=--install ;;
+    esac
+    echo "running: $LIBTOOLIZE $install --copy"
+    $LIBTOOLIZE $install --copy
+  fi
+
+  find . ! -type d -print | sort >$tempbase.1
+  old_IFS=$IFS
+  IFS=$nl
+  for file in $(comm -13 $tempbase.0 $tempbase.1); do
+    IFS=$old_IFS
+    parent=${file%/*}
+    version_controlled_file "$parent" "$file" || {
+      for dot_ig in x $vc_ignore; do
+        test $dot_ig = x && continue
+        ig=$parent/$dot_ig
+        insert_vc_ignore "$ig" "${file##*/}"
       done
-      if test $file = Makefile.am && test "X$gnulib_mk" != XMakefile.am; then
-        copied=$copied${sep}$gnulib_mk; sep=$nl
-        remove_intl='/^[^#].*\/intl/s/^/#/;'"s!$bt_regex/!!g"
-        sed "$remove_intl" $1/$dir/$file |
-        cmp - $dir/$gnulib_mk > /dev/null || {
-          echo "$me: Copying $1/$dir/$file to $dir/$gnulib_mk ..." &&
-          rm -f $dir/$gnulib_mk &&
-          sed "$remove_intl" $1/$dir/$file >$dir/$gnulib_mk &&
-          gnulib_mk_hook $dir/$gnulib_mk
-        }
-      elif { test "${2+set}" = set && test -r $2/$dir/$file; } ||
-           version_controlled_file $dir $file; then
-        echo "$me: $dir/$file overrides $1/$dir/$file"
-      else
-        copied=$copied$sep$file; sep=$nl
-        if test $file = gettext.m4; then
-          echo "$me: patching m4/gettext.m4 to remove need for intl/* ..."
-          rm -f $dir/$file
-          sed '
-            /^AC_DEFUN(\[AM_INTL_SUBDIR],/,/^]/c\
-              AC_DEFUN([AM_INTL_SUBDIR], [])
-            /^AC_DEFUN(\[gt_INTL_SUBDIR_CORE],/,/^]/c\
-              AC_DEFUN([gt_INTL_SUBDIR_CORE], [])
-            $a\
-              AC_DEFUN([gl_LOCK_EARLY], [])
-          ' $1/$dir/$file >$dir/$file
-        else
-          cp_mark_as_generated $1/$dir/$file $dir/$file
-        fi
-      fi || exit
-    done
-
-    for dot_ig in x $vc_ignore; do
-      test $dot_ig = x && continue
-      ig=$dir/$dot_ig
-      if test -n "$copied"; then
-        insert_vc_ignore $ig "$copied"
-        # If an ignored file name ends with .in.h, then also add
-        # the name with just ".h".  Many gnulib headers are generated,
-        # e.g., stdint.in.h -> stdint.h, dirent.in.h ->..., etc.
-        # Likewise for .gperf -> .h, .y -> .c, and .sin -> .sed
-        f=`echo "$copied" |
-          sed '
-            s/\.in\.h$/.h/
-            s/\.sin$/.sed/
-            s/\.y$/.c/
-            s/\.gperf$/.h/
-          '
-        `
-        insert_vc_ignore $ig "$f"
-
-        # For files like sys_stat.in.h and sys_time.in.h, record as
-        # ignorable the directory we might eventually create: sys/.
-        f=`echo "$copied"|sed 's/sys_.*\.in\.h$/sys/'`
-        insert_vc_ignore $ig "$f"
-      fi
-    done
+    }
   done
-}
+  IFS=$old_IFS
 
-
-# Create boot temporary directories to import from gnulib and gettext.
-rm -fr $bt $bt2 &&
-mkdir $bt $bt2 || exit
+  rm -f $tempbase.0 $tempbase.1
+  trap - 1 2 13 15
+fi
 
 # Import from gnulib.
 
 gnulib_tool_options="\
  --import\
  --no-changelog\
- --aux-dir $bt/$build_aux\
- --doc-base $bt/$doc_base\
+ --aux-dir $build_aux\
+ --doc-base $doc_base\
  --lib $gnulib_name\
- --m4-base $bt/$m4_base/\
- --source-base $bt/$source_base/\
- --tests-base $bt/$tests_base\
+ --m4-base $m4_base/\
+ --source-base $source_base/\
+ --tests-base $tests_base\
  --local-dir $local_gl_dir\
  $gnulib_tool_option_extras\
 "
@@ -850,25 +868,14 @@ if test $use_libtool = 1; then
 fi
 echo "$0: $gnulib_tool $gnulib_tool_options --import ..."
 $gnulib_tool $gnulib_tool_options --import $gnulib_modules &&
-slurp $bt || exit
 
 for file in $gnulib_files; do
-  symlink_to_dir "$GNULIB_SRCDIR" $file || exit
+  symlink_to_dir "$GNULIB_SRCDIR" $file \
+    || die "failed to symlink $file"
 done
 
-
-# Import from gettext.
-with_gettext=yes
-grep '^[	 ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \
-    with_gettext=no
-
-if test $with_gettext = yes; then
-  echo "$0: (cd $bt2; ${AUTOPOINT-autopoint}) ..."
-  cp configure.ac $bt2 &&
-  (cd $bt2 && ${AUTOPOINT-autopoint} && rm configure.ac) &&
-  slurp $bt2 $bt || exit
-fi
-rm -fr $bt $bt2 || exit
+bootstrap_post_import_hook \
+  || die "bootstrap_post_import_hook failed"
 
 # Remove any dangling symlink matching "*.m4" or "*.[ch]" in some
 # gnulib-populated directories.  Such .m4 files would cause aclocal to fail.
@@ -882,37 +889,31 @@ find "$m4_base" "$source_base" \
   -depth \( -name '*.m4' -o -name '*.[ch]' \) \
   -type l -xtype l -delete > /dev/null 2>&1
 
-# Reconfigure, getting other files.
-
-# Skip autoheader if it's not needed.
-grep -E '^[	 ]*AC_CONFIG_HEADERS?\>' configure.ac >/dev/null ||
-  AUTOHEADER=true
-
-for command in \
-  libtool \
-  "${ACLOCAL-aclocal} --force -I '$m4_base' $ACLOCAL_FLAGS" \
-  "${AUTOCONF-autoconf} --force" \
-  "${AUTOHEADER-autoheader} --force" \
-  "${AUTOMAKE-automake} --add-missing --copy --force-missing"
-do
-  if test "$command" = libtool; then
-    test $use_libtool = 0 \
-      && continue
-    command="${LIBTOOLIZE-libtoolize} -c -f"
-  fi
-  echo "$0: $command ..."
-  eval "$command" || exit
-done
+# Some systems (RHEL 5) are using ancient autotools, for which the
+# --no-recursive option had not been invented.  Detect that lack and
+# omit the option when it's not supported.  FIXME in 2017: remove this
+# hack when RHEL 5 autotools are updated, or when they become irrelevant.
+no_recursive=
+case $($AUTORECONF --help) in
+  *--no-recursive*) no_recursive=--no-recursive;;
+esac
 
+# Tell autoreconf not to invoke autopoint or libtoolize; they were run above.
+echo "running: AUTOPOINT=true LIBTOOLIZE=true " \
+    "$AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS"
+AUTOPOINT=true LIBTOOLIZE=true \
+    $AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS \
+  || die "autoreconf failed"
 
 # Get some extra files from gnulib, overriding existing files.
 for file in $gnulib_extra_files; do
   case $file in
   */INSTALL) dst=INSTALL;;
-  build-aux/*) dst=$build_aux/`expr "$file" : 'build-aux/\(.*\)'`;;
+  build-aux/*) dst=$build_aux/${file#build-aux/};;
   *) dst=$file;;
   esac
-  symlink_to_dir "$GNULIB_SRCDIR" $file $dst || exit
+  symlink_to_dir "$GNULIB_SRCDIR" $file $dst \
+    || die "failed to symlink $file"
 done
 
 if test $with_gettext = yes; then
@@ -928,7 +929,19 @@ if test $with_gettext = yes; then
       a\
           '"$XGETTEXT_OPTIONS"' $${end_of_xgettext_options+}
     }
-  ' po/Makevars.template >po/Makevars || exit 1
+  ' po/Makevars.template >po/Makevars \
+    || die 'cannot generate po/Makevars'
+
+  # If the 'gettext' module is in use, grab the latest Makefile.in.in.
+  # If only the 'gettext-h' module is in use, assume autopoint already
+  # put the correct version of this file into place.
+  case $gnulib_modules in
+  *gettext-h*) ;;
+  *gettext*)
+    cp $GNULIB_SRCDIR/build-aux/po/Makefile.in.in po/Makefile.in.in \
+      || die "cannot create po/Makefile.in.in"
+    ;;
+  esac
 
   if test -d runtime-po; then
     # Similarly for runtime-po/Makevars, but not quite the same.
@@ -942,7 +955,8 @@ if test $with_gettext = yes; then
         a\
             '"$XGETTEXT_OPTIONS_RUNTIME"' $${end_of_xgettext_options+}
       }
-    ' po/Makevars.template >runtime-po/Makevars || exit 1
+    ' po/Makevars.template >runtime-po/Makevars \
+    || die 'cannot generate runtime-po/Makevars'
 
     # Copy identical files from po to runtime-po.
     (cd po && cp -p Makefile.in.in *-quot *.header *.sed *.sin ../runtime-po)

bootstrap.conf

@@ -1,5 +1,5 @@
 # bootstrap.conf - Bootstrap configuration.
-# Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
 # Inc.
 #
 # This file is part of GNU Wget.
@@ -28,6 +28,7 @@ gnulib_modules="
 accept
 alloca
 announce-gen
+base32
 bind
 c-ctype
 clock-time
@@ -35,41 +36,51 @@ close
 connect
 fcntl
 futimens
+ftello
 getaddrinfo
 getopt-gnu
 getpass-gnu
 getpeername
 getsockname
+git-version-gen
 gnupload
 ioctl
+iconv
 iconv-h
 listen
 maintainer-makefile
 mbtowc
 mkdir
+mkstemp
+mkostemp
 crypto/md5
+crypto/sha1
 pipe
 quote
 quotearg
 recv
+regex
 select
 send
 setsockopt
+sigprocmask
 sigpipe
 snprintf
 socket
 stdbool
 strcasestr
 strerror_r-posix
+strtok_r
+tmpdir
 unlocked-io
 update-copyright
 vasprintf
+vsnprintf
 write
 "
 
 gnulib_extra_files="
 	$build_aux/install-sh
-	$build_aux/missing
 	$build_aux/mdate-sh
 	$build_aux/texinfo.tex
 	$build_aux/depcomp

build-aux/bzr-version-gen

@@ -1,57 +0,0 @@
-#!/bin/sh
-
-scriptversion=2011-08-09.13; # UTC
-
-# Copyright (C) 2010, 2011 Free Software Foundation, Inc.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# Additional permission under GNU GPL version 3 section 7
-
-
-# Written by Giuseppe Scrivano.
-
-if test -f .tarball-version
-then
-    cat .tarball-version | tr -d '\n'
-    exit 0
-fi
-
-DIRTY=""
-
-test -n "`bzr diff | tr -d '\n'`" && DIRTY="-dirty"
-
-REVNO=`bzr revno`
-
-TAG=`bzr tags -r $REVNO | cut -d' ' -f1`
-if test -z "$TAG"
-then
-    TAG=`bzr tags -r ..$REVNO | cut -d' ' -f1`
-
-    # No tags yet
-    test -z "$TAG" && TAG="unknown"
-
-    TAG=$TAG-$REVNO
-fi
-
-
-printf "%s%s" "$TAG" "$DIRTY"
-
-# Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:

configure.ac

@@ -31,7 +31,7 @@ dnl Process this file with autoconf to produce a configure script.
 dnl
 
 AC_INIT([wget],
-  [m4_esyscmd([build-aux/bzr-version-gen])],
+        m4_esyscmd([build-aux/git-version-gen .tarball-version]),
 	[bug-wget@gnu.org])
 AC_PREREQ(2.61)
 
@@ -62,7 +62,11 @@ dnl Process features.
 dnl
 
 AC_ARG_WITH(ssl,
-[[  --without-ssl           disable SSL autodetection]])
+[[  --without-ssl           disable SSL autodetection
+  --with-ssl={gnutls,openssl} specify the SSL backend.  GNU TLS is the default.]])
+
+AC_ARG_WITH(zlib,
+[[  --without-zlib          disable zlib ]])
 
 AC_ARG_ENABLE(opie,
 [  --disable-opie          disable support for opie or s/key FTP login],
@@ -194,9 +198,9 @@ dnl Checks for library functions.
 dnl
 AC_FUNC_MMAP
 AC_FUNC_FSEEKO
-AC_CHECK_FUNCS(strptime timegm vsnprintf vasprintf drand48)
+AC_CHECK_FUNCS(strptime timegm vsnprintf vasprintf drand48 pathconf)
 AC_CHECK_FUNCS(strtoll usleep ftello sigblock sigsetjmp memrchr wcwidth mbtowc)
-AC_CHECK_FUNCS(sleep symlink)
+AC_CHECK_FUNCS(sleep symlink utime)
 
 if test x"$ENABLE_OPIE" = xyes; then
   AC_LIBOBJ([ftp-opie])
@@ -233,90 +237,124 @@ dnl
 dnl Checks for libraries.
 dnl
 
+AS_IF([test x"$with_zlib" != xno], [
+  with_zlib=yes
+  AC_CHECK_LIB(z, compress)
+])
+
 AS_IF([test x"$with_ssl" = xopenssl], [
-    dnl some versions of openssl use zlib compression
-    AC_CHECK_LIB(z, compress)
+  dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
+  dnl doesn't record its dependency on libdl, so we need to make sure
+  dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
+  dnl dlopen(), but HP-UX uses shl_load().
+  AC_CHECK_LIB(dl, dlopen, [], [
+    AC_CHECK_LIB(dl, shl_load)
+  ])
 
-    dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
-    dnl doesn't record its dependency on libdl, so we need to make sure
-    dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
-    dnl dlopen(), but HP-UX uses shl_load().
-    AC_CHECK_LIB(dl, dlopen, [], [
-      AC_CHECK_LIB(dl, shl_load)
-    ])
-
-    case $host_os in
-      *mingw32* )
-        dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
+  ssl_found=no
+  case $host_os in
+    *mingw32* )
+      dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
         
-        AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
-        if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
-        then
-          AC_CHECK_LIB(ssl32, SSL_connect,,
-                    AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
+      AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
+      if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
+      then
+        AC_CHECK_LIB(ssl32, SSL_connect, [
+          ssl_found=yes
           AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
+          AC_LIBOBJ([openssl])
+          LIBS="${LIBS} -lssl32"
+          AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
+        ],
+        AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
           
-        else
-          LIBS+=' -lgdi32'
-          dnl fallback and test static libs
-          
-          AC_CHECK_LIB(crypto, EVP_MD_CTX_init,,,)
-          if test x"$ac_cv_lib_crypto_EVP_MD_CTX_init" != xno
-          then
-            AC_CHECK_LIB(ssl, SSL_connect,, AC_MSG_ERROR([openssl not found]))
-            AC_MSG_NOTICE([Enabling support for SSL via OpenSSL])
-          else
-            AC_MSG_ERROR([openssl not found: shared (eay32/ssl32) or static (crypto/ssl) libs needed])
-          fi
-          
-        fi
-        dnl add zdll lib as dep for above tests?
-      ;;
-      *)
+      else
+        LIBS+=' -lgdi32'
+        dnl fallback and test static libs
+      fi
+      dnl add zdll lib as dep for above tests?
+    ;;
+  esac
 
-        AC_CHECK_LIB(crypto, EVP_MD_CTX_init)
-        AC_CHECK_LIB(ssl, ERR_func_error_string,,
-                  AC_MSG_ERROR([openssl development libraries not found]))
-        AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (static)])
-      ;;
-    esac
-    AC_LIBOBJ([openssl])
-], [
-  # --with-ssl is not gnutls: check if it's no
-  AS_IF([test x"$with_ssl" != xno], [
-    dnl Now actually check for -lssl
-
-    AC_CHECK_LIB(z, compress)
-    AC_CHECK_LIB(gpg-error, gpg_err_init)
-    AC_CHECK_LIB(gcrypt, gcry_control)
-    AC_CHECK_LIB(gnutls, main)
-
-    if test x"$ac_cv_lib_gnutls_main" != xno
+  AS_IF([test x$ssl_found != xyes], [
+    dnl Now actually check for -lssl if it wasn't already found
+    AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+#include <openssl/err.h>
+#include <openssl/rand.h>
+#include <openssl/des.h>
+#include <openssl/md4.h>
+#include <openssl/md5.h>
+    ], [SSL_library_init ()])
+    if test x"$LIBSSL" != x
     then
+      ssl_found=yes
+      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+      AC_LIBOBJ([openssl])
+      LIBS="$LIBSSL $LIBS"
+    elif test x"$with_ssl" != x
+    then
+      AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
+    fi
+  ])
+
+], [
+  # --with-ssl is not openssl: check if it's no
+  AS_IF([test x"$with_ssl" != xno], [
+    dnl default is -lgnutls
+    with_ssl=gnutls
+
+    dnl Now actually check for -lgnutls
+    AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
+#include <gnutls/gnutls.h>
+    ], [gnutls_global_init()])
+    if test x"$LIBGNUTLS" != x
+    then
+      ssl_found=yes
       AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
       AC_LIBOBJ([gnutls])
+      LIBS="$LIBGNUTLS $LIBS"
     else
-      AC_MSG_ERROR([--with-ssl was given, but GNUTLS is not available.])
+      AC_MSG_ERROR([--with-ssl=gnutls was given, but GNUTLS is not available.])
     fi
-  ]) # endif: --with-ssl == no?
+
+    AC_CHECK_FUNCS(gnutls_priority_set_direct)
+  ]) # endif: --with-ssl != no?
 ]) # endif: --with-ssl == openssl?
 
-
 dnl Enable NTLM if requested and if SSL is available.
-if test x"$LIBSSL" != x
+if test x"$LIBSSL" != x || test "$ac_cv_lib_ssl32_SSL_connect" = yes
 then
   if test x"$ENABLE_NTLM" != xno
   then
+    ENABLE_NTLM=yes
     AC_DEFINE([ENABLE_NTLM], 1,
      [Define if you want the NTLM authorization support compiled in.])
     AC_LIBOBJ([http-ntlm])
   fi
 else
-  dnl If SSL is unavailable and the user explicitly requested NTLM,
-  dnl abort.
-  if test x"$ENABLE_NTLM" = xyes
+  AC_CHECK_LIB(nettle, nettle_md4_init, [HAVE_NETTLE=yes], [HAVE_NETTLE=no; AC_MSG_WARN(*** libnettle was not found. You will not be able to use NTLM)])
+
+  if test x"$HAVE_NETTLE" = xyes
   then
-    AC_MSG_ERROR([NTLM authorization requested and OpenSSL not found; aborting])
+    AC_SUBST(NETTLE_LIBS, "-lnettle")
+    AC_DEFINE([HAVE_NETTLE], [1], [Use libnettle])
+    if test x"$ENABLE_NTLM" != xno
+    then
+      ENABLE_NTLM=yes
+      AC_DEFINE([ENABLE_NTLM], 1,
+       [Define if you want the NTLM authorization support compiled in.])
+      AC_LIBOBJ([http-ntlm])
+      LIBS="$NETTLE_LIBS $LIBS"
+    fi
+  else
+    dnl If SSL is unavailable and the user explicitly requested NTLM,
+    dnl abort.
+    if test x"$ENABLE_NTLM" = xyes
+    then
+      AC_MSG_ERROR([NTLM authorization requested and SSL not enabled; aborting])
+    fi
   fi
 fi
 
@@ -493,7 +531,31 @@ if test "X$iri" != "Xno"; then
   fi
 fi
 
+dnl
+dnl Check for UUID
+dnl
 
+AC_CHECK_HEADER(uuid/uuid.h,
+                AC_CHECK_LIB(uuid, uuid_generate,
+                  [LIBS="${LIBS} -luuid"
+                   AC_DEFINE([HAVE_LIBUUID], 1,
+                             [Define if libuuid is available.])
+                  ])
+)
+
+dnl
+dnl Check for PCRE
+dnl
+
+AC_CHECK_HEADER(pcre.h,
+                AC_CHECK_LIB(pcre, pcre_compile,
+                  [LIBS="${LIBS} -lpcre"
+                   AC_DEFINE([HAVE_LIBPCRE], 1,
+                             [Define if libpcre is available.])
+                  ])
+)
+
+ 
 dnl Needed by src/Makefile.am
 AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
 
@@ -506,3 +568,20 @@ AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
                  lib/Makefile])
 AC_CONFIG_HEADERS([src/config.h])
 AC_OUTPUT
+
+AC_MSG_NOTICE([Summary of build options:
+
+  Version:           $PACKAGE_VERSION
+  Host OS:           $host_os
+  Install prefix:    $prefix
+  Compiler:          $CC
+  CFlags:            $CFLAGS $CPPFLAGS
+  LDFlags:           $LDFLAGS
+  Libs:              $LIBS
+  SSL:               $with_ssl
+  Zlib:              $with_zlib
+  Digest:            $ENABLE_DIGEST
+  NTLM:              $ENABLE_NTLM
+  OPIE:              $ENABLE_OPIE
+  Debugging:         $ENABLE_DEBUG
+])

doc/ChangeLog

@@ -1,3 +1,117 @@
+2013-12-29  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* wget.texi: Update to GFDL 1.3.
+
+2013-12-22  Mike Frysinger  <vapier@gentoo.org>
+
+	* sample.wgetrc: add links to the manual.
+
+2013-10-06  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi: add/explain quoting of wildcard patterns
+
+2013-09-04  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* sample.wgetrc: added "secureprotocol" example
+	* wget.texi (HTTPS (SSL/TLS) Options): Document PFS.
+
+2013-08-23  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* sample.wgetrc: added "httpsonly" example
+
+2013-08-22  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi: added description for --https-only
+
+2013-08-13  Hrvoje Niksic  <hniksic@xemacs.org>
+
+	* wget.texi (Download Options): Fix misspelling.
+
+2013-07-09  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Download Options): Add documentation for --backups.
+	(Wgetrc Commands): Add documentation for backups.
+	Reported by: Tomas Hozza <thozza@redhat.com>.
+
+2013-07-11  Tomas Hozza  <thozza@redhat.com>
+
+	* wget.texi: Document --regex-type and --preserve-permissions.
+
+2013-06-17  Dave Reisner  <dreisner@archlinux.org> (tiny change)
+
+	* texi2pod.pl: Fix formatting error that causes build to fail with
+	Perl 5.18
+
+2013-06-17  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi (POST): Explain the new redirection rules.
+	* wget.texi (Other HTTP Methods): Same.
+	* wget.texi (body-data): Fix typo in description.
+
+2013-05-10  Darshit Shah <darnir@gmail.com>  (tiny change)
+
+	* wget.texi (No of tries): Fix typo to make it clear that --tries
+	option sets number of tries not retries.
+	Reported by: Hauke Hoffman <haukebjoernhoffman@googlemail.com>
+
+2013-04-28  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Types of Files...): Document --accept-regex,
+	--reject-regex.
+	Reported by: Tomas Hozza <thozza@redhat.com>.
+
+2013-04-14  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Download Options): Document "mega" dot style.
+
+2013-04-05  Darshit Shah <darnir@gmail.com>
+
+	* doc/wget.texi: Fix ambiguous wording in --post-data section. Make it
+	clear that wget does not check for the format of the post-data.
+	* doc/wget.texi: Add documentation for --method, --body-data and
+	--body-file.
+
+2012-10-08  Stefano Lattarini <stefano.lattarini@gmail.com>  (tiny change)
+
+	docs: fix errors and warnings with Texinfo 5
+	Or rather, with the development version 4.13.90, which will
+	eventually become Texinfo 5.0.
+	* wget.texi: Use '@item' instead of '@itemx' in several places,
+	as Texinfo 5 refuses to process an '@itemx' that is not preceded
+	by an '@item'.
+
+2012-08-28  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+        * doc/wget.texi: remove -nv from --report-speed
+
+2012-08-04  mancha <mancha@mac.hush.com> (tiny change)
+
+	* wget.texi: Export ENVIRONMENT to the man page.
+
+2012-06-09  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Logging and Input File Options): Document "--report-speed".
+	(HTTPS (SSL/TLS) Options): Document WARC.
+
+	* texi2pod.pl: Revert change from 2011-08-06.
+
+2012-05-13  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Types of Files): Document --accept-regex and
+	--reject-regex.
+
+2011-10-02  Henrik Holst <henrik.holst@millistream.com> (tiny change)
+
+	* wget.texi (HTTP Options): Document option --content-on-error.
+
+2011-09-27  Karl Berry <karl@freefriends.org> (tiny change)
+
+	* wget.texi: Make dir entry consistent with others.
+
+2011-08-18  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* texi2pod.pl: Don't assume the perl executable is under /usr/bin/.
+
 2011-08-06  Giuseppe Scrivano  <gscrivano@gnu.org>
 
 	* wget.texi (Wgetrc Commands): Document show_all_dns_entries.

doc/sample.wgetrc

@@ -5,7 +5,10 @@
 ## You can use this file to change the default behaviour of wget or to
 ## avoid having to type many many command-line options. This file does
 ## not contain a comprehensive list of commands -- look at the manual
-## to find out what you can put into this file.
+## to find out what you can put into this file. You can find this here:
+##   $ info wget.info 'Startup File'
+## Or online here:
+##   https://www.gnu.org/software/wget/manual/wget.html#Startup-File
 ##
 ## Wget initialization file can reside in /usr/local/etc/wgetrc
 ## (global, for all users) or $HOME/.wgetrc (for a single user).
@@ -123,3 +126,9 @@
 
 # Force the default remote server encoding
 #remoteencoding = UTF-8
+
+# Turn on to prevent following non-HTTPS links when in recursive mode
+#httpsonly = off
+
+# Tune HTTPS security (auto, SSLv2, SSLv3, TLSv1, PFS)
+#secureprotocol = auto

doc/texi2pod.pl

@@ -1,6 +1,7 @@
-#! /usr/bin/perl -w
+#! /usr/bin/env perl
 
-#   Copyright (C) 1999, 2000, 2001, 2003, 2010 Free Software Foundation, Inc.
+#   Copyright (C) 1999, 2000, 2001, 2003, 2007, 2009, 2010, 2011 Free
+#   Software Foundation, Inc.
 
 # This file is part of GCC.
 
@@ -15,14 +16,15 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with GCC; see the file COPYING.  If not, write to
-# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
-# Boston MA 02110-1301, USA.
+# along with GCC.  If not, see <http://www.gnu.org/licenses/>.
 
 # This does trivial (and I mean _trivial_) conversion of Texinfo
 # markup to Perl POD format.  It's intended to be used to extract
 # something suitable for a manpage from a Texinfo document.
 
+use warnings;
+BEGIN { eval { require warnings; } and warnings->import; }
+
 $output = 0;
 $skipping = 0;
 %sects = ();
@@ -36,7 +38,6 @@ $shift = "";
 $fnno = 1;
 $inf = "";
 $ibase = "";
-@ipath = ();
 
 while ($_ = shift) {
     if (/^-D(.*)$/) {
@@ -52,13 +53,6 @@ while ($_ = shift) {
 	die "flags may only contain letters, digits, hyphens, dashes and underscores\n"
 	    unless $flag =~ /^[a-zA-Z0-9_-]+$/;
 	$defs{$flag} = $value;
-    } elsif (/^-I(.*)$/) {
-	if ($1 ne "") {
-	    $flag = $1;
-	} else {
-	    $flag = shift;
-	}
-        push (@ipath, $flag);
     } elsif (/^-/) {
 	usage();
     } else {
@@ -162,8 +156,6 @@ while(<$inf>) {
 	} elsif ($ended =~ /^(?:itemize|enumerate|[fv]?table)$/) {
 	    $_ = "\n=back\n";
 	    $ic = pop @icstack;
-	} elsif ($ended eq "multitable") {
-	    $_ = "\n=back\n";
 	} else {
 	    die "unknown command \@end $ended at line $.\n";
 	}
@@ -213,18 +205,14 @@ while(<$inf>) {
 
     # Now the ones that have to be replaced by special escapes
     # (which will be turned back into text by unmunge())
-    # Replace @@ before @{ and @} in order to parse @samp{@@} correctly.
     s/&/&amp;/g;
     s/\@\@/&at;/g;
     s/\@\{/&lbrace;/g;
     s/\@\}/&rbrace;/g;
-    s/\@`\{(.)\}/&$1grave;/g;
 
-    # Inside a verbatim block, handle @var, @samp and @url specially.
+    # Inside a verbatim block, handle @var specially.
     if ($shift ne "") {
 	s/\@var\{([^\}]*)\}/<$1>/g;
-	s/\@samp\{([^\}]*)\}/"$1"/g;
-	s/\@url\{([^\}]*)\}/<$1>/g;
     }
 
     # POD doesn't interpret E<> inside a verbatim block.
@@ -243,23 +231,17 @@ while(<$inf>) {
 	$inf = gensym();
 	$file = postprocess($1);
 
-	# Try cwd and $ibase, then explicit -I paths.
-	$done = 0;
-	foreach $path ("", $ibase, @ipath) {
-	    $mypath = $file;
-	    $mypath = $path . "/" . $mypath if ($path ne "");
-	    open($inf, "<" . $mypath) and ($done = 1, last);
-	}
-	die "cannot find $file" if !$done;
+	# Try cwd and $ibase.
+	open($inf, "<" . $file)
+	    or open($inf, "<" . $ibase . "/" . $file)
+		or die "cannot open $file or $ibase/$file: $!\n";
 	next;
     };
 
-    /^\@(?:section|unnumbered|unnumberedsec|center|heading)\s+(.+)$/
+    /^\@(?:section|unnumbered|unnumberedsec|center)\s+(.+)$/
 	and $_ = "\n=head2 $1\n";
     /^\@subsection\s+(.+)$/
 	and $_ = "\n=head3 $1\n";
-    /^\@subsubsection\s+(.+)$/
-	and $_ = "\n=head4 $1\n";
 
     # Block command handlers:
     /^\@itemize(?:\s+(\@[a-z]+|\*|-))?/ and do {
@@ -268,7 +250,7 @@ while(<$inf>) {
 	if (defined $1) {
 	    $ic = $1;
 	} else {
-	    $ic = '*';
+	    $ic = '@bullet';
 	}
 	$_ = "\n=over 4\n";
 	$endw = "itemize";
@@ -286,12 +268,6 @@ while(<$inf>) {
 	$endw = "enumerate";
     };
 
-    /^\@multitable\s.*/ and do {
-	push @endwstack, $endw;
-	$endw = "multitable";
-	$_ = "\n=over 4\n";
-    };
-
     /^\@([fv]?table)\s+(\@[a-z]+)/ and do {
 	push @endwstack, $endw;
 	push @icstack, $ic;
@@ -301,7 +277,6 @@ while(<$inf>) {
 	$ic =~ s/\@(?:code|kbd)/C/;
 	$ic =~ s/\@(?:dfn|var|emph|cite|i)/I/;
 	$ic =~ s/\@(?:file)/F/;
-	$ic =~ s/\@(?:asis)//;
 	$_ = "\n=over 4\n";
     };
 
@@ -312,29 +287,14 @@ while(<$inf>) {
 	$_ = "";	# need a paragraph break
     };
 
-    /^\@item\s+(.*\S)\s*$/ and $endw eq "multitable" and do {
-	@columns = ();
-	for $column (split (/\s*\@tab\s*/, $1)) {
-	    # @strong{...} is used a @headitem work-alike
-	    $column =~ s/^\@strong{(.*)}$/$1/;
-	    push @columns, $column;
-	}
-	$_ = "\n=item ".join (" : ", @columns)."\n";
-    };
-
     /^\@itemx?\s*(.+)?$/ and do {
 	if (defined $1) {
-            if ($ic) {
-		if ($endw eq "enumerate") {
-		    $_ = "\n=item $ic $1\n";
-		    $ic =~ s/(\d+)/$1 + 1/eg;
-		} else {
-		    # Entity escapes prevent munging by the <>
-		    # processing below.
-		    $_ = "\n=item $ic\&LT;$1\&GT;\n";
-		}
+            my $thing = $1;
+            if ($ic =~ /\@asis/) {
+                $_ = "\n=item C<$thing>\n";
             } else {
-                $_ = "\n=item $1\n";
+                # Entity escapes prevent munging by the <> processing below.
+                $_ = "\n=item $ic\&LT;$thing\&GT;\n";
             }
 	} else {
 	    $_ = "\n=item $ic\n";
@@ -355,11 +315,12 @@ die "No filename or title\n" unless defined $fn && defined $tl;
 $sects{NAME} = "$fn \- $tl\n";
 $sects{FOOTNOTES} .= "=back\n" if exists $sects{FOOTNOTES};
 
-for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT FILES
-	      BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) {
+for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT EXITSTATUS
+           FILES BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) {
     if(exists $sects{$sect}) {
 	$head = $sect;
 	$head =~ s/SEEALSO/SEE ALSO/;
+	$head =~ s/EXITSTATUS/EXIT STATUS/;
 	print "=head1 $head\n\n";
 	print scalar unmunge ($sects{$sect});
 	print "\n";
@@ -391,13 +352,11 @@ sub postprocess
     s/\@r\{([^\}]*)\}/R<$1>/g;
     s/\@(?:dfn|var|emph|cite|i)\{([^\}]*)\}/I<$1>/g;
     s/\@(?:code|kbd)\{([^\}]*)\}/C<$1>/g;
-    s/\@(?:samp|strong|key|option|env|command|b)\{([^\}]*)\}/B<$1>/g;
+    s/\@(?:gccoptlist|samp|strong|key|option|env|command|b)\{([^\}]*)\}/B<$1>/g;
     s/\@sc\{([^\}]*)\}/\U$1/g;
-    s/\@acronym\{([^\}]*)\}/\U$1/g;
     s/\@file\{([^\}]*)\}/F<$1>/g;
     s/\@w\{([^\}]*)\}/S<$1>/g;
     s/\@(?:dmn|math)\{([^\}]*)\}/$1/g;
-    s/\@\///g;
 
     # keep references of the form @ref{...}, print them bold
     s/\@(?:ref)\{([^\}]*)\}/B<$1>/g;
@@ -419,9 +378,6 @@ sub postprocess
     s/\@gol//g;
     s/\@\*\s*\n?//g;
 
-    # Anchors are thrown away
-    s/\@anchor\{(?:[^\}]*)\}//g;
-
     # @uref can take one, two, or three arguments, with different
     # semantics each time.  @url and @email are just like @uref with
     # one argument, for our purposes.
@@ -429,10 +385,6 @@ sub postprocess
     s/\@uref\{([^\},]*),([^\},]*)\}/$2 (C<$1>)/g;
     s/\@uref\{([^\},]*),([^\},]*),([^\},]*)\}/$3/g;
 
-    # Handle gccoptlist here, so it can contain the above formatting
-    # commands.
-    s/\@gccoptlist\{([^\}]*)\}/B<$1>/g;
-
     # Un-escape <> at this point.
     s/&LT;/</g;
     s/&GT;/>/g;
@@ -466,7 +418,6 @@ sub unmunge
     # Replace escaped symbols with their equivalents.
     local $_ = $_[0];
 
-    s/&(.)grave;/E<$1grave>/g;
     s/&lt;/E<lt>/g;
     s/&gt;/E<gt>/g;
     s/&lbrace;/\{/g;

doc/wget.texi

@@ -20,9 +20,9 @@
 @set Wget Wget
 @c man title Wget The non-interactive network downloader.
 
-@dircategory Network Applications
+@dircategory Network applications
 @direntry
-* Wget: (wget).         The non-interactive network downloader.
+* Wget: (wget).                 Non-interactive network downloader.
 @end direntry
 
 @copying
@@ -47,11 +47,11 @@ notice identical to this one except for the removal of this paragraph
 (this paragraph not being relevant to the printed manual).
 @end ignore
 Permission is granted to copy, distribute and/or modify this document
-under the terms of the GNU Free Documentation License, Version 1.2 or
+under the terms of the GNU Free Documentation License, Version 1.3 or
 any later version published by the Free Software Foundation; with no
-Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts.  A
-copy of the license is included in the section entitled ``GNU Free
-Documentation License''.
+Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+Texts.  A copy of the license is included in the section entitled
+``GNU Free Documentation License''.
 @c man end
 @end copying
 
@@ -479,6 +479,9 @@ Turn off verbose without being completely quiet (use @samp{-q} for
 that), which means that error messages and basic information still get
 printed.
 
+@item --report-speed=@var{type}
+Output bandwidth as @var{type}.  The only accepted value is @samp{bits}.
+
 @cindex input-file
 @item -i @var{file}
 @itemx --input-file=@var{file}
@@ -547,10 +550,10 @@ IPs.
 
 @cindex retries
 @cindex tries
-@cindex number of retries
+@cindex number of tries
 @item -t @var{number}
 @itemx --tries=@var{number}
-Set number of retries to @var{number}.  Specify 0 or @samp{inf} for
+Set number of tries to @var{number}. Specify 0 or @samp{inf} for
 infinite retrying.  The default is to retry 20 times, with the exception
 of fatal errors like ``connection refused'' or ``not found'' (404),
 which are not retried.
@@ -627,6 +630,13 @@ Note that when @samp{-nc} is specified, files with the suffixes
 @samp{.html} or @samp{.htm} will be loaded from the local disk and
 parsed as if they had been retrieved from the Web.
 
+@cindex backing up files
+@item --backups=@var{backups}
+Before (over)writing a file, back up an existing file by adding a
+@samp{.1} suffix (@samp{_1} on VMS) to the file name.  Such backup
+files are rotated to @samp{.2}, @samp{.3}, and so on, up to
+@var{backups} (and lost beyond that).
+
 @cindex continue retrieval
 @cindex incomplete downloads
 @cindex resume download
@@ -712,9 +722,12 @@ different meaning to one dot.  With the @code{default} style each dot
 represents 1K, there are ten dots in a cluster and 50 dots in a line.
 The @code{binary} style has a more ``computer''-like orientation---8K
 dots, 16-dots clusters and 48 dots per line (which makes for 384K
-lines).  The @code{mega} style is suitable for downloading very large
+lines).  The @code{mega} style is suitable for downloading large
 files---each dot represents 64K retrieved, there are eight dots in a
 cluster, and 48 dots on each line (so each line contains 3M).
+If @code{mega} is not enough then you can use the @code{giga}
+style---each dot represents 1M retrieved, there are eight dots in a
+cluster, and 32 dots on each line (so each line contains 32M).
 
 Note that you can set the default style using the @code{progress}
 command in @file{.wgetrc}.  That setting may be overridden from the
@@ -729,7 +742,7 @@ Turn on time-stamping.  @xref{Time-Stamping}, for details.
 @item --no-use-server-timestamps
 Don't set the local file's timestamp by the one on the server.
 
-By default, when a file is downloaded, it's timestamps are set to
+By default, when a file is downloaded, its timestamps are set to
 match those from the remote file. This allows the use of
 @samp{--timestamping} on subsequent invocations of wget. However, it
 is sometimes useful to base the local file's timestamp on when it was
@@ -872,7 +885,7 @@ recommendation to block many unrelated users from a web site due to the
 actions of one.
 
 @cindex proxy
-@itemx --no-proxy
+@item --no-proxy
 Don't use proxies, even if the appropriate @code{*_proxy} environment
 variable is defined.
 
@@ -973,7 +986,7 @@ are outside the range of @sc{ascii} characters (that is, greater than
 whose encoding does not match the one used locally.
 
 @cindex IPv6
-@itemx -4
+@item -4
 @itemx --inet4-only
 @itemx -6
 @itemx --inet6-only
@@ -1454,6 +1467,11 @@ like everything else. Wget does not currently support
 @code{application/x-www-form-urlencoded}. Only one of
 @samp{--post-data} and @samp{--post-file} should be specified.
 
+Please note that wget does not require the content to be of the form
+@code{key1=value1&key2=value2}, and neither does it test for it. Wget will
+simply transmit whatever data is provided to it. Most servers however expect
+the POST data to be in the above format when processing HTML Forms.
+
 Please be aware that Wget needs to know the size of the POST data in
 advance.  Therefore the argument to @code{--post-file} must be a regular
 file; specifying a FIFO or something like @file{/dev/stdin} won't work.
@@ -1464,14 +1482,15 @@ use chunked unless it knows it's talking to an HTTP/1.1 server.  And it
 can't know that until it receives a response, which in turn requires the
 request to have been completed -- a chicken-and-egg problem.
 
-Note: if Wget is redirected after the POST request is completed, it
-will not send the POST data to the redirected URL.  This is because
-URLs that process POST often respond with a redirection to a regular
-page, which does not desire or accept POST.  It is not completely
-clear that this behavior is optimal; if it doesn't work out, it might
-be changed in the future.
+Note: As of version 1.15 if Wget is redirected after the POST request is
+completed, its behaviour will depend on the response code returned by the
+server.  In case of a 301 Moved Permanently, 302 Moved Temporarily or
+307 Temporary Redirect, Wget will, in accordance with RFC2616, continue
+to send a POST request.
+In case a server wants the client to change the Request method upon
+redirection, it should send a 303 See Other response code.
 
-This example shows how to log to a server using POST and then proceed to
+This example shows how to log in to a server using POST and then proceed to
 download the desired pages, presumably only accessible to authorized
 users:
 
@@ -1494,6 +1513,37 @@ them (and neither will browsers) and the @file{cookies.txt} file will
 be empty.  In that case use @samp{--keep-session-cookies} along with
 @samp{--save-cookies} to force saving of session cookies.
 
+@cindex Other HTTP Methods
+@item --method=@var{HTTP-Method}
+For the purpose of RESTful scripting, Wget allows sending of other HTTP Methods
+without the need to explicitly set them using @samp{--header=Header-Line}.
+Wget will use whatever string is passed to it after @samp{--method} as the HTTP
+Method to the server.
+
+@item --body-data=@var{Data-String}
+@itemx --body-file=@var{Data-File}
+Must be set when additional data needs to be sent to the server along with the
+Method specified using @samp{--method}.  @samp{--body-data} sends @var{string} as
+data, whereas @samp{--body-file} sends the contents of @var{file}.  Other than that,
+they work in exactly the same way.
+
+Currently, @samp{--body-file} is @emph{not} for transmitting files as a whole.
+Wget does not currently support @code{multipart/form-data} for transmitting data;
+only @code{application/x-www-form-urlencoded}. In the future, this may be changed
+so that wget sends the @samp{--body-file} as a complete file instead of sending its
+contents to the server. Please be aware that Wget needs to know the contents of
+BODY Data in advance, and hence the argument to @samp{--body-file} should be a
+regular file. See @samp{--post-file} for a more detailed explanation.
+Only one of @samp{--body-data} and @samp{--body-file} should be specified.
+
+If Wget is redirected after the request is completed, Wget will
+suspend the current method and send a GET request till the redirection
+is completed.  This is true for all redirection response codes except
+307 Temporary Redirect which is used to explicitly specify that the
+request method should @emph{not} change.  Another exception is when
+the method is set to @code{POST}, in which case the redirection rules
+specified under @samp{--post-data} are followed.
+
 @cindex Content-Disposition
 @item --content-disposition
 
@@ -1506,6 +1556,12 @@ This option is useful for some file-downloading CGI programs that use
 @code{Content-Disposition} headers to describe what the name of a
 downloaded file should be.
 
+@cindex Content On Error
+@item --content-on-error
+
+If this is set to on, wget will not skip the content when the server responds
+with a http status code that indicates error.
+
 @cindex Trust server names
 @item --trust-server-names
 
@@ -1539,16 +1595,24 @@ without SSL support, none of these options are available.
 @cindex SSL protocol, choose
 @item --secure-protocol=@var{protocol}
 Choose the secure protocol to be used.  Legal values are @samp{auto},
-@samp{SSLv2}, @samp{SSLv3}, and @samp{TLSv1}.  If @samp{auto} is used,
-the SSL library is given the liberty of choosing the appropriate
+@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1} and @samp{PFS}.  If @samp{auto}
+is used, the SSL library is given the liberty of choosing the appropriate
 protocol automatically, which is achieved by sending an SSLv2 greeting
 and announcing support for SSLv3 and TLSv1.  This is the default.
 
 Specifying @samp{SSLv2}, @samp{SSLv3}, or @samp{TLSv1} forces the use
 of the corresponding protocol.  This is useful when talking to old and
-buggy SSL server implementations that make it hard for OpenSSL to
-choose the correct protocol version.  Fortunately, such servers are
-quite rare.
+buggy SSL server implementations that make it hard for the underlying
+SSL library to choose the correct protocol version.  Fortunately, such
+servers are quite rare.
+
+Specifying @samp{PFS} enforces the use of the so-called Perfect Forward
+Security cipher suites. In short, PFS adds security by creating a one-time
+key for each SSL connection. It has a bit more CPU impact on client and server.
+We use known to be secure ciphers (e.g. no MD4) and the TLS protocol.
+
+@item --https-only
+When in recursive mode, only HTTPS links are followed.
 
 @cindex SSL certificate, check
 @item --no-check-certificate
@@ -1652,6 +1716,36 @@ not used), EGD is never contacted.  EGD is not needed on modern Unix
 systems that support @file{/dev/random}.
 @end table
 
+@cindex WARC
+@table @samp
+@item --warc-file=@var{file}
+Use @var{file} as the destination WARC file.
+
+@item --warc-header=@var{string}
+Use @var{string} into as the warcinfo record.
+
+@item --warc-max-size=@var{size}
+Set the maximum size of the WARC files to @var{size}.
+
+@item --warc-cdx
+Write CDX index files.
+
+@item --warc-dedup=@var{file}
+Do not store records listed in this CDX file.
+
+@item --no-warc-compression
+Do not compress WARC files with GZIP.
+
+@item --no-warc-digests
+Do not calculate SHA1 digests.
+
+@item --no-warc-keep-log
+Do not store the log file in a WARC record.
+
+@item --warc-tempdir=@var{dir}
+Specify the location for temporary files created by the WARC writer.
+@end table
+
 @node FTP Options, Recursive Retrieval Options, HTTPS (SSL/TLS) Options, Invoking
 @section FTP Options
 
@@ -1737,6 +1831,10 @@ in some rare firewall configurations, active FTP actually works when
 passive FTP doesn't.  If you suspect this to be the case, use this
 option, or set @code{passive_ftp=off} in your init file.
 
+@cindex file permissions
+@item --preserve-permissions
+Preserve remote file permissions instead of permissions set by umask.
+
 @cindex symbolic links, retrieving
 @item --retr-symlinks
 Usually, when retrieving @sc{ftp} directories recursively and a symbolic
@@ -1973,6 +2071,17 @@ accept or reject (@pxref{Types of Files}). Note that if
 any of the wildcard characters, @samp{*}, @samp{?}, @samp{[} or
 @samp{]}, appear in an element of @var{acclist} or @var{rejlist},
 it will be treated as a pattern, rather than a suffix.
+In this case, you have to enclose the pattern into quotes to prevent
+your shell from expanding it, like in @samp{-A "*.mp3"} or @samp{-A '*.mp3'}.
+
+@item --accept-regex @var{urlregex}
+@itemx --reject-regex @var{urlregex}
+Specify a regular expression to accept or reject the complete URL.
+
+@item --regex-type @var{regextype}
+Specify the regular expression type.  Possible types are @samp{posix} or
+@samp{pcre}.  Note that to be able to use @samp{pcre} type, wget has to be
+compiled with libpcre support.
 
 @item -D @var{domain-list}
 @itemx --domains=@var{domain-list}
@@ -2021,8 +2130,10 @@ dedicated @samp{--page-requisites} option.
 Ignore case when matching files and directories.  This influences the
 behavior of -R, -A, -I, and -X options, as well as globbing
 implemented when downloading from FTP sites.  For example, with this
-option, @samp{-A *.txt} will match @samp{file1.txt}, but also
+option, @samp{-A "*.txt"} will match @samp{file1.txt}, but also
 @samp{file2.TXT}, @samp{file3.TxT}, and so on.
+The quotes in the example are to prevent the shell from expanding the
+pattern.
 
 @item -H
 @itemx --span-hosts
@@ -2278,6 +2389,8 @@ in @file{.wgetrc}.
 @item -A @var{acclist}
 @itemx --accept @var{acclist}
 @itemx accept = @var{acclist}
+@itemx --accept-regex @var{urlregex}
+@itemx accept-regex = @var{urlregex}
 The argument to @samp{--accept} option is a list of file suffixes or
 patterns that Wget will download during recursive retrieval.  A suffix
 is the ending part of a file, and consists of ``normal'' letters,
@@ -2294,6 +2407,9 @@ a description of how pattern matching works.
 Of course, any number of suffixes and patterns can be combined into a
 comma-separated list, and given as an argument to @samp{-A}.
 
+The argument to @samp{--accept-regex} option is a regular expression which
+is matched against the complete URL.
+
 @cindex reject wildcards
 @cindex reject suffixes
 @cindex wildcards, reject
@@ -2301,6 +2417,8 @@ comma-separated list, and given as an argument to @samp{-A}.
 @item -R @var{rejlist}
 @itemx --reject @var{rejlist}
 @itemx reject = @var{rejlist}
+@itemx --reject-regex @var{urlregex}
+@itemx reject-regex = @var{urlregex}
 The @samp{--reject} option works the same way as @samp{--accept}, only
 its logic is the reverse; Wget will download all files @emph{except} the
 ones matching the suffixes (or patterns) in the list.
@@ -2312,6 +2430,9 @@ Analogously, to download all files except the ones beginning with
 expansion by the shell.
 @end table
 
+The argument to @samp{--accept-regex} option is a regular expression which
+is matched against the complete URL.
+
 @noindent
 The @samp{-A} and @samp{-R} options may be combined to achieve even
 better fine-tuning of which files to retrieve.  E.g. @samp{wget -A
@@ -2780,9 +2901,11 @@ enables it).
 Enable/disable saving pre-converted files with the suffix
 @samp{.orig}---the same as @samp{-K} (which enables it).
 
-@c @item backups = @var{number}
-@c #### Document me!
-@c
+@item backups = @var{number}
+Use up to @var{number} backups for a file.  Backups are rotated by
+adding an incremental counter that starts at @samp{1}.  The default is
+@samp{0}.
+
 @item base = @var{string}
 Consider relative @sc{url}s in input files (specified via the
 @samp{input} command or the @samp{--input-file}/@samp{-i} option,
@@ -3044,7 +3167,7 @@ display properly---the same as @samp{-p}.
 Change setting of passive @sc{ftp}, equivalent to the
 @samp{--passive-ftp} option.
 
-@itemx password = @var{string}
+@item password = @var{string}
 Specify password @var{string} for both @sc{ftp} and @sc{http} file retrieval. 
 This command can be overridden using the @samp{ftp_password} and 
 @samp{http_password} command for @sc{ftp} and @sc{http} respectively.
@@ -3526,34 +3649,36 @@ internal networks from the rest of Internet.  In order to obtain
 information from the Web, their users connect and retrieve remote data
 using an authorized proxy.
 
+@c man begin ENVIRONMENT
 Wget supports proxies for both @sc{http} and @sc{ftp} retrievals.  The
 standard way to specify proxy location, which Wget recognizes, is using
 the following environment variables:
 
-@table @code
+@table @env
 @item http_proxy
 @itemx https_proxy
-If set, the @code{http_proxy} and @code{https_proxy} variables should
+If set, the @env{http_proxy} and @env{https_proxy} variables should
 contain the @sc{url}s of the proxies for @sc{http} and @sc{https}
 connections respectively.
 
 @item ftp_proxy
 This variable should contain the @sc{url} of the proxy for @sc{ftp}
-connections.  It is quite common that @code{http_proxy} and
-@code{ftp_proxy} are set to the same @sc{url}.
+connections.  It is quite common that @env{http_proxy} and
+@env{ftp_proxy} are set to the same @sc{url}.
 
 @item no_proxy
 This variable should contain a comma-separated list of domain extensions
 proxy should @emph{not} be used for.  For instance, if the value of
-@code{no_proxy} is @samp{.mit.edu}, proxy will not be used to retrieve
+@env{no_proxy} is @samp{.mit.edu}, proxy will not be used to retrieve
 documents from MIT.
 @end table
+@c man end
 
 In addition to the environment variables, proxy location and settings
 may be specified from within Wget itself.
 
 @table @samp
-@itemx --no-proxy
+@item --no-proxy
 @itemx proxy = on/off
 This option and the corresponding command may be used to suppress the
 use of proxy, even if the appropriate environment variables are set.
@@ -4265,7 +4390,7 @@ subscribers of the Wget mailing list.
 @appendix Copying this manual
   
 @menu
-* GNU Free Documentation License::  Licnse for copying this manual.
+* GNU Free Documentation License::  License for copying this manual.
 @end menu
 
 @node GNU Free Documentation License,  , Copying this manual, Copying this manual

lib/Makefile.am

@@ -1,18 +0,0 @@
-# GNU Wget
-
-# Copyright (C) 2010, 2011 Free Software Foundation, Inc.
-
-## This program is free software: you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published by
-## the Free Software Foundation, either version 3 of the License, or
-## (at your option) any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-include gnulib.mk

m4/wget.m4

@@ -51,8 +51,8 @@ dnl /usr/local/include (!), which GCC uses before /usr/include.
 
 AC_DEFUN([WGET_FNMATCH], [
   AC_MSG_CHECKING([for working fnmatch.h])
-  AC_COMPILE_IFELSE([#include <fnmatch.h>
-                    ], [
+  AC_COMPILE_IFELSE([AC_LANG_SOURCE([#include <fnmatch.h>
+                    ])], [
     AC_MSG_RESULT(yes)
     AC_DEFINE([HAVE_WORKING_FNMATCH_H], 1,
               [Define if fnmatch.h can be included.])

po/POTFILES.in

@@ -5,8 +5,12 @@
 
 # Package source files
 lib/error.c
+lib/gai_strerror.c
 lib/getopt.c
 lib/quotearg.c
+lib/spawn-pipe.c
+lib/w32spawn.h
+lib/wait-process.c
 lib/xalloc-die.c
 src/connect.c
 src/convert.c
@@ -32,3 +36,4 @@ src/retr.c
 src/spider.c
 src/url.c
 src/utils.c
+src/warc.c

src/ChangeLog

@@ -1,3 +1,919 @@
+2014-01-05  Håkon Vågsether <hauk142@gmail.com> (tiny change)
+
+	* http.c (http_loop): Fix checking the URL length when filename is
+	specified.
+
+2013-12-29  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* init.c (home_dir): Remove useless 'if'.
+	* warc.c (warc_start_new_file): Likewise.
+	(warc_process_cdx_line): Likewise.
+	(warc_write_response_record): Likewise.
+
+2013-12-26  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (ssl_connect_wget): Fix connect timeout failure
+
+2013-11-02  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* http.c (gethttp): Increase max header value length to 512.
+
+2013-01-30  Pavel Mateja  <pavel@netsafe.cz> (tiny change)
+
+	* http.c (gethttp): Specify "Host" for CONNECT method.
+
+2013-10-30  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* http.c (skip_short_body): Remove assert which is always true.
+	Reported by: David Binderman <dcb314@hotmail.com>
+
+2013-10-17  Andrea Urbani  <matfanjol@mail.com>
+
+	* ftp.c (getftp): force "LIST" or "LIST -a"
+	  according to the remote system type. If the remote
+	  system is not known, it tries, only the first time,
+	  "LIST -a", after "LIST" and decides which one to use.
+	  For more information look for "__LIST_A_EXPLANATION__"
+	* ftp.h (enum ustype): New ustype enum.
+	* ftp.h (ftp_syst): New enum ustype *unix_type parameter.
+	* ftp.h (ftp_list): Removed enum stype rs parameter, added.
+	  bool avoid_list_a, bool avoid_list, bool *list_a_used
+	  parameters.
+	* ftp.h (wget_ftp_fstatus): New AVOID_LIST_A, AVOID_LIST,
+	  LIST_AFTER_LIST_A_CHECK_DONE values.
+	* ftp-basic.c (ftp_list): it handles the new avoid_list_a,
+	  avoid_list and list_a_used parameters.
+	* ftp.h (ftp_syst): it stores information about the "215 UNIX"
+	  systems into the new unix_type parameter.
+
+2013-10-26  Bykov Aleksey <gnfalex@rambler.ru>
+
+	* utils.c (match_tail): Fix cookies reject
+
+	* ftp-ls.c (ftp_parse_unix_ls): Fix parsing month name in uppercase
+
+2013-09-09  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (ssl_connect_wget): changed checking of option "PFS"
+	  to be better prepared for some kinds of backports.
+	  Reported by: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
+
+2013-10-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* url.c (url_parse): Try to convert UTF-8 URLs to IDN.
+	* html-url.c (append_url): Parse URLs specifying an IRI structure.
+
+2013-09-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* recur.c (download_child_p): fix compile error when
+	  configured using --without-ssl.
+
+2013-09-07  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (ssl_connect_wget): use gnutls_check_version()
+	  to check if option "PFS" is available
+	  Reported by: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
+
+2013-09-03  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* main.c: Add new value 'PFS' to --secure-protocol to
+	  enforce the so-called Perfect Forward Security.
+	* init.c (cmd_spec_secure_protocol): added secure_protocol_pfs
+	* openssl.c, gnutls.c, options.h: likewise
+
+2013-08-22  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* main.c: Add new option --https-only.
+	* options.h: Likewise.
+	* recur.c (download_child_p): add check for HTTPS.
+
+2013-08-09  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (ssl_init): Prevent CA files from being loaded twice
+	  if possible.
+	* gnutls.c (ssl_check_certificate): Added some error messages
+	* gnutls.c: Fixed some compiler warnings
+
+2013-08-08  Will Dietz  <w@wdtz.org> (tiny change):
+
+	* main.c (format_and_print_line): Wrap correctly long tokens.
+
+2013-07-16  Darshit Shah  <darnir@gmail.com>
+
+	* wget.h (err_t): Added new errors, ATTRMISSING and UNKNOWNATTR to
+	handle missing attributes and Unknown attribute values respectively in
+	HTTP Headers.
+	* exits.c (get_status_for_err): ATTRMISSING is a Protocol Error while
+	UNKNOWNATTR is a general error, presumably because of a feature that
+	is not yet implemented.
+	* http.c (gethttp): Call create_authorization_line () separately. In
+	case the auth_err flag has been set with an error, handle it and exit.
+	* http.c (create_authorization_line): Pass a pointer, auth_err to set
+	the flag for different kinds of errors encountered.
+	* http.c (http_loop): Handle the errors raised by the authentication
+	handlers.
+	* http.c (digest_authentication_encode): Pass pointer auth_err to set
+	the error flags.
+	Set qop to NULL in case the value of the qop / algorithm attribute is
+	unknown to Wget. Set an	appropriate error too.
+
+2013-07-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* http.c (digest_authentication_encode): Fix a crash when the algorithm
+	is not specified in the server response.  Free dynamic memory used by
+	the function when the function exits.
+	* http-ntlm.c [HAVE_NETTLE]: Include <nettle/md4.h> and <nettle/des.h>.
+	(setup_des_key) [HAVE_NETTLE]: New function to deal with
+	libnettle.
+	(calc_resp) [HAVE_NETTLE]: Add support for libnettle.
+	(mkhash) [HAVE_NETTLE]: Likewise.
+	Reported by: Tim Ruehsen  <tim.ruehsen@gmx.de>.
+
+2013-07-13  Steven M. Schweda <sms@antinode.info>
+
+	* warc.c (warc_tempfile): Fix a portability issue on VMS.
+
+2013-07-10  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (read_response_body) [ALLOW_CLOBBER]: Move definition to..
+	* options.h (struct options): Make `backups' an int.
+	* url.h [ALLOW_CLOBBER]: .. Here.  Do not clobber when backups are used.
+	* url.c (url_file_name): Use the ALLOW_CLOBBER macro instead of
+	repeating the code.
+
+2013-07-08  Steven M. Schweda <sms@antinode.info>
+
+	* retr.c (rotate_backups): Support for VMS files.
+
+2013-07-12  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (digest_authentication_encode): Set default value of
+	`algorithm' to "MD5".  Check if `qop' is not-NULL before access it.
+
+2013-07-11  Karsten Hopp  <karsten@redhat.com>
+
+	* openssl.c (struct openssl_read_args, struct scwt_context): New struct.
+	(openssl_read, ssl_connect_with_timeout_callback): New function.
+	(ssl_connect_wget): respect connect timeout.
+
+2013-07-11  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+        * gnutls.c (ssl_connect_wget): respect connect timeout.
+
+2013-07-11  Tomas Hozza  <thozza@redhat.com>
+
+	* ftp.c (ftp_loop): Use ftp_retrieve_glob() also in case
+	--preserve-permissions was specified.
+
+2013-03-20  Tomas Hozza  <thozza@redhat.com>
+
+	* http.c (gethttp): Set "sock" to -1 if it's not and we have no
+	persistent connection
+
+2013-04-26  Tomas Hozza  <thozza@redhat.com> (tiny change)
+
+	* log.c (redirect_output): Use DEFAULT_LOGFILE in diagnostic message
+	when `logfile' is NULL.
+	* utils.c (unique_create): Ensure `logfile' has always a value.
+
+2013-06-26  Darshit Shah <darnir@gmail.com>
+
+	* http.c (gethttp): Reverse change by commit 90896 that prevented
+	downloading response body data to non-GET Request methods.
+
+2013-06-19 Ciprian Vieru <devel.php@gmail.com> (tiny change)
+
+        * html-url.c: Define TAG_TD, TAG_TH, TAG_VIDEO, TAG_AUDIO, TAG_SOURCE.
+	(tag_url_attributes, known_tags): Likewise.
+
+2013-06-22  Ángel González <keisial@gmail.com>
+
+	* init.c (cmd_string_uppercase): Rewrite function.
+
+2013-06-19  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* connect.c (socket_ip_address): zero out ip address structure to
+	avoid access to uninitialized values by inet_ntop().
+	* ftp.c (ftp_loop_internal): fix segfault caused by warc_tmp NULL pointer.
+
+2013-05-21  Ray Satiro  <raysatiro@yahoo.com>
+
+	* url.c (url_file_name): Use MAX_PATH in Windows.
+
+2013-06-13  Darshit Shah  <darnir@gmail.com>
+
+	* http.c (gethttp): Follow RFC 2616 and httpbis specifications when
+	handling redirections. Do not suspend the method on 301/302 redirects.
+	(gethttp): If method if not GET, we do not intend to download
+	anything.
+	* main.c (main): Set spider mode when opt.method is HEAD. This will
+	prevent Wget from downloading any file.
+	* retr.c (SUSPEND_METHOD): Rename macro SUSPEND_POST_DATA to
+	SUSPEND_METHOD to more accurately reflect its use. Similarly rename
+	related variables.
+
+2013-05-14 Bykov Aleksey <gnfalex@rambler.ru>
+
+	* warc.c (warc_tempfile): For fix "Could not open temporary WARC manifest
+	file." issue in Win system force to use `mkostemp(filename, O_TEMPORARY)'
+	instead  of `mkstemp(filename)'. Thank to Angel Gonzalez for help.
+
+2013-05-14  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* cookies.c (cookie_jar_load): Replaced read_whole_file() by getline().
+	* init.c (run_wgetrc): Likewise.
+	* netrc.c (parse_netrc): Likewise.
+	* utils.c: Likewise.
+	* ftp.c (getftp): Likewise.
+	* ftp-ls.c (ftp_parse_unix_ls, ftp_parse_winnt_ls, ftp_parse_vms_ls): Likewise.
+	* ftp-ls.c (clean_line): Accept the string length as parameter.
+	* ftp-ls.c: Replaced indent tabs by spaces.
+	* ftp.c: Likewise.
+	* utils.c: Removed read_whole_file() definition.
+	* netrc.c: Removed read_whole_file() definition for STANDALONE.
+	* utils.h: Removed read_whole_file() declaration.
+
+2013-05-09  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* utils.c (acceptable): use standard string functions instead of
+	self-written code.
+	(match_tail): Likewise.
+	(suffix): Likewise.
+	(has_wildcards_p): Likewise.
+	(test_subdir_p): Fix some warnings.
+	(test_dir_matches_p): Likewise.
+
+2013-05-05  mancha  <mancha1@hush.com> (tiny change)
+
+	* gnutls.c (ssl_connect_wget): Don't abort on non-fatal alerts
+        received during handshake. For example, when connecting to servers
+        using TSL-SNI that send warning-level unrecognized_name alerts.
+
+2013-05-04  Darshit Shah <darnir@gmail.com>
+
+	* init.c (cmd_string_uppercase): Fix issue that cased invalid headers
+	when converting to uppercase.
+
+2013-05-01  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* retr.c (retrieve_url): New variable `saved_method'.
+	(SUSPEND_POST_DATA): Save the method to `saved_method'.
+	(RESTORE_POST_DATA): Restore the method from `saved_method'.
+
+2013-05-01  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* init.c: Declare `cmd_string_uppercase'.
+	(commands): Now `method' uses cmd_string_uppercase.
+	(cmd_string_uppercase): New method
+	* http.c (gethttp): Do not transform opt.method to uppercase.
+	Reported by: Stefano Lattarini <stefano.lattarini@gmail.com>
+
+2013-04-24  Darshit Shah <darnir@gmail.com>
+
+	* http.c (gethttp): Remove check for opt.post_data and
+	opt.post_file_name.
+	* main.c (main): Change location in code where --post-data and
+	--post-file options are converted to --body-data --body-file.
+
+2013-04-21  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* http.c: Copy opt.body_data to the WARC file, instead of
+	opt.post_data (the old option).
+
+2013-04-12  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* warc.c: Generate unique UUIDs for the manifest and the record
+	holding the command-line arguments.
+	Write the manifest to a "metadata" record to follow the WARC
+	implementation guidelines.
+	* warc.h: Declare new function warc_write_metadata_record.
+
+2013-03-15  Darshit Shah <darnir@gmail.com>
+
+	* http.c (post_file): Rename function to body_file_send to more
+	accurately reflect its use.
+	* http.c (gethttp): Add support for --method, --body-data and
+	--body-file
+	* init.c (commands): Same.
+	* options.h (options): Same.
+	* main.c (option_data): Same.
+	* main.c (print_help): Add --method command.
+	* main.c (main): Make old --post-{data,file} commands aliases to
+	--method.
+	Add sanity checks for --method, --body-data and --body-file.
+	* retr.c (SUSPEND_POST_DATA): Edit Macro Definition to use body_data.
+	* retr.c (RESTORE_POST_DATA): Same.
+
+2013-03-31  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* warc.c: Correctly write the field length in the skip length field
+	of .warc.gz files. (Following the GZIP spec in RFC 1952.)
+
+2013-03-12  Darshit Shah <darnir@gmail.com>
+
+	* http.c (gethttp): Make wget return FILEBADFILE error and abort if
+	post-file does not exist.
+
+	* http.c (http_loop): Handle FILEBADFILE as a valid err.
+
+	* exits.c (get_status_for_err): Mark FILEBADFILE as an IO error.
+
+2013-02-15  Darshit Shah <darnir@gmail.com>
+
+	* cookies.c (cookie_handle_set_cookie): Set cookie->discard_requested
+	to true on domain mismatch.
+
+2012-12-20  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (ssl_connect_wget): added +VERS-SSL3.0 to fix
+	--secure-protocol=SSLv2/SSLv3.
+
+2012-12-09  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* main.c (main): Keep looking for "config" until there are
+	arguments to parse.
+	Reported by: Adrien Dumont <adr.dum@hotmail.fr>
+
+	* test.c: Include "wget.h".
+
+2012-12-08  Michael Stapelberg <michael@stapelberg.de> (tiny change)
+
+	* retr.c (retrieve_url): Set iri->orig_url to NULL after it is
+	freed.
+
+2012-11-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.h (MAX_INT_TO_STRING_LEN): Define macro.
+	* warc.c (warc_write_block_from_file): Use `MAX_INT_TO_STRING_LEN'
+	to find the buffer size.
+	(warc_write_cdx_record): Likewise.
+
+2012-09-03  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* http.c (digest_authentication_encode): Add support for RFC 2617
+	MD5-sess authentication algorithm.
+	Feature request and testing by: Avinash <pavinash@gmail.com>
+
+2012-11-14  Ángel González  <keisial@gmail.com>
+
+	* warc.c (warc_sha1_stream_with_payload): Fix compilation under
+	gcc -std=c89.
+
+2012-11-24  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* warc.c (warc_write_cdx_record): Use `number_to_string' to
+	convert the offset to a string.
+
+2012-11-24  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* warc.c (warc_write_block_from_file): Use `number_to_string' to
+	convert the content-length to a string.
+
+2012-11-15  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* retr.c (write_data): Fix comment.
+
+2012-11-13  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* retr.c (fd_read_body): Correctly check the return code from
+	write_data.
+	Reported by: Torsten Scheck <torsten.scheck@gmx.de>
+
+2012-08-29  Rohit Mathulla <rohit_mathulla@yahoo.com> (tiny change)
+
+	* html-url.c (get_urls_file): Convert shorthand URLs.
+
+2012-10-07  Ray Satiro <raysatiro@yahoo.com>
+
+	* url.c: Change the functions of a growable string object to null
+	terminate the string.
+	* url.c (append_null): New function to null terminate a growable
+	string object.
+	* url.c (shorten_length): New function to shorten the length of a
+	growable string object.
+
+2012-10-03 Merinov Nikolay <kim.roader@gmail.com>
+
+	* ftp.c (ftp_loop_internal): Ignore --no-clobber option when
+	receiving directory listing.
+
+2012-10-07  Tim Ruehsen  <tim.ruehsen@gmx.de>
+	    Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* utils.c (get_max_length): If `pathconf' is not available
+	fallback to PATH_MAX.
+
+2012-10-06  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (http_loop): Send a HEAD request when -c and
+	--content-disposition are used together.
+
+2012-09-29 Merinov Nikolay <kim.roader@gmail.com>
+
+	* ftp-ls.c (ftp_parse_winnt_ls): Support filename extracting with
+	new listing format.
+
+2012-09-29  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* url.h (CHOMP_BUFFER): Add definition.
+	* url.c (url_file_name): New local variables `fname_len_check' and
+	`max_length'. Check that the length of the file name is acceptable.
+	* utils.h (get_max_length): Declare function.
+	* utils.c (get_max_length): New function.
+
+2012-09-28  Steven Schubiger  <stsc@member.fsf.org>
+
+	* src/recur.c (retrieve_tree): Combine duplicated code.
+
+2012-09-02  Nguyễn Thái Ngọc Duy  <pclouds@gmail.com> (tiny change)
+
+	* src/main.c (main): mark more strings for translation.
+	* src/mswindows.c (fork_to_background): Likewise.
+	* src/recur.c (download_child_p): Likewise.
+
+2012-08-28  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (ssl_check_certificate): deinit gnutls_x509_crt_t.
+	* gnutls.c (ssl_init): don't error if CA directory is empty.
+
+2012-07-07  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* html-url.c (cleanup_html_url): Remove "static" modifier.
+	* init.c (cleanup_html_url): Likewise.
+	Reported by: Mike Frysinger <vapier@gentoo.org>.
+
+2012-08-25  Hrvoje Niksic  <hniksic@gmail.com>
+
+	* warc.c (warc_find_duplicate_cdx_record): Use hash_table_get
+	instead of hash_table_get_pair.
+
+2012-08-21  <y-iida@secom.co.jp> (tiny change)
+
+	* connect.c (connect_to_ip) [ENABLE_IPV6]: Attempt to use IPv6.
+	* http.c (gethttp): Likewise.
+
+2012-07-03  Steven Schubiger  <stsc@member.fsf.org>
+
+	* init.c: Include warc.h for warc_close in cleanup function.
+
+2012-07-08  Steven Schubiger  <stsc@member.fsf.org>
+
+	* exits.h: Fix comment.
+	* exits.c: Likewise.
+
+2012-07-07  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	(digest_authentication_encode): Add support for RFC 2617 Digest
+	Access Authentication.
+
+2012-07-07  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (http_loop): Fix log message.
+	* main.c (main): Likewise.
+	Reported by: Petr Pisar <petr.pisar@atlas.cz>
+
+2012-06-17  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.h: Define `CLOSEFAILED'.
+	* init.c: Include "exits.h".
+	(cleanup): Check `fclose' failure.
+	* exits.c (get_status_for_err): Handle `CLOSEFAILED'.
+
+2012-06-16  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* main.c (main): Move some cleanup related function to...
+	* init.c (cleanup): ...here.
+
+	* main.c: Do not include "stdout.h".
+	(main): Do not register `close_stdout' at exit.
+	Reported by: Micah Cowan  <micah@cowan.name>.
+
+2012-06-09  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* main.c (print_help): Move --report-speed under the section
+	"Logging and input file".
+
+2012-06-06  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* main.c (print_help): Rename --bits to --report-bps.
+	(cmdline_options): Likewise.
+	* init.c (commands): Rename --report-bps to --report-speed.
+	(cmd_spec_report_speed): New function.
+
+	* options.h (struct options): Rename `bits_fmt' to `report_bps'.
+	* main.c (print_help): Rename --bits to --report-bps.
+	(cmdline_options): Likewise.
+	* init.c (commands): Likewise
+
+	* progress.c (create_image): Adjust caller.
+	* retr.c (retr_rate): Likewise.
+	* utils.c (convert_to_bits): Likewise.
+
+2012-06-04  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* main.c (main): Check for filename != NULL.
+	* warc.c (warc_process_cdx_line): Fix memory leak.
+	* utils.c (match_posix_regex, compile_posix_regex): Remove dead
+   assignment.
+	* openssl.c (ssl_init): Fix old-style function definition.
+
+2012-06-02  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* connect.c: Include <sys/socket.h> and <sys/select.h>.
+
+2012-05-30  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* warc.c: Fix segfault if CDX record is not found.
+
+2011-05-26  Steven Schweda  <sms@antinode.info>
+	* connect.c [HAVE_SYS_SOCKET_H]: Include <sys/socket.h>.
+	[HAVE_SYS_SELECT_H]: Include <sys/select.h>.
+
+2012-05-26  Mike Frysinger <vapier@gentoo.org>
+
+	* warc.c: Change type of `warc_current_gzfile' to gzFile.
+
+2012-05-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* warc.c (warc_load_cdx_dedup_file): Change type of `line_length' to
+	ssize_t.
+	Suggested by: Ángel González <keisial@gmail.com>
+
+2012-05-18  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (wgnutls_poll): Honor the specified `timeout' value.
+	(wgnutls_peek): Likewise.
+
+2012-05-19  illusionoflife  <illusion.of.life92@gmail.com> (tiny change)
+
+	* convert.c (register_html,register_css): Fixed functions signature to
+	not accept unused argument
+	* retr.c (retrieve_url): Changed register_{css,html} usage according
+	new signature.
+
+2012-05-16  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* warc.h: Cut length lines to 80 columns.
+	* warc.c: Likewise.
+
+2012-05-14  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (wgnutls_read_timeout): removed warnings, moved fcntl stuff
+	outside loop.
+
+	* hash.h (hash_table_put): Make argument "value" const.
+	* hash.c (hash_table_put): Make argument value const.  Cast `value' to
+	void.
+	* http.c (request_set_header): Make argument `name' const.  Cast `value'
+	and `name' to void*.
+	(request_remove_header): Make argument `name' const.
+	* url.c (url_file_name): Make `index_filename' static.
+	* warc.h (warc_write_cdx_record): Make `url', `timestamp', `mime_type',
+	`payload_digest', `redirect_location', `warc_filename', response_uuid'
+	arguments const. Make `checksum' const.
+	* warc.c (warc_write_date_header): Make the `timestamp' argument const.
+	Make `extension' const.
+	(warc_write_cdx_record): Make `url', `timestamp', `mime_type',
+	`payload_digest', `redirect_location', `warc_filename', response_uuid'
+	arguments const. Make `checksum' const.
+
+2012-05-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* gnutls.c (credentials): Change type to
+	gnutls_certificate_credentials_t.
+	(ssl_init): Do not use deprecated types.
+	(ssl_connect_wget): Likewise.
+
+2012-04-11  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* init.c: Add --accept-regex, --reject-regex and --regex-type.
+	* main.c: Likewise.
+	* options.c: Likewise.
+	* recur.c: Likewise.
+	* utils.c: Add regex-related functions.
+	* utils.h: Add regex-related functions.
+
+2012-03-30  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* convert.c (convert_links_in_hashtable): Mmake it static.
+	* cookies.c (parse_set_cookie): Remove empty else branches.
+	* css-url.c: Include "css-url.h".
+	(get_uri_string): Make it static.
+	* css-url.h (get_urls_css): Add protoype.
+	* gnutls.c (ssl_init): Add prototype.
+	* html-parse.c (tagstack_push): Make it static.
+	* html-parse.c (tagstack_pop): Make it static.
+	* html-parse.c (tagstack_find): Make it static.
+	* html-url.c (cleanup_html_url): Make it static.
+	* progress.c (count_cols): Make it static.
+	* progress.c (get_eta): Make it static.
+	* retr.h (convert_to_bits): Remove prototype.
+	* util.h (convert_to_bits): Add prototype.
+	* spider.c (spider_cleanup): Make it static.
+	* warc.c (warc_write_start_record): Add prototype.
+	* warc.c (warc_write_end_record): Add prototype.
+	* warc.c (warc_start_cdx_file): Add prototype.
+	* warc.c (warc_init): Add prototype.
+	* warc.c (warc_load_cdx_dedup_file): Add prototype.
+	* warc.c (warc_write_metadata): Add prototype.
+	* warc.c (warc_close): Add prototype.
+	* warc.c (warc_tempfile): Add prototype.
+	* warc.c (warc_write_warcinfo_record): Make it static.
+	* warc.c (warc_load_cdx_dedup_file): Make it static.
+	* warc.c (warc_write_metadata): Make it static.
+	* warc.h (warc_init): Fix prototype.
+	* warc.h (warc_close): Fix prototype.
+	* warc.h (warc_tempfile): Fix prototype.
+
+2012-03-30  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* url.c: Use empty query in local filenames.
+
+2012-04-22  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* main.c (main): Dynamically allocate `opt.progress_type'.
+
+2012-04-21  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* ftp-basic.c (ftp_pasv): Fix memory leak.
+
+	* http.c (gethttp): Fix memory leak.
+
+	* ftp.c (getftp): Silent compiler warning.
+
+2009-06-14  Phil Pennock  <mutt-dev@spodhuis.org> (tiny change)
+	* host.h: Declare `is_valid_ip_address'.
+	* host.c (is_valid_ip_address): New function.
+	* http.c (gethttp): Specify the hostname to ssl_connect_wget.
+	* gnutls.c (ssl_connect_wget): Specify the server name.
+	* openssl.c (ssl_connect_wget): Likewise.
+	* ssl.h: Change method signature for ssl_connect_wget.
+
+2012-04-13  Tim Ruehsen  <tim.ruehsen@gmx.de> (tiny change)
+
+	* warc.c (warc_load_cdx_dedup_file): Fix a memory leak by freeing
+	`lineptr'.
+
+2012-04-07  Daniel Kahn Gillmor <dkg@fifthhorseman.net> (tiny change)
+
+	* gnutls.c (key_type_to_gnutls_type): New function.
+	(ssl_init): Use correctly the specified gnutls certificate.
+
+2012-04-01  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* html-url.c: Prevent crash on incomplete STYLE tag.
+
+2012-04-01  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* gnutls.c (wgnutls_read_timeout): Ensure timer is freed.
+
+	* gnutls.c (wgnutls_read_timeout): Do not use timer if it is not
+	allocated.
+	Reported by: Xu Zhongxing <xu_zhong_xing@163.com>
+
+2012-03-30  Tim Ruehsen  <tim.ruehsen@gmx.de> (tiny change)
+
+	* warc.c: make warc_uuid_str() implementation depend on HAVE_LIBUUID.
+
+2012-03-29  Tim Ruehsen <tim.ruehsen@gmx.de> (tiny change)
+
+	* utils.c (library): Include <sys/time.h>.
+
+2012-03-25  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* utils.c: Include <sys/ioctl.h>.
+
+	* ptimer.c: Include <sys/time.h>.
+
+	* connect.c: Include <sys/socket.h>, <sys/select.h>, <sys/time.h>.
+	Reported by: Ray Satiro <raysatiro@yahoo.com>.
+
+2012-03-25 Ray Satiro <raysatiro@yahoo.com>
+
+	* build_info.c.in: Check that HAVE_LIBSSL32 is defined when OpenSSL
+	is used.
+
+2012-03-07  Steven Schubiger  <stsc@member.fsf.org>
+
+	* init.c (wgetrc_user_file_name): Correct typo.
+
+2012-03-06   Sasikantha Babu   <sasikanth.v19@gmail.com>
+
+	* utils.c (convert_to_bits): Added new function convert_to_bits to
+	convert bytes to bits.
+	* retr.c (calc_rate): Modified the function to handle --bits
+	option and download rate calculated as bits per sec (SI-prefix)
+	for --bits otherwise bytes (IEC-prefix).
+	(retr_rate): Rates will display in bits per sec for --bits.
+	* options.h (struct opt): Added --bit option bool variable bits_fmt.
+	* main.c (print_help) : Added help for --bit.
+	* init.c: Defined command for --bit option.
+	* retr.h: Added function prototype.
+
+2012-02-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* main.c: Include "closeout.h"
+	(main): Register close_stdout at exit.
+
+2012-02-01  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* warc.c: Fix large file support with ftello, fseeko.
+	* warc.h: Fix large file support.
+	* http.c: Fix large file support.
+
+2012-02-23  Giuseppe Scrivano  <giuseppe@southpole.se>
+
+	* main.c (main): Write diagnostic messages to `stderr' not to `stdout'.
+
+	* main.c (main): Fail gracefully if `malloc' fails.
+
+	* gnutls.c (wgnutls_read): Remove unused variables `timer' and `flags'.
+
+2012-02-17  Steven Schubiger  <stsc@member.fsf.org>
+
+	* warc.c: Add license header.
+
+2012-01-27  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* retr.c (fd_read_body): If the response is chunked, the chunk
+	headers are now written to the WARC file, making the WARC file
+	an exact copy of the HTTP response.
+
+2012-01-27  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* retr.c (fd_read_body): Fix a memory leak with chunked responses.
+	* http.c (skip_short_body): Fix the same memory leak.
+
+2012-01-09  Gijs van Tulder  <gvtulder@gmail.com>
+
+	* init.c: Disable WARC compression if zlib is disabled.
+	* main.c: Do not show the 'no-warc-compression' option if zlib is
+	disabled.
+	* warc.c: Do not compress WARC files if zlib is disabled.
+
+2012-01-09  Sasikantha Babu   <sasikanth.v19@gmail.com> (tiny change)
+	* connect.c (connect_to_ip): properly formatted ipv6 address display.
+	(socket_family): New function - returns socket family type.
+	* http.c (gethttp): properly formatted ipv6 address display.
+
+2011-11-09  Gijs van Tulder  <address@hidden>
+
+	* warc.c: Call gzdopen() with wb9 instead of wb+9, which fails on
+	zlib version >= 1.2.4.
+
+2011-11-04  Steven Schweda  <address@hidden>
+
+	* warc.c [! WINDOWS]: Include <libgen.h>.
+	(warc_write_warcinfo_record): Assign a new allocated buffer and
+	free it on errors.
+
+2011-11-01  Steven Schweda  <address@hidden>
+
+	* gnutls.c (ssl_init): Ensure GNU TLS is loaded only once.
+
+2011-10-07  Steven Schweda  <address@hidden>
+
+	* connect.c: Add HAVE_SYS_SELECT_H and HAVE_SYS_SOCKET_H conditions
+	on includes of <sys/select.h> and <sys/socket.h>, respectively.
+	* ftp.c (getftp): Move BIN_TYPE_TRANSFER macro into VMS-specific
+	section.  On VMS, use Stream_LF attributes for listing files.  Pass
+	BIN_TYPE_FILE to fopen_excl() instead of constant-everywhere "true".
+	* ftp.c (ftp_retrieve_list): Restore lost test of opt.preserve_perm
+	(--preserve-permissions) on the chmod() operation.
+	* init.c, main.c: Remove "deprecated" from opt.preserve_perm
+	(--preserve-permissions).
+	* init.c (initialize): Use distinct messages for errors in C macro
+	SYSTEM_WGETRC and environment-variable SYSTEM_WGETRC.  Avoid use of
+	C macro SYSTEM_WGETRC when it's not defined.
+	* log.c (log_close): Avoid closing logfp when it's stderr.
+	* main.c (print_help):  Restore --preserve-permissions.
+	* main.c (main): Avoid using a negative value of longindex as a
+	subscript (for long_options[]) when searching for "--config".
+	* main.c (main): Exit the program using exit() instead of "return".
+	(VMS handles these differently, and exit() is better.)
+	* openssl.c (ssl_init): Add type cast (SSL_METHOD *) to newly "const"
+	"meth" argument to accommodate OpenSSL version 0.9.8, where that
+	argument is not "const" in the OpenSSL function (SSL_CTX_new).
+	* test.c: Declare "program_argstring".
+	* utils.c (fopen_excl): Comment typography.
+	* warc.h: New file.
+	* warc.c: New file.
+
+2011-10-02  Henrik Holst <henrik.holst@millistream.com> (tiny change)
+	* http.c (gethttp): If 'contentonerror' is used then do not
+        skip the http body on 4xx and 5xx errors.
+
+	* init.c (commands): Add 'contentonerror'.
+
+	* main.c (print_help, option_data): Add new option 'contentonerror'
+        to make wget not skip the http content on 4xx and 5xx errors.
+
+	* options.h: New variable 'content_on_error'.
+
+2011-09-19  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* main.c (print_version): Update copyright year.
+	(print_version): Fix typo.
+
+2011-09-13  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* ftp.c (ftp_retrieve_glob): Propagate correctly the `res' error
+	code.
+
+2011-09-07  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (gethttp): Don't inhibit arest request if opt.timestamping is
+	set.
+	Reported by <natrio@list.ru>
+
+2011-09-06  Jakob Matthes <jakob.matthes@gmail.com> (tiny change)
+
+	* main.c (print_version): Do not exit prematurely when --help is passed.
+
+2011-09-04  Christian Jullien <eligis@orange.fr> (tiny change)
+
+	* gnutls.c: Include <sys/fcntl.h>.
+
+2011-09-02  Mojca Miklavec <mojca.miklavec.lists@gmail.com> (tiny change)
+
+	* main.c (print_version): Do not exit prematurely when --version is passed.
+
+2011-08-30  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* gnutls.c (wgnutls_read_timeout): Use the non blocking socket only for
+	`gnutls_record_recv'.  Set errno to ETIMEDOUT on a read timeout.
+
+2011-08-29  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* gnutls.c (wgnutls_read_timeout): New function.
+	(wgnutls_read): Use wgnutls_read_timeout.
+	(wgnutls_peek): Likewise.
+
+2011-08-27  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* main.c (print_help): Exit with an error status if print to stdout
+	fails.
+	(print_usage): Change method signature and return a status code.
+	(print_version): Likewise.
+
+2011-08-26  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* gnutls.c: Include "ptimer.h".
+	(wgnutls_read): Honor read timeout.
+
+	* openssl.c (ssl_init): Make `meth' const.
+
+2011-08-25  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* utils.c [HAVE_UTIME && HAVE_UTIME_H]: Include <utime.h>.
+	[HAVE_UTIME && HAVE_SYS_UTIME_H]: Include <sys/utime.h>.
+	(touch) [HAVE_UTIME: Prefers utime over futimens when it is available.
+	It was reported that Cygwin has a not working futimens.
+
+2011-08-19  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* init.c (home_dir) [MSDOS]: Move local variable `len' here.
+
+2011-08-18  Giuseppe Scrivano  <giuseppe@southpole.se>
+
+	* http.c (gethttp): Fix a memory leak on some errors.  Free the head
+	buffer.
+
+	* Makefile.am: Use an additional file "css_.c" which in turn includes
+	"wget.h" and immediately "css.c".
+
+	* http.c (gethttp): Reset chunked_transfer_encoding on redirections.
+
+2011-08-13  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* retr.c (fd_read_body): Ensure max is not already defined.
+
+	* mswindows.h (snprintf): Remove definition.
+	(vsnprintf): Likewise.
+
+2011-08-12  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* css.l: Remove include "wget.h".
+	Reported by: Perry Smith <pedzsan@gmail.com>.
+
+2011-08-11  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* utils.c (abort_run_with_timeout): Use sigprocmask instead of
+	sigsetmask.
+
+	* gnutls.c (ssl_connect_wget): Remove call to deprecated function
+	`gnutls_certificate_type_set_priority'.
+
+	* Makefile.am (version.c): Don't invoke hg to set version string.
+
+2011-08-10  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	Fix a linker error on systems with an older gnutls version.
+	* gnutls.c (ssl_connect_wget)
+	[HAVE_GNUTLS_PRIORITY_SET_DIRECT]:  Use gnutls_priority_set_direct.
+	[! HAVE_GNUTLS_PRIORITY_SET_DIRECT]:  Use gnutls_protocol_set_priority.
+	Reported by: Karl Berry <karl@freefriends.org>
+
 2011-08-06  Jochen Roderburg <Roderburg@Uni-Koeln.DE>
 
 	* host.c (lookup_host): Ensure it doesn't print more IPs than available

src/Makefile.am

@@ -39,24 +39,27 @@ endif
 DEFS     = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
 LIBS     = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME)
 
+EXTRA_DIST = css.l css.c css_.c build_info.c.in
+
 bin_PROGRAMS = wget
 wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c    		  \
-	       css.l css-url.c \
+	       css_.c css-url.c \
 	       ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c \
 	       http.c init.c log.c main.c netrc.c progress.c ptimer.c     \
-	       recur.c res.c retr.c spider.c url.c	          	  \
+	       recur.c res.c retr.c spider.c url.c warc.c	          	  \
 	       utils.c exits.c build_info.c $(IRI_OBJ)			  \
 	       css-url.h css-tokens.h connect.h convert.h cookies.h       \
 	       ftp.h hash.h host.h html-parse.h html-url.h      \
 	       http.h http-ntlm.h init.h log.h mswindows.h netrc.h        \
 	       options.h progress.h ptimer.h recur.h res.h retr.h         \
-	       spider.h ssl.h sysdep.h url.h utils.h wget.h iri.h 	  \
+	       spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h 	  \
 	       exits.h gettext.h
 nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
 LDADD = $(LIBOBJS) ../lib/libgnu.a
 AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib
 
+
 ../lib/libgnu.a:
 	cd ../lib && $(MAKE) $(AM_MAKEFLAGS)
 
@@ -71,21 +74,27 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	echo '/* version.c */' > $@
 	echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@
 	echo '' >> $@
-	echo 'const char *version_string = "@VERSION@"' >> $@
-	-hg log -R "$(top_srcdir)" -r . --template='" ({node|short})"\n' \
-	    2>/dev/null >> $@
-	echo ';' >> $@
+	echo 'const char *version_string = "@VERSION@";' >> $@
 	echo 'const char *compilation_string = "'$(COMPILE)'";' \
 	    | $(ESCAPEQUOTE) >> $@
 	echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \
 	$(AM_LDFLAGS) $(LDFLAGS) $(LIBS) $(wget_LDADD)'";' \
 	    | $(ESCAPEQUOTE) >> $@
 
+css.c: $(srcdir)/css.l
+	$(LEX) $(LFLAGS) -o $@ $^
+
+css_.c: css.c
+	echo '#include "wget.h"' > $@
+	cat css.c >> $@
+
+distclean-local:
+	rm -f css.c css_.c
+
 check_LIBRARIES = libunittest.a
 libunittest_a_SOURCES = $(wget_SOURCES) test.c build_info.c test.h
 nodist_libunittest_a_SOURCES = version.c
 libunittest_a_CPPFLAGS = -DTESTING "-I$(top_builddir)/lib" "-I$(top_srcdir)/lib"
 libunittest_a_LIBADD = $(LIBOBJS)
-EXTRA_DIST = build_info.c.in
 
 CLEANFILES = *~ *.bak core core.[0-9]* build_info.c version.c

src/build_info.c.in

@@ -9,5 +9,5 @@ ntlm            defined ENABLE_NTLM
 opie            defined ENABLE_OPIE
 
 ssl choice:
-    openssl     defined HAVE_LIBSSL
+    openssl     defined HAVE_LIBSSL || defined HAVE_LIBSSL32
     gnutls      defined HAVE_LIBGNUTLS

src/connect.c

@@ -53,9 +53,7 @@ as that of the covered work.  */
 
 #include <errno.h>
 #include <string.h>
-#ifdef HAVE_SYS_TIME_H
-# include <sys/time.h>
-#endif
+#include <sys/time.h>
 #include "utils.h"
 #include "host.h"
 #include "connect.h"
@@ -293,7 +291,14 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
 					  xfree (str);
         }
       else
-        logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port);
+       {
+           if (ip->family == AF_INET)
+               logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port);
+#ifdef ENABLE_IPV6
+           else if (ip->family == AF_INET6)
+               logprintf (LOG_VERBOSE, _("Connecting to [%s]:%d... "), txt_addr, port);
+#endif
+       }
     }
 
   /* Store the sockaddr info to SA.  */
@@ -554,6 +559,7 @@ socket_ip_address (int sock, ip_address *ip, int endpoint)
   if (ret < 0)
     return false;
 
+  memset(ip, 0, sizeof(ip_address));
   ip->family = sockaddr->sa_family;
   switch (sockaddr->sa_family)
     {
@@ -581,6 +587,36 @@ socket_ip_address (int sock, ip_address *ip, int endpoint)
     }
 }
 
+/* Get the socket family of connection on FD and store
+   Return family type on success, -1 otherwise.
+
+   If ENDPOINT is ENDPOINT_LOCAL, it returns the sock family of the local
+   (client) side of the socket.  Else if ENDPOINT is ENDPOINT_PEER, it
+   returns the sock family of the remote (peer's) side of the socket.  */
+
+int
+socket_family (int sock, int endpoint)
+{
+  struct sockaddr_storage storage;
+  struct sockaddr *sockaddr = (struct sockaddr *) &storage;
+  socklen_t addrlen = sizeof (storage);
+  int ret;
+
+  memset (sockaddr, 0, addrlen);
+
+  if (endpoint == ENDPOINT_LOCAL)
+    ret = getsockname (sock, sockaddr, &addrlen);
+  else if (endpoint == ENDPOINT_PEER)
+    ret = getpeername (sock, sockaddr, &addrlen);
+  else
+    abort ();
+
+  if (ret < 0)
+    return -1;
+
+  return sockaddr->sa_family;
+}
+
 /* Return true if the error from the connect code can be considered
    retryable.  Wget normally retries after errors, but the exception
    are the "unsupported protocol" type errors (possible on IPv4/IPv6

src/connect.h

@@ -51,6 +51,7 @@ enum {
   ENDPOINT_PEER
 };
 bool socket_ip_address (int, ip_address *, int);
+int  socket_family (int sock, int endpoint);
 
 bool retryable_socket_connect_error (int);
 

src/convert.c

@@ -58,7 +58,7 @@ struct hash_table *downloaded_css_set;
 static void convert_links (const char *, struct urlpos *);
 
 
-void
+static void
 convert_links_in_hashtable (struct hash_table *downloaded_set,
                             int is_css,
                             int *file_count)
@@ -124,6 +124,9 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
           set_uri_encoding (pi, opt.locale, true);
 
           u = url_parse (cur_url->url->url, NULL, pi, true);
+          if (!u)
+	    continue;
+
           local_name = hash_table_get (dl_url_file_map, u->url);
 
           /* Decide on the conversion type.  */
@@ -870,7 +873,7 @@ register_delete_file (const char *file)
 /* Register that FILE is an HTML file that has been downloaded. */
 
 void
-register_html (const char *url, const char *file)
+register_html (const char *file)
 {
   if (!downloaded_html_set)
     downloaded_html_set = make_string_hash_table (0);
@@ -880,7 +883,7 @@ register_html (const char *url, const char *file)
 /* Register that FILE is a CSS file that has been downloaded. */
 
 void
-register_css (const char *url, const char *file)
+register_css (const char *file)
 {
   if (!downloaded_css_set)
     downloaded_css_set = make_string_hash_table (0);

src/convert.h

@@ -101,8 +101,8 @@ downloaded_file_t downloaded_file (downloaded_file_t, const char *);
 
 void register_download (const char *, const char *);
 void register_redirection (const char *, const char *);
-void register_html (const char *, const char *);
-void register_css (const char *, const char *);
+void register_html (const char *);
+void register_css (const char *);
 void register_delete_file (const char *);
 void convert_all_links (void);
 void convert_cleanup (void);

src/cookies.c

@@ -391,6 +391,9 @@ parse_set_cookie (const char *set_cookie, bool silent)
             goto error;
           BOUNDED_TO_ALLOCA (value.b, value.e, value_copy);
 
+          /* Check if expiration spec is valid.
+             If not, assume default (cookie doesn't expire, but valid only for
+	     this session.) */
           expires = http_atotm (value_copy);
           if (expires != (time_t) -1)
             {
@@ -402,10 +405,6 @@ parse_set_cookie (const char *set_cookie, bool silent)
               if (cookie->expiry_time < cookies_now)
                 cookie->discard_requested = 1;
             }
-          else
-            /* Error in expiration spec.  Assume default (cookie doesn't
-               expire, but valid only for this session.)  */
-            ;
         }
       else if (TOKEN_IS (name, "max-age"))
         {
@@ -433,9 +432,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
           /* ignore value completely */
           cookie->secure = 1;
         }
-      else
-        /* Ignore unrecognized attribute. */
-        ;
+      /* else: Ignore unrecognized attribute. */
     }
   if (*ptr)
     /* extract_param has encountered a syntax error */
@@ -676,9 +673,6 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
 
   if (!cookie->domain)
     {
-    copy_domain:
-      /* If the domain was not provided, we use the one we're talking
-         to, and set exact match.  */
       cookie->domain = xstrdup (host);
       cookie->domain_exact = 1;
       /* Set the port, but only if it's non-default. */
@@ -690,11 +684,12 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
       if (!check_domain_match (cookie->domain, host))
         {
           logprintf (LOG_NOTQUIET,
-                     _("Cookie coming from %s attempted to set domain to %s\n"),
-                     quotearg_style (escape_quoting_style, host),
+                     _("Cookie coming from %s attempted to set domain to "),
+                     quotearg_style (escape_quoting_style, host));
+          logprintf (LOG_NOTQUIET,
+                     _("%s\n"),
                      quotearg_style (escape_quoting_style, cookie->domain));
-          xfree (cookie->domain);
-          goto copy_domain;
+          cookie->discard_requested = true;
         }
     }
 
@@ -1134,7 +1129,9 @@ domain_port (const char *domain_b, const char *domain_e,
 void
 cookie_jar_load (struct cookie_jar *jar, const char *file)
 {
-  char *line;
+  char *line = NULL;
+  size_t bufsize = 0;
+
   FILE *fp = fopen (file, "r");
   if (!fp)
     {
@@ -1142,9 +1139,10 @@ cookie_jar_load (struct cookie_jar *jar, const char *file)
                  quote (file), strerror (errno));
       return;
     }
+
   cookies_now = time (NULL);
 
-  for (; ((line = read_whole_line (fp)) != NULL); xfree (line))
+  while (getline (&line, &bufsize, fp) > 0)
     {
       struct cookie *cookie;
       char *p = line;
@@ -1238,6 +1236,8 @@ cookie_jar_load (struct cookie_jar *jar, const char *file)
     abort_cookie:
       delete_cookie (cookie);
     }
+
+  xfree(line);
   fclose (fp);
 }
 

src/css-url.c

@@ -55,6 +55,7 @@ as that of the covered work.  */
 #include "convert.h"
 #include "html-url.h"
 #include "css-tokens.h"
+#include "css-url.h"
 
 /* from lex.yy.c */
 extern char *yytext;
@@ -107,7 +108,7 @@ const char *token_names[] = {
   whitespace after the opening parenthesis and before the closing
   parenthesis.
 */
-char *
+static char *
 get_uri_string (const char *at, int *pos, int *length)
 {
   char *uri;

src/css-url.h

@@ -30,6 +30,7 @@ as that of the covered work.  */
 #ifndef CSS_URL_H
 #define CSS_URL_H
 
+void get_urls_css (struct map_context *, int, int);
 void get_urls_css (struct map_context *, int, int);
 struct urlpos *get_urls_css_file (const char *, const char *);
 

src/css.l

@@ -36,7 +36,6 @@ as that of the covered work.  */
 
 #define YY_NO_INPUT
 
-#include "wget.h"
 #include "css-tokens.h"
 
 %}

src/exits.c

@@ -1,7 +1,5 @@
-/* Command line parsing.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+/* Exit status handling.
+   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
 
    This file is part of GNU Wget.
 
@@ -60,7 +58,7 @@ get_status_for_err (uerr_t err)
     case RETROK:
       return WGET_EXIT_SUCCESS;
     case FOPENERR: case FOPEN_EXCL_ERR: case FWRITEERR: case WRITEFAILED:
-    case UNLINKERR:
+    case UNLINKERR: case CLOSEFAILED: case FILEBADFILE:
       return WGET_EXIT_IO_FAIL;
     case NOCONERROR: case HOSTERR: case CONSOCKERR: case CONERROR:
     case CONSSLERR: case CONIMPOSSIBLE: case FTPRERR: case FTPINVPASV:
@@ -70,7 +68,7 @@ get_status_for_err (uerr_t err)
       return WGET_EXIT_SSL_AUTH_FAIL;
     case FTPLOGINC: case FTPLOGREFUSED: case AUTHFAILED:
       return WGET_EXIT_SERVER_AUTH_FAIL;
-    case HEOF: case HERR:
+    case HEOF: case HERR: case ATTRMISSING:
       return WGET_EXIT_PROTOCOL_ERROR;
     case WRONGCODE: case FTPPORTERR: case FTPSYSERR:
     case FTPNSFOD: case FTPUNKNOWNTYPE: case FTPSRVERR:
@@ -78,7 +76,7 @@ get_status_for_err (uerr_t err)
     case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN:
     case PROXERR:
       return WGET_EXIT_SERVER_ERROR;
-    case URLERROR: case QUOTEXC: case SSLINITFAILED:
+    case URLERROR: case QUOTEXC: case SSLINITFAILED: case UNKNOWNATTR:
     default:
       return WGET_EXIT_UNKNOWN;
     }

src/exits.h

@@ -1,5 +1,5 @@
-/* Internationalization related declarations.
-   Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+/* Exit status related declarations.
+   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/ftp-basic.c

@@ -524,7 +524,10 @@ ftp_pasv (int csock, ip_address *addr, int *port)
   for (s += 4; *s && !c_isdigit (*s); s++)
     ;
   if (!*s)
-    return FTPINVPASV;
+    {
+      xfree (respline);
+      return FTPINVPASV;
+    }
   for (i = 0; i < 6; i++)
     {
       tmp[i] = 0;
@@ -593,7 +596,10 @@ ftp_lpsv (int csock, ip_address *addr, int *port)
   for (s += 4; *s && !c_isdigit (*s); s++)
     ;
   if (!*s)
-    return FTPINVPASV;
+    {
+      xfree (respline);
+      return FTPINVPASV;
+    }
 
   /* First, get the address family */
   af = 0;
@@ -954,23 +960,32 @@ ftp_retr (int csock, const char *file)
 /* Sends the LIST command to the server.  If FILE is NULL, send just
    `LIST' (no space).  */
 uerr_t
-ftp_list (int csock, const char *file, enum stype rs)
+ftp_list (int csock, const char *file, bool avoid_list_a, bool avoid_list,
+          bool *list_a_used)
 {
   char *request, *respline;
   int nwritten;
   uerr_t err;
   bool ok = false;
   size_t i = 0;
-  /* Try `LIST -a' first and revert to `LIST' in case of failure.  */
+
+  *list_a_used = false;
+
+  /* 2013-10-12 Andrea Urbani (matfanjol)
+     For more information about LIST and "LIST -a" please look at ftp.c,
+     function getftp, text "__LIST_A_EXPLANATION__".
+
+     If somebody changes the following commands, please, checks also the
+     later "i" variable.  */
   const char *list_commands[] = { "LIST -a",
                                   "LIST" };
 
-  /* 2008-01-29  SMS.  For a VMS FTP server, where "LIST -a" may not
-     fail, but will never do what is desired here, skip directly to the
-     simple "LIST" command (assumed to be the last one in the list).
-  */
-  if (rs == ST_VMS)
-    i = countof (list_commands)- 1;
+  if (avoid_list_a)
+    {
+      i = countof (list_commands)- 1;
+      DEBUGP (("(skipping \"LIST -a\")"));
+    }
+
 
   do {
     /* Send request.  */
@@ -994,6 +1009,8 @@ ftp_list (int csock, const char *file, enum stype rs)
           {
             err = FTPOK;
             ok = true;
+            /* Which list command was used? */
+            *list_a_used = (i == 0);
           }
         else
           {
@@ -1002,6 +1019,12 @@ ftp_list (int csock, const char *file, enum stype rs)
         xfree (respline);
       }
     ++i;
+    if ((avoid_list) && (i == 1))
+      {
+        /* I skip LIST */
+        ++i;
+        DEBUGP (("(skipping \"LIST\")"));
+      }
   } while (i < countof (list_commands) && !ok);
 
   return err;
@@ -1009,7 +1032,7 @@ ftp_list (int csock, const char *file, enum stype rs)
 
 /* Sends the SYST command to the server. */
 uerr_t
-ftp_syst (int csock, enum stype *server_type)
+ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)
 {
   char *request, *respline;
   int nwritten;
@@ -1042,12 +1065,23 @@ ftp_syst (int csock, enum stype *server_type)
      first word of the server response)?  */
   request = strtok (NULL, " ");
 
+  *unix_type = UST_OTHER;
+
   if (request == NULL)
     *server_type = ST_OTHER;
   else if (!strcasecmp (request, "VMS"))
     *server_type = ST_VMS;
   else if (!strcasecmp (request, "UNIX"))
-    *server_type = ST_UNIX;
+    {
+      *server_type = ST_UNIX;
+      /* 2013-10-17 Andrea Urbani (matfanjol)
+         I check more in depth the system type */
+      if (!strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
+        *unix_type = UST_TYPE_L8;
+      else if (!strncasecmp (ftp_last_respline,
+                             "215 UNIX MultiNet Unix Emulation V5.3(93)", 41))
+        *unix_type = UST_MULTINET;
+    }
   else if (!strcasecmp (request, "WINDOWS_NT")
            || !strcasecmp (request, "WINDOWS2000"))
     *server_type = ST_WINNT;

src/ftp-ls.c

@@ -68,16 +68,17 @@ symperms (const char *s)
    replaces all <TAB> character with <SPACE>. Returns the length of the
    modified line. */
 static int
-clean_line(char *line)
+clean_line (char *line, int len)
 {
-  int len = strlen (line);
-  if (!len) return 0;
-  if (line[len - 1] == '\n')
+  if (len <= 0) return 0;
+
+  while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
     line[--len] = '\0';
+
   if (!len) return 0;
-  if (line[len - 1] == '\r')
-    line[--len] = '\0';
+
   for ( ; *line ; line++ ) if (*line == '\t') *line = ' ';
+
   return len;
 }
 
@@ -102,8 +103,9 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
   int hour, min, sec, ptype;
   struct tm timestruct, *tnow;
   time_t timenow;
+  size_t bufsize = 0;
 
-  char *line, *tok, *ptok;      /* tokenizer */
+  char *line = NULL, *tok, *ptok;      /* tokenizer */
   struct fileinfo *dir, *l, cur; /* list creation */
 
   fp = fopen (file, "rb");
@@ -115,22 +117,16 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
   dir = l = NULL;
 
   /* Line loop to end of file: */
-  while ((line = read_whole_line (fp)) != NULL)
+  while ((len = getline (&line, &bufsize, fp)) > 0)
     {
-      len = clean_line (line);
+      len = clean_line (line, len);
       /* Skip if total...  */
       if (!strncasecmp (line, "total", 5))
-        {
-          xfree (line);
-          continue;
-        }
+        continue;
       /* Get the first token (permissions).  */
       tok = strtok (line, " ");
       if (!tok)
-        {
-          xfree (line);
-          continue;
-        }
+        continue;
 
       cur.name = NULL;
       cur.linkto = NULL;
@@ -203,7 +199,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
           if (next < 0)         /* a month name was not encountered */
             {
               for (i = 0; i < 12; i++)
-                if (!strcmp (tok, months[i]))
+                if (!strcasecmp (tok, months[i]))
                   break;
               /* If we got a month, it means the token before it is the
                  size, and the filename is three tokens away.  */
@@ -368,7 +364,6 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
           DEBUGP (("Skipping.\n"));
           xfree_null (cur.name);
           xfree_null (cur.linkto);
-          xfree (line);
           continue;
         }
 
@@ -416,10 +411,9 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
       timestruct.tm_isdst = -1;
       l->tstamp = mktime (&timestruct); /* store the time-stamp */
       l->ptype = ptype;
-
-      xfree (line);
     }
 
+  xfree (line);
   fclose (fp);
   return dir;
 }
@@ -431,9 +425,11 @@ ftp_parse_winnt_ls (const char *file)
   int len;
   int year, month, day;         /* for time analysis */
   int hour, min;
+  size_t bufsize = 0;
   struct tm timestruct;
 
-  char *line, *tok;             /* tokenizer */
+  char *line = NULL, *tok;             /* tokenizer */
+  char *filename;
   struct fileinfo *dir, *l, cur; /* list creation */
 
   fp = fopen (file, "rb");
@@ -445,43 +441,53 @@ ftp_parse_winnt_ls (const char *file)
   dir = l = NULL;
 
   /* Line loop to end of file: */
-  while ((line = read_whole_line (fp)) != NULL)
+  while ((len = getline (&line, &bufsize, fp)) > 0)
     {
-      len = clean_line (line);
+      len = clean_line (line, len);
 
-      /* Extracting name is a bit of black magic and we have to do it
-         before `strtok' inserted extra \0 characters in the line
-         string. For the moment let us just suppose that the name starts at
-         column 39 of the listing. This way we could also recognize
-         filenames that begin with a series of space characters (but who
-         really wants to use such filenames anyway?). */
-      if (len < 40) goto continue_loop;
-      tok = line + 39;
-      cur.name = xstrdup(tok);
-      DEBUGP (("Name: '%s'\n", cur.name));
+      /* Name begins at 39 column of the listing if date presented in `mm-dd-yy'
+         format or at 41 column if date presented in `mm-dd-yyyy' format. Thus,
+         we cannot extract name before we parse date. Using this information we
+         also can recognize filenames that begin with a series of space
+         characters (but who really wants to use such filenames anyway?). */
+      if (len < 40) continue;
+      filename = line + 39;
 
-      /* First column: mm-dd-yy. Should atoi() on the month fail, january
-         will be assumed.  */
+      /* First column: mm-dd-yy or mm-dd-yyyy. Should atoi() on the month fail,
+         january will be assumed.  */
       tok = strtok(line, "-");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       month = atoi(tok) - 1;
       if (month < 0) month = 0;
       tok = strtok(NULL, "-");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       day = atoi(tok);
       tok = strtok(NULL, " ");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       year = atoi(tok);
       /* Assuming the epoch starting at 1.1.1970 */
-      if (year <= 70) year += 100;
+      if (year <= 70)
+        {
+          year += 100;
+        }
+      else if (year >= 1900)
+        {
+          year -= 1900;
+          filename += 2;
+        }
+      /* Now it is possible to determine the position of the first symbol in
+         filename. */
+      cur.name = xstrdup(filename);
+      DEBUGP (("Name: '%s'\n", cur.name));
+
 
       /* Second column: hh:mm[AP]M, listing does not contain value for
          seconds */
       tok = strtok(NULL,  ":");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       hour = atoi(tok);
       tok = strtok(NULL,  "M");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       min = atoi(tok);
       /* Adjust hour from AM/PM. Just for the record, the sequence goes
          11:00AM, 12:00PM, 01:00PM ... 11:00PM, 12:00AM, 01:00AM . */
@@ -512,9 +518,9 @@ ftp_parse_winnt_ls (const char *file)
          directories as the listing does not give us a clue) and filetype
          here. */
       tok = strtok(NULL, " ");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       while ((tok != NULL) && (*tok == '\0'))  tok = strtok(NULL, " ");
-      if (tok == NULL) goto continue_loop;
+      if (tok == NULL) continue;
       if (*tok == '<')
         {
           cur.type  = FT_DIRECTORY;
@@ -553,11 +559,9 @@ ftp_parse_winnt_ls (const char *file)
           memcpy (l, &cur, sizeof (cur));
           l->next = NULL;
         }
-
-continue_loop:
-      xfree (line);
     }
 
+  xfree (line);
   fclose(fp);
   return dir;
 }
@@ -678,11 +682,12 @@ ftp_parse_vms_ls (const char *file)
   FILE *fp;
   int dt, i, j, len;
   int perms;
+  size_t bufsize = 0;
   time_t timenow;
   struct tm *timestruct;
   char date_str[ 32];
 
-  char *line, *tok;		 /* tokenizer */
+  char *line = NULL, *tok; /* tokenizer */
   struct fileinfo *dir, *l, cur; /* list creation */
 
   fp = fopen (file, "r");
@@ -695,52 +700,34 @@ ftp_parse_vms_ls (const char *file)
 
   /* Skip blank lines, Directory heading, and more blank lines. */
 
-  j = 0; /* Expecting initial blank line(s). */
-  while (1)
+  for (j = 0; (i = getline (&line, &bufsize, fp)) > 0; )
     {
-      line = read_whole_line (fp);
-      if (line == NULL)
+      i = clean_line (line, i);
+      if (i <= 0)
+        continue; /* Ignore blank line. */
+
+      if ((j == 0) && (line[i - 1] == ']'))
         {
-        break;
+          /* Found Directory heading line.  Next non-blank line
+          is significant. */
+          j = 1;
+        }
+      else if (!strncmp (line, "Total of ", 9))
+        {
+          /* Found "Total of ..." footing line.  No valid data
+             will follow (empty directory). */
+          i = 0; /* Arrange for early exit. */
+          break;
         }
       else
         {
-          i = clean_line (line);
-          if (i <= 0)
-            {
-              xfree (line); /* Free useless line storage. */
-              continue; /* Blank line.  Keep looking. */
-            }
-          else
-            {
-              if ((j == 0) && (line[ i- 1] == ']'))
-                {
-                  /* Found Directory heading line.  Next non-blank line
-                  is significant.
-                  */
-                  j = 1;
-                }
-              else if (!strncmp (line, "Total of ", 9))
-                {
-                  /* Found "Total of ..." footing line.  No valid data
-                     will follow (empty directory).
-                  */
-                  xfree (line); /* Free useless line storage. */
-                  line = NULL; /* Arrange for early exit. */
-                  break;
-                }
-              else
-                {
-                  break; /* Must be significant data. */
-                }
-            }
-          xfree (line); /* Free useless line storage. */
+          break; /* Must be significant data. */
         }
     }
 
   /* Read remainder of file until the next blank line or EOF. */
 
-  while (line != NULL)
+  while (i > 0)
     {
       char *p;
 
@@ -831,9 +818,8 @@ ftp_parse_vms_ls (const char *file)
       if (tok == NULL)
         {
           DEBUGP (("Getting additional line.\n"));
-          xfree (line);
-          line = read_whole_line (fp);
-          if (!line)
+          i = getline (&line, &bufsize, fp);
+          if (i <= 0)
             {
               DEBUGP (("EOF.  Leaving listing parser.\n"));
               break;
@@ -842,14 +828,14 @@ ftp_parse_vms_ls (const char *file)
           /* Second line must begin with " ".  Otherwise, it's a first
              line (and we may be confused).
           */
+          i = clean_line (line, i);
           if (i <= 0)
             {
               /* Blank line.  End of significant file listing. */
               DEBUGP (("Blank line.  Leaving listing parser.\n"));
-              xfree (line); /* Free useless line storage. */
               break;
             }
-          else if (line[ 0] != ' ')
+          else if (line[0] != ' ')
             {
               DEBUGP (("Non-blank in column 1.  Must be a new file name?\n"));
               continue;
@@ -861,7 +847,6 @@ ftp_parse_vms_ls (const char *file)
                 {
                   /* Unexpected non-empty but apparently blank line. */
                   DEBUGP (("Null token.  Leaving listing parser.\n"));
-                  xfree (line); /* Free useless line storage. */
                   break;
                 }
             }
@@ -900,7 +885,7 @@ ftp_parse_vms_ls (const char *file)
                        (sizeof( date_str)- strlen (date_str) - 1));
               DEBUGP (("Date time: >%s<\n", date_str));
             }
-          else if (strchr ( tok, '[') != NULL)
+          else if (strchr (tok, '[') != NULL)
             {
               /* Owner.  (Ignore.) */
               DEBUGP (("Owner.\n"));
@@ -910,7 +895,7 @@ ftp_parse_vms_ls (const char *file)
               /* Protections (permissions). */
               perms = 0;
               j = 0;
-              for (i = 0; i < strlen( tok); i++)
+              for (i = 0; i < strlen(tok); i++)
                 {
                   switch (tok[ i])
                     {
@@ -1004,21 +989,19 @@ ftp_parse_vms_ls (const char *file)
           l->next = NULL;
         }
 
-      /* Free old line storage.  Read a new line. */
-      xfree (line);
-      line = read_whole_line (fp);
-      if (line != NULL)
+      i = getline (&line, &bufsize, fp);
+      if (i > 0)
         {
-          i = clean_line (line);
+          i = clean_line (line, i);
           if (i <= 0)
-	    {
+            {
               /* Blank line.  End of significant file listing. */
-              xfree (line); /* Free useless line storage. */
-	      break;
-	    }
+              break;
+            }
         }
     }
 
+  xfree (line);
   fclose (fp);
   return dir;
 }

src/ftp.c

@@ -49,6 +49,7 @@ as that of the covered work.  */
 #include "netrc.h"
 #include "convert.h"            /* for downloaded_file */
 #include "recur.h"              /* for INFINITE_RECURSION */
+#include "warc.h"
 
 #ifdef __VMS
 # include "vms.h"
@@ -69,6 +70,7 @@ typedef struct
   int csock;                    /* control connection socket */
   double dltime;                /* time of the download in msecs */
   enum stype rs;                /* remote system reported by ftp server */
+  enum ustype rsu;              /* when rs is ST_UNIX, here there are more details */
   char *id;                     /* initial directory */
   char *target;                 /* target file name */
   struct url *proxy;            /* FTWK-style proxy */
@@ -237,25 +239,27 @@ static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
 
 /* Retrieves a file with denoted parameters through opening an FTP
    connection to the server.  It always closes the data connection,
-   and closes the control connection in case of error.  */
+   and closes the control connection in case of error.  If warc_tmp
+   is non-NULL, the downloaded data will be written there as well.  */
 static uerr_t
 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
-        wgint restval, ccon *con, int count)
+        wgint restval, ccon *con, int count, FILE *warc_tmp)
 {
   int csock, dtsock, local_sock, res;
   uerr_t err = RETROK;          /* appease the compiler */
   FILE *fp;
-  char *user, *passwd, *respline;
-  char *tms;
-  const char *tmrate;
+  char *respline, *tms;
+  const char *user, *passwd, *tmrate;
   int cmd = con->cmd;
   bool pasv_mode_open = false;
   wgint expected_bytes = 0;
   bool got_expected_bytes = false;
   bool rest_failed = false;
   int flags;
-  wgint rd_size;
+  wgint rd_size, previous_rd_size = 0;
   char type_char;
+  bool try_again;
+  bool list_a_used = false;
 
   assert (con != NULL);
   assert (con->target != NULL);
@@ -287,13 +291,6 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
     {
       char    *host = con->proxy ? con->proxy->host : u->host;
       int      port = con->proxy ? con->proxy->port : u->port;
-      char *logname = user;
-
-      if (con->proxy)
-        {
-          /* If proxy is in use, log in as username@target-site. */
-          logname = concat_strings (user, "@", u->host, (char *) 0);
-        }
 
       /* Login to the server: */
 
@@ -301,20 +298,10 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
 
       csock = connect_to_host (host, port);
       if (csock == E_HOST)
-        {
-          if (con->proxy)
-            xfree (logname);
-
           return HOSTERR;
-        }
       else if (csock < 0)
-        {
-          if (con->proxy)
-            xfree (logname);
-
           return (retryable_socket_connect_error (errno)
                   ? CONERROR : CONIMPOSSIBLE);
-        }
 
       if (cmd & LEAVE_PENDING)
         con->csock = csock;
@@ -326,10 +313,15 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
                  quotearg_style (escape_quoting_style, user));
       if (opt.server_response)
         logputs (LOG_ALWAYS, "\n");
-      err = ftp_login (csock, logname, passwd);
-
       if (con->proxy)
-        xfree (logname);
+        {
+          /* If proxy is in use, log in as username@target-site. */
+          char *logname = concat_strings (user, "@", u->host, (char *) 0);
+          err = ftp_login (csock, logname, passwd);
+          xfree (logname);
+        }
+      else
+        err = ftp_login (csock, user, passwd);
 
       /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
       switch (err)
@@ -376,7 +368,7 @@ Error in server response, closing control connection.\n"));
       /* Third: Get the system type */
       if (!opt.server_response)
         logprintf (LOG_VERBOSE, "==> SYST ... ");
-      err = ftp_syst (csock, &con->rs);
+      err = ftp_syst (csock, &con->rs, &con->rsu);
       /* FTPRERR */
       switch (err)
         {
@@ -401,6 +393,44 @@ Error in server response, closing control connection.\n"));
       if (!opt.server_response && err != FTPSRVERR)
         logputs (LOG_VERBOSE, _("done.    "));
 
+      /* 2013-10-17 Andrea Urbani (matfanjol)
+         According to the system type I choose which
+         list command will be used.
+         If I don't know that system, I will try, the
+         first time of each session, "LIST -a" and
+         "LIST". (see __LIST_A_EXPLANATION__ below) */
+      switch (con->rs)
+        {
+        case ST_VMS:
+          /* About ST_VMS there is an old note:
+             2008-01-29  SMS.  For a VMS FTP server, where "LIST -a" may not
+             fail, but will never do what is desired here,
+             skip directly to the simple "LIST" command
+             (assumed to be the last one in the list).  */
+          DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n"));
+          con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+          con->st |= AVOID_LIST_A;
+          break;
+        case ST_UNIX:
+          if (con->rsu == UST_MULTINET)
+            {
+              DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" "
+                       "as standard list command\n"));
+              con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+              con->st |= AVOID_LIST_A;
+            }
+          else if (con->rsu == UST_TYPE_L8)
+            {
+              DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" "
+                       "as standard list command\n"));
+              con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+              con->st |= AVOID_LIST;
+            }
+          break;
+        default:
+          break;
+        }
+
       /* Fourth: Find the initial ftp directory */
 
       if (!opt.server_response)
@@ -512,7 +542,7 @@ Error in server response, closing control connection.\n"));
         logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
       else
         {
-          char *targ = NULL;
+          const char *targ = NULL;
           int cwd_count;
           int cwd_end;
           int cwd_start;
@@ -627,16 +657,16 @@ Error in server response, closing control connection.\n"));
              The VMS restriction may be relaxed when the squirrely code
              above is reformed.
           */
-	  if ((con->rs == ST_VMS) && (target[0] != '/'))
-	    {
-	      cwd_start = 0;
-	      DEBUGP (("Using two-step CWD for relative path.\n"));
-	    }
-	  else
-	    {
+          if ((con->rs == ST_VMS) && (target[0] != '/'))
+            {
+              cwd_start = 0;
+              DEBUGP (("Using two-step CWD for relative path.\n"));
+            }
+          else
+            {
               /* Go straight to the target. */
-	      cwd_start = 1;
-	    }
+              cwd_start = 1;
+            }
 
           /* At least one VMS FTP server (TCPware V5.6-2) can switch to
              a UNIX emulation mode when given a UNIX-like directory
@@ -654,10 +684,10 @@ Error in server response, closing control connection.\n"));
              Unlike the rest of this block, this particular behavior
              _is_ VMS-specific, so it gets its own VMS test.
           */
-	  if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
+          if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
             {
               cwd_end = 3;
-	      DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
+              DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
             }
           else
             {
@@ -667,22 +697,22 @@ Error in server response, closing control connection.\n"));
           /* 2004-09-20 SMS. */
           /* Sorry about the deviant indenting.  Laziness. */
 
-	  for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
-	{
+          for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
+            {
           switch (cwd_count)
             {
               case 0:
-	        /* Step one (optional): Go to the initial directory,
-	           exactly as reported by the server.
-	        */
-	        targ = con->id;
+                /* Step one (optional): Go to the initial directory,
+                   exactly as reported by the server.
+                */
+                targ = con->id;
                 break;
 
               case 1:
-	        /* Step two: Go to the target directory.  (Absolute or
-	           relative will work now.)
-	        */
-	        targ = target;
+                /* Step two: Go to the target directory.  (Absolute or
+                   relative will work now.)
+                */
+                targ = target;
                 break;
 
               case 2:
@@ -695,7 +725,7 @@ Error in server response, closing control connection.\n"));
               default:
                 /* Can't happen. */
                 assert (1);
-	    }
+            }
 
           if (!opt.server_response)
             logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
@@ -786,6 +816,9 @@ Error in server response, closing control connection.\n"));
       return RETRFINISHED;
     }
 
+  do
+  {
+  try_again = false;
   /* If anything is to be retrieved, PORT (or PASV) must be sent.  */
   if (cmd & (DO_LIST | DO_RETR))
     {
@@ -949,42 +982,42 @@ Error in server response, closing control connection.\n"));
   if (cmd & DO_RETR)
     {
       /* If we're in spider mode, don't really retrieve anything except
-	 the directory listing and verify whether the given "file" exists.  */
+         the directory listing and verify whether the given "file" exists.  */
       if (opt.spider)
         {
-	  bool exists = false;
-	  uerr_t res;
-	  struct fileinfo *f;
-	  res = ftp_get_listing (u, con, &f);
-	  /* Set the DO_RETR command flag again, because it gets unset when
-	     calling ftp_get_listing() and would otherwise cause an assertion
-	     failure earlier on when this function gets repeatedly called
-	     (e.g., when recursing).  */
-	  con->cmd |= DO_RETR;
-	  if (res == RETROK)
-	    {
-	      while (f)
-		{
-		  if (!strcmp (f->name, u->file))
-		    {
-		      exists = true;
-		      break;
-		    }
-		  f = f->next;
-		}
+          bool exists = false;
+          uerr_t res;
+          struct fileinfo *f;
+          res = ftp_get_listing (u, con, &f);
+          /* Set the DO_RETR command flag again, because it gets unset when
+             calling ftp_get_listing() and would otherwise cause an assertion
+             failure earlier on when this function gets repeatedly called
+             (e.g., when recursing).  */
+          con->cmd |= DO_RETR;
+          if (res == RETROK)
+            {
+              while (f)
+                {
+                  if (!strcmp (f->name, u->file))
+                    {
+                      exists = true;
+                      break;
+                    }
+                  f = f->next;
+                }
               if (exists)
                 {
                   logputs (LOG_VERBOSE, "\n");
                   logprintf (LOG_NOTQUIET, _("File %s exists.\n"),
                              quote (u->file));
                 }
-	      else
+              else
                 {
-		  logputs (LOG_VERBOSE, "\n");
-		  logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
-			     quote (u->file));
-		}
-	    }
+                  logputs (LOG_VERBOSE, "\n");
+                  logprintf (LOG_NOTQUIET, _("No such file %s.\n"),
+                             quote (u->file));
+                }
+            }
           fd_close (csock);
           con->csock = -1;
           fd_close (dtsock);
@@ -1052,7 +1085,8 @@ Error in server response, closing control connection.\n"));
       /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
          without arguments is better than `LIST .'; confirmed by
          RFC959.  */
-      err = ftp_list (csock, NULL, con->rs);
+      err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used);
+
       /* FTPRERR, WRITEFAILED */
       switch (err)
         {
@@ -1152,13 +1186,25 @@ Error in server response, closing control connection.\n"));
    Elsewhere, define a constant "binary" flag.
    Isn't it nice to have distinct text and binary file types?
 */
-# define BIN_TYPE_TRANSFER (type_char != 'A')
+/* 2011-09-30 SMS.
+   Added listing files to the set of non-"binary" (text, Stream_LF)
+   files.  (Wget works either way, but other programs, like, say, text
+   editors, work better on listing files which have text attributes.)
+   Now we use "binary" attributes for a binary ("IMAGE") transfer,
+   unless "--ftp-stmlf" was specified, and we always use non-"binary"
+   (text, Stream_LF) attributes for a listing file, or for an ASCII
+   transfer.
+   Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to
+   fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true".
+*/
 #ifdef __VMS
+# define BIN_TYPE_TRANSFER (type_char != 'A')
+# define BIN_TYPE_FILE \
+   ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
 # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
 # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS
-# define BIN_TYPE_FILE (BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0))
 #else /* def __VMS */
-# define BIN_TYPE_FILE 1
+# define BIN_TYPE_FILE true
 #endif /* def __VMS [else] */
 
       if (restval && !(con->cmd & DO_LIST))
@@ -1182,21 +1228,21 @@ Error in server response, closing control connection.\n"));
         }
       else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
                || opt.output_document || count > 0)
-        {	  
-	  if (opt.unlink && file_exists_p (con->target))
-	    {
-	      int res = unlink (con->target);
-	      if (res < 0)
-		{
-		  logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
-			     strerror (errno));
-		  fd_close (csock);
-		  con->csock = -1;
-		  fd_close (dtsock);
-		  fd_close (local_sock);
-		  return UNLINKERR;
-		}
-	    }
+        {
+          if (opt.unlink && file_exists_p (con->target))
+            {
+              int res = unlink (con->target);
+              if (res < 0)
+                {
+                  logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
+                    strerror (errno));
+                    fd_close (csock);
+                    con->csock = -1;
+                    fd_close (dtsock);
+                    fd_close (local_sock);
+                    return UNLINKERR;
+                }
+            }
 
 #ifdef __VMS
           int open_id;
@@ -1217,7 +1263,7 @@ Error in server response, closing control connection.\n"));
         }
       else
         {
-          fp = fopen_excl (con->target, true);
+          fp = fopen_excl (con->target, BIN_TYPE_FILE);
           if (!fp && errno == EEXIST)
             {
               /* We cannot just invent a new name and use it (which is
@@ -1262,7 +1308,7 @@ Error in server response, closing control connection.\n"));
   rd_size = 0;
   res = fd_read_body (dtsock, fp,
                       expected_bytes ? expected_bytes - restval : 0,
-                      restval, &rd_size, qtyread, &con->dltime, flags);
+                      restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
 
   tms = datetime_str (time (NULL));
   tmrate = retr_rate (rd_size, con->dltime);
@@ -1273,15 +1319,18 @@ Error in server response, closing control connection.\n"));
   if (!output_stream || con->cmd & DO_LIST)
     fclose (fp);
 
-  /* If fd_read_body couldn't write to fp, bail out.  */
-  if (res == -2)
+  /* If fd_read_body couldn't write to fp or warc_tmp, bail out.  */
+  if (res == -2 || (warc_tmp != NULL && res == -3))
     {
       logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
                  con->target, strerror (errno));
       fd_close (csock);
       con->csock = -1;
       fd_close (dtsock);
-      return FWRITEERR;
+      if (res == -2)
+        return FWRITEERR;
+      else if (res == -3)
+        return WARC_TMP_FWRITEERR;
     }
   else if (res == -1)
     {
@@ -1339,8 +1388,10 @@ Error in server response, closing control connection.\n"));
     }
   /* If it was a listing, and opt.server_response is true,
      print it out.  */
-  if (opt.server_response && (con->cmd & DO_LIST))
+  if (con->cmd & DO_LIST)
     {
+      if (opt.server_response)
+        {
 /* 2005-02-25 SMS.
    Much of this work may already have been done, but repeating it should
    do no damage beyond wasting time.
@@ -1363,22 +1414,115 @@ Error in server response, closing control connection.\n"));
         logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
       else
         {
-          char *line;
-          /* The lines are being read with read_whole_line because of
+          char *line = NULL;
+          size_t bufsize = 0;
+          ssize_t len;
+
+          /* The lines are being read with getline because of
              no-buffering on opt.lfile.  */
-          while ((line = read_whole_line (fp)) != NULL)
+          while ((len = getline (&line, &bufsize, fp)) > 0)
             {
-              char *p = strchr (line, '\0');
-              while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
-                *--p = '\0';
+              while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
+                line[--len] = '\0';
               logprintf (LOG_ALWAYS, "%s\n",
                          quotearg_style (escape_quoting_style, line));
-              xfree (line);
             }
+          xfree (line);
           fclose (fp);
         }
-    } /* con->cmd & DO_LIST && server_response */
+        } /* server_response */
 
+      /* 2013-10-17 Andrea Urbani (matfanjol)
+         < __LIST_A_EXPLANATION__ >
+          After the SYST command, looks if it knows that system.
+          If yes, wget will force the use of "LIST" or "LIST -a".
+          If no, wget will try, only the first time of each session, before the
+          "LIST -a" command and after the "LIST".
+          If "LIST -a" works and returns more or equal data of the "LIST",
+          "LIST -a" will be the standard list command for all the session.
+          If "LIST -a" fails or returns less data than "LIST" (think on the case
+          of an existing file called "-a"), "LIST" will be the standard list
+          command for all the session.
+          ("LIST -a" is used to get also the hidden files)
+
+          */
+      if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE))
+        {
+          /* We still have to check "LIST" after the first "LIST -a" to see
+             if with "LIST" we get more data than "LIST -a", that means
+             "LIST -a" returned files/folders with "-a" name. */
+          if (con->st & AVOID_LIST_A)
+            {
+              /* LIST was used in this cycle.
+                 Let's see the result. */
+              if (rd_size > previous_rd_size)
+                {
+                  /* LIST returns more data than "LIST -a".
+                     "LIST" is the official command to use. */
+                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                  DEBUGP (("LIST returned more data than \"LIST -a\": "
+                           "I will use \"LIST\" as standard list command\n"));
+                }
+              else if (previous_rd_size > rd_size)
+                {
+                  /* "LIST -a" returned more data then LIST.
+                     "LIST -a" is the official command to use. */
+                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                  con->st |= AVOID_LIST;
+                  con->st &= ~AVOID_LIST_A;
+                  /* Sorry, please, download again the "LIST -a"... */
+                  try_again = true;
+                  DEBUGP (("LIST returned less data than \"LIST -a\": I will "
+                           "use \"LIST -a\" as standard list command\n"));
+                }
+              else
+                {
+                  /* LIST and "LIST -a" return the same data. */
+                  if (rd_size == 0)
+                    {
+                      /* Same empty data. We will check both again because
+                         we cannot check if "LIST -a" has returned an empty
+                         folder instead of a folder content. */
+                      con->st &= ~AVOID_LIST_A;
+                    }
+                  else
+                    {
+                      /* Same data, so, better to take "LIST -a" that
+                         shows also hidden files/folders (when present) */
+                      con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                      con->st |= AVOID_LIST;
+                      con->st &= ~AVOID_LIST_A;
+                      DEBUGP (("LIST returned the same amount of data of "
+                               "\"LIST -a\": I will use \"LIST -a\" as standard "
+                               "list command\n"));
+                    }
+                }
+            }
+          else
+            {
+              /* In this cycle "LIST -a" should being used. Is it true? */
+              if (list_a_used)
+                {
+                  /* Yes, it is.
+                     OK, let's save the amount of data and try again
+                     with LIST */
+                  previous_rd_size = rd_size;
+                  try_again = true;
+                  con->st |= AVOID_LIST_A;
+                }
+              else
+                {
+                  /* No: something happens and LIST was used.
+                     This means "LIST -a" raises an error. */
+                  con->st |= LIST_AFTER_LIST_A_CHECK_DONE;
+                  con->st |= AVOID_LIST_A;
+                  DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" "
+                           "as standard list command\n"));
+                }
+            }
+        }
+    }
+  } while (try_again);
   return RETRFINISHED;
 }
 
@@ -1397,6 +1541,11 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   uerr_t err;
   struct_stat st;
 
+  /* Declare WARC variables. */
+  bool warc_enabled = (opt.warc_filename != NULL);
+  FILE *warc_tmp = NULL;
+  ip_address *warc_ip = NULL;
+
   /* Get the target, and set the name for the message accordingly. */
   if ((f == NULL) && (con->target))
     {
@@ -1415,7 +1564,12 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
 
   /* If the output_document was given, then this check was already done and
      the file didn't exist. Hence the !opt.output_document */
-  if (opt.noclobber && !opt.output_document && file_exists_p (con->target))
+
+  /* If we receive .listing file it is necessary to determine system type of the ftp
+     server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
+     order to establish connection with the server and get system type. */
+  if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
+      && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
     {
       logprintf (LOG_VERBOSE,
                  _("File %s already there; not retrieving.\n"), quote (con->target));
@@ -1460,6 +1614,21 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
             con->cmd |= DO_CWD;
         }
 
+      /* For file RETR requests, we can write a WARC record.
+         We record the file contents to a temporary file. */
+      if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL)
+        {
+          warc_tmp = warc_tempfile ();
+          if (warc_tmp == NULL)
+            return WARC_TMP_FOPENERR;
+
+          if (!con->proxy && con->csock != -1)
+            {
+              warc_ip = (ip_address *) alloca (sizeof (ip_address));
+              socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER);
+            }
+        }
+
       /* Decide whether or not to restart.  */
       if (con->cmd & DO_LIST)
         restval = 0;
@@ -1497,7 +1666,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
         len = f->size;
       else
         len = 0;
-      err = getftp (u, len, &qtyread, restval, con, count);
+
+      /* If we are working on a WARC record, getftp should also write
+         to the warc_tmp file. */
+      err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
 
       if (con->csock == -1)
         con->st &= ~DONE_CWD;
@@ -1508,8 +1680,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
         {
         case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
         case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
-        case UNLINKERR:
+        case UNLINKERR: case WARC_TMP_FWRITEERR:
           /* Fatal errors, give up.  */
+          if (warc_tmp != NULL)
+            fclose (warc_tmp);
           return err;
         case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
         case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
@@ -1577,7 +1751,20 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
           xfree (hurl);
         }
 
-      if ((con->cmd & DO_LIST))
+      if (warc_enabled && (con->cmd & DO_RETR))
+        {
+          /* Create and store a WARC resource record for the retrieved file. */
+          bool warc_res;
+
+          warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
+                                                  warc_ip, NULL, warc_tmp, -1);
+          if (! warc_res)
+            return WARC_ERR;
+
+          /* warc_write_resource_record has also closed warc_tmp. */
+        }
+
+      if (con->cmd & DO_LIST)
         /* This is a directory listing file. */
         {
           if (!opt.remove_listing)
@@ -1880,8 +2067,10 @@ Already have correct symlink %s -> %s\n\n"),
 
       set_local_file (&actual_target, con->target);
 
-      /* If downloading a plain file, set valid (non-zero) permissions. */
-      if (dlthis && (actual_target != NULL) && (f->type == FT_PLAINFILE))
+      /* If downloading a plain file, and the user requested it, then
+         set valid (non-zero) permissions. */
+      if (dlthis && (actual_target != NULL) &&
+       (f->type == FT_PLAINFILE) && opt.preserve_perm)
         {
           if (f->perms)
             chmod (actual_target, f->perms);
@@ -1914,7 +2103,9 @@ Already have correct symlink %s -> %s\n\n"),
       xfree (ofile);
 
       /* Break on fatals.  */
-      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
+      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR
+          || err == WARC_ERR || err == WARC_TMP_FOPENERR
+          || err == WARC_TMP_FWRITEERR)
         break;
       con->cmd &= ~ (DO_CWD | DO_LOGIN);
       f = f->next;
@@ -2125,7 +2316,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
   if (start)
     {
       /* Just get everything.  */
-      ftp_retrieve_list (u, start, con);
+      res = ftp_retrieve_list (u, start, con);
     }
   else
     {
@@ -2154,8 +2345,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
   if (opt.quota && total_downloaded_bytes > opt.quota)
     return QUOTEXC;
   else
-    /* #### Should we return `res' here?  */
-    return RETROK;
+    return res;
 }
 
 /* The wrapper that calls an appropriate routine according to contents
@@ -2233,11 +2423,11 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
             file_part = u->path;
           ispattern = has_wildcards_p (file_part);
         }
-      if (ispattern || recursive || opt.timestamping)
+      if (ispattern || recursive || opt.timestamping || opt.preserve_perm)
         {
           /* ftp_retrieve_glob is a catch-all function that gets called
-             if we need globbing, time-stamping or recursion.  Its
-             third argument is just what we really need.  */
+             if we need globbing, time-stamping, recursion or preserve
+             permissions.  Its third argument is just what we really need.  */
           res = ftp_retrieve_glob (u, &con,
                                    ispattern ? GLOB_GLOBALL : GLOB_GETONE);
         }

src/ftp.h

@@ -45,6 +45,14 @@ enum stype
   ST_OTHER
 };
 
+/* Extensions of the ST_UNIX */
+enum ustype
+{
+  UST_TYPE_L8,
+  UST_MULTINET,
+  UST_OTHER
+};
+
 extern char ftp_last_respline[];
 
 uerr_t ftp_response (int, char **);
@@ -61,8 +69,8 @@ uerr_t ftp_type (int, int);
 uerr_t ftp_cwd (int, const char *);
 uerr_t ftp_retr (int, const char *);
 uerr_t ftp_rest (int, wgint);
-uerr_t ftp_list (int, const char *, enum stype);
-uerr_t ftp_syst (int, enum stype *);
+uerr_t ftp_list (int, const char *, bool, bool, bool *);
+uerr_t ftp_syst (int, enum stype *, enum ustype *);
 uerr_t ftp_pwd (int, char **);
 uerr_t ftp_size (int, const char *, wgint *);
 
@@ -124,8 +132,23 @@ enum wget_ftp_fstatus
   NOTHING       = 0x0000,	/* Nothing done yet.  */
   ON_YOUR_OWN   = 0x0001,	/* The ftp_loop_internal sets the
 				   defaults.  */
-  DONE_CWD      = 0x0002	/* The current working directory is
+  DONE_CWD      = 0x0002,	/* The current working directory is
 				   correct.  */
+
+  /* 2013-10-17 Andrea Urbani (matfanjol)
+     For more information about the following entries, please,
+     look at ftp.c, function getftp, text "__LIST_A_EXPLANATION__". */
+  AVOID_LIST_A  = 0x0004,	/* It tells us if during this
+				 session we have to avoid the use
+				 of "LIST -a".*/
+  AVOID_LIST    = 0x0008,	/* It tells us if during this
+				 session we have to avoid to use
+				 "LIST". */
+  LIST_AFTER_LIST_A_CHECK_DONE  = 0x0010
+				/* It tells us if we have already
+				 checked "LIST" after the first
+				 "LIST -a" to handle the case of
+				 file/folders named "-a". */
 };
 
 struct fileinfo *ftp_parse_ls (const char *, const enum stype);

src/gnutls.c

@@ -1,5 +1,5 @@
 /* SSL support via GnuTLS library.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -45,69 +45,155 @@ as that of the covered work.  */
 #include "utils.h"
 #include "connect.h"
 #include "url.h"
+#include "ptimer.h"
+#include "hash.h"
 #include "ssl.h"
 
+#include <sys/fcntl.h>
+
 #ifdef WIN32
 # include "w32sock.h"
 #endif
 
+#include "host.h"
+
+static int
+key_type_to_gnutls_type (enum keyfile_type type)
+{
+  switch (type)
+    {
+    case keyfile_pem:
+      return GNUTLS_X509_FMT_PEM;
+    case keyfile_asn1:
+      return GNUTLS_X509_FMT_DER;
+    default:
+      abort ();
+    }
+}
+
 /* Note: some of the functions private to this file have names that
    begin with "wgnutls_" (e.g. wgnutls_read) so that they wouldn't be
    confused with actual gnutls functions -- such as the gnutls_read
    preprocessor macro.  */
 
-static gnutls_certificate_credentials credentials;
-
+static gnutls_certificate_credentials_t credentials;
 bool
-ssl_init ()
+ssl_init (void)
 {
+  /* Becomes true if GnuTLS is initialized. */
+  static bool ssl_initialized = false;
   const char *ca_directory;
   DIR *dir;
+  int ncerts = -1;
+
+  /* GnuTLS should be initialized only once. */
+  if (ssl_initialized)
+    return true;
 
   gnutls_global_init ();
   gnutls_certificate_allocate_credentials (&credentials);
-  gnutls_certificate_set_verify_flags(credentials,
-                                      GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
+  gnutls_certificate_set_verify_flags (credentials,
+                                       GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
 
-  ca_directory = opt.ca_directory ? opt.ca_directory : "/etc/ssl/certs";
+#if GNUTLS_VERSION_MAJOR >= 3
+  if (!opt.ca_directory)
+    ncerts = gnutls_certificate_set_x509_system_trust (credentials);
+#endif
 
-  dir = opendir (ca_directory);
-  if (dir == NULL)
+  /* If GnuTLS version is too old or CA loading failed, fallback to old behaviour.
+   * Also use old behaviour if the CA directory is user-provided.  */
+  if (ncerts <= 0)
     {
-      if (opt.ca_directory)
-        logprintf (LOG_NOTQUIET, _("ERROR: Cannot open directory %s.\n"),
-                   opt.ca_directory);
-    }
-  else
-    {
-      struct dirent *dent;
-      while ((dent = readdir (dir)) != NULL)
+      ca_directory = opt.ca_directory ? opt.ca_directory : "/etc/ssl/certs";
+      if ((dir = opendir (ca_directory)) == NULL)
         {
-          struct stat st;
-          char *ca_file;
-          asprintf (&ca_file, "%s/%s", ca_directory, dent->d_name);
+          if (opt.ca_directory && *opt.ca_directory)
+            logprintf (LOG_NOTQUIET, _("ERROR: Cannot open directory %s.\n"),
+                       opt.ca_directory);
+        }
+      else
+        {
+          struct hash_table *inode_map = hash_table_new (196, NULL, NULL);
+          struct dirent *dent;
+          size_t dirlen = strlen(ca_directory);
+          int rc;
 
-          stat (ca_file, &st);
+          ncerts = 0;
 
-          if (S_ISREG (st.st_mode))
-            gnutls_certificate_set_x509_trust_file (credentials, ca_file,
-                                                    GNUTLS_X509_FMT_PEM);
+          while ((dent = readdir (dir)) != NULL)
+            {
+              struct stat st;
+              char ca_file[dirlen + strlen(dent->d_name) + 2];
 
-          free (ca_file);
+              snprintf (ca_file, sizeof(ca_file), "%s/%s", ca_directory, dent->d_name);
+              if (stat (ca_file, &st) != 0)
+                continue;
+
+              if (! S_ISREG (st.st_mode))
+                continue;
+
+              /* avoid loading the same file twice by checking the inode.  */
+              if (hash_table_contains (inode_map, (void *)(intptr_t) st.st_ino))
+                continue;
+
+              hash_table_put (inode_map, (void *)(intptr_t) st.st_ino, NULL);
+              if ((rc = gnutls_certificate_set_x509_trust_file (credentials, ca_file,
+                                                                GNUTLS_X509_FMT_PEM)) <= 0)
+                logprintf (LOG_NOTQUIET, _("ERROR: Failed to open cert %s: (%d).\n"),
+                           ca_file, rc);
+              else
+                ncerts += rc;
+            }
+
+          hash_table_destroy (inode_map);
+          closedir (dir);
+        }
+    }
+
+  DEBUGP (("Certificates loaded: %d\n", ncerts));
+
+  /* Use the private key from the cert file unless otherwise specified. */
+  if (opt.cert_file && !opt.private_key)
+    {
+      opt.private_key = opt.cert_file;
+      opt.private_key_type = opt.cert_type;
+    }
+  /* Use the cert from the private key file unless otherwise specified. */
+  if (!opt.cert_file && opt.private_key)
+    {
+      opt.cert_file = opt.private_key;
+      opt.cert_type = opt.private_key_type;
+    }
+
+  if (opt.cert_file && opt.private_key)
+    {
+      int type;
+      if (opt.private_key_type != opt.cert_type)
+        {
+          /* GnuTLS can't handle this */
+          logprintf (LOG_NOTQUIET, _("ERROR: GnuTLS requires the key and the \
+cert to be of the same type.\n"));
         }
 
-      closedir (dir);
+      type = key_type_to_gnutls_type (opt.private_key_type);
+
+      gnutls_certificate_set_x509_key_file (credentials, opt.cert_file,
+                                            opt.private_key,
+                                            type);
     }
 
   if (opt.ca_cert)
     gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
                                             GNUTLS_X509_FMT_PEM);
+
+  ssl_initialized = true;
+
   return true;
 }
 
 struct wgnutls_transport_context
 {
-  gnutls_session session;       /* GnuTLS session handle */
+  gnutls_session_t session;       /* GnuTLS session handle */
   int last_error;               /* last error returned by read/write/... */
 
   /* Since GnuTLS doesn't support the equivalent to recv(...,
@@ -122,6 +208,78 @@ struct wgnutls_transport_context
 # define MIN(i, j) ((i) <= (j) ? (i) : (j))
 #endif
 
+
+static int
+wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout)
+{
+#ifdef F_GETFL
+  int flags = 0;
+#endif
+  int ret = 0;
+  struct ptimer *timer = NULL;
+  struct wgnutls_transport_context *ctx = arg;
+  int timed_out = 0;
+
+  if (timeout)
+    {
+#ifdef F_GETFL
+      flags = fcntl (fd, F_GETFL, 0);
+      if (flags < 0)
+        return flags;
+      if (fcntl (fd, F_SETFL, flags | O_NONBLOCK))
+        return -1;
+#else
+      /* XXX: Assume it was blocking before.  */
+      const int one = 1;
+      if (ioctl (fd, FIONBIO, &one) < 0)
+        return -1;
+#endif
+
+      timer = ptimer_new ();
+      if (timer == NULL)
+        return -1;
+    }
+
+  do
+    {
+      double next_timeout = 0;
+      if (timeout)
+        {
+          next_timeout = timeout - ptimer_measure (timer);
+          if (next_timeout < 0)
+            break;
+        }
+
+      ret = GNUTLS_E_AGAIN;
+      if (timeout == 0 || gnutls_record_check_pending (ctx->session)
+          || select_fd (fd, next_timeout, WAIT_FOR_READ))
+        {
+          ret = gnutls_record_recv (ctx->session, buf, bufsize);
+          timed_out = timeout && ptimer_measure (timer) >= timeout;
+        }
+    }
+  while (ret == GNUTLS_E_INTERRUPTED || (ret == GNUTLS_E_AGAIN && !timed_out));
+
+  if (timeout)
+    {
+      ptimer_destroy (timer);
+
+#ifdef F_GETFL
+      if (fcntl (fd, F_SETFL, flags) < 0)
+        return -1;
+#else
+      const int zero = 0;
+      if (ioctl (fd, FIONBIO, &zero) < 0)
+        return -1;
+#endif
+
+      if (timed_out && ret == GNUTLS_E_AGAIN)
+        errno = ETIMEDOUT;
+    }
+
+  return ret;
+}
+
 static int
 wgnutls_read (int fd, char *buf, int bufsize, void *arg)
 {
@@ -140,10 +298,7 @@ wgnutls_read (int fd, char *buf, int bufsize, void *arg)
       return copysize;
     }
 
-  do
-    ret = gnutls_record_recv (ctx->session, buf, bufsize);
-  while (ret == GNUTLS_E_INTERRUPTED || ret == GNUTLS_E_AGAIN);
-
+  ret = wgnutls_read_timeout (fd, buf, bufsize, arg, opt.read_timeout);
   if (ret < 0)
     ctx->last_error = ret;
 
@@ -151,7 +306,7 @@ wgnutls_read (int fd, char *buf, int bufsize, void *arg)
 }
 
 static int
-wgnutls_write (int fd, char *buf, int bufsize, void *arg)
+wgnutls_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg)
 {
   int ret;
   struct wgnutls_transport_context *ctx = arg;
@@ -167,8 +322,12 @@ static int
 wgnutls_poll (int fd, double timeout, int wait_for, void *arg)
 {
   struct wgnutls_transport_context *ctx = arg;
-  return ctx->peeklen || gnutls_record_check_pending (ctx->session)
-    || select_fd (fd, timeout, wait_for);
+
+  if (timeout)
+    return ctx->peeklen || gnutls_record_check_pending (ctx->session)
+      || select_fd (fd, timeout, wait_for);
+  else
+    return ctx->peeklen || gnutls_record_check_pending (ctx->session);
 }
 
 static int
@@ -177,21 +336,24 @@ wgnutls_peek (int fd, char *buf, int bufsize, void *arg)
   int read = 0;
   struct wgnutls_transport_context *ctx = arg;
   int offset = MIN (bufsize, ctx->peeklen);
-  if (bufsize > sizeof ctx->peekbuf)
-    bufsize = sizeof ctx->peekbuf;
 
   if (ctx->peeklen)
-    memcpy (buf, ctx->peekbuf, offset);
+    {
+      memcpy (buf, ctx->peekbuf, offset);
+      return offset;
+    }
+
+  if (bufsize > (int) sizeof ctx->peekbuf)
+    bufsize = sizeof ctx->peekbuf;
 
   if (bufsize > offset)
     {
-      if (gnutls_record_check_pending (ctx->session) <= 0
+      if (opt.read_timeout && gnutls_record_check_pending (ctx->session) == 0
           && select_fd (fd, 0.0, WAIT_FOR_READ) <= 0)
         read = 0;
       else
-        read = gnutls_record_recv (ctx->session, buf + offset,
-                                   bufsize - offset);
-        
+        read = wgnutls_read_timeout (fd, buf + offset, bufsize - offset,
+                                     ctx, opt.read_timeout);
       if (read < 0)
         {
           if (offset)
@@ -212,7 +374,7 @@ wgnutls_peek (int fd, char *buf, int bufsize, void *arg)
 }
 
 static const char *
-wgnutls_errstr (int fd, void *arg)
+wgnutls_errstr (int fd _GL_UNUSED, void *arg)
 {
   struct wgnutls_transport_context *ctx = arg;
   return gnutls_strerror (ctx->last_error);
@@ -238,38 +400,81 @@ static struct transport_implementation wgnutls_transport =
 };
 
 bool
-ssl_connect_wget (int fd)
+ssl_connect_wget (int fd, const char *hostname)
 {
-  static const int cert_type_priority[] = {
-    GNUTLS_CRT_X509, GNUTLS_CRT_OPENPGP, 0
-  };
+#ifdef F_GETFL
+  int flags = 0;
+#endif
   struct wgnutls_transport_context *ctx;
-  gnutls_session session;
-  int err;
+  gnutls_session_t session;
+  int err,alert;
   gnutls_init (&session, GNUTLS_CLIENT);
+  const char *str;
+
+  /* We set the server name but only if it's not an IP address. */
+  if (! is_valid_ip_address (hostname))
+    {
+      gnutls_server_name_set (session, GNUTLS_NAME_DNS, hostname,
+                              strlen (hostname));
+    }
+
   gnutls_set_default_priority (session);
-  gnutls_certificate_type_set_priority (session, cert_type_priority);
   gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
 #endif
-  gnutls_transport_set_ptr (session, (gnutls_transport_ptr) FD_TO_SOCKET (fd));
+#ifdef HAVE_INTPTR_T
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) (intptr_t) FD_TO_SOCKET (fd));
+#else
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd));
+#endif
 
   err = 0;
+#if HAVE_GNUTLS_PRIORITY_SET_DIRECT
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
       break;
     case secure_protocol_sslv2:
     case secure_protocol_sslv3:
-      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL", NULL);
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL);
       break;
     case secure_protocol_tlsv1:
       err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
       break;
+    case secure_protocol_pfs:
+      err = gnutls_priority_set_direct (session, "PFS", NULL);
+      if (err != GNUTLS_E_SUCCESS)
+        /* fallback if PFS is not available */
+        err = gnutls_priority_set_direct (session, "NORMAL:-RSA", NULL);
+      break;
     default:
       abort ();
     }
+#else
+  int allowed_protocols[4] = {0, 0, 0, 0};
+  switch (opt.secure_protocol)
+    {
+    case secure_protocol_auto:
+      break;
+    case secure_protocol_sslv2:
+    case secure_protocol_sslv3:
+      allowed_protocols[0] = GNUTLS_SSL3;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1:
+      allowed_protocols[0] = GNUTLS_TLS1_0;
+      allowed_protocols[1] = GNUTLS_TLS1_1;
+      allowed_protocols[2] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    default:
+      abort ();
+    }
+#endif
+
   if (err < 0)
     {
       logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
@@ -277,10 +482,82 @@ ssl_connect_wget (int fd)
       return false;
     }
 
-  err = gnutls_handshake (session);
+  if (opt.connect_timeout)
+    {
+#ifdef F_GETFL
+      flags = fcntl (fd, F_GETFL, 0);
+      if (flags < 0)
+        return flags;
+      if (fcntl (fd, F_SETFL, flags | O_NONBLOCK))
+        return -1;
+#else
+      /* XXX: Assume it was blocking before.  */
+      const int one = 1;
+      if (ioctl (fd, FIONBIO, &one) < 0)
+        return -1;
+#endif
+    }
+
+  /* We don't stop the handshake process for non-fatal errors */
+  do
+    {
+      err = gnutls_handshake (session);
+
+      if (opt.connect_timeout && err == GNUTLS_E_AGAIN)
+        {
+          if (gnutls_record_get_direction (session))
+            {
+              /* wait for writeability */
+              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_WRITE);
+            }
+          else
+            {
+              /* wait for readability */
+              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_READ);
+            }
+
+          if (err <= 0)
+            {
+              if (err == 0)
+                {
+                  errno = ETIMEDOUT;
+                  err = -1;
+                }
+              break;
+            }
+
+           err = GNUTLS_E_AGAIN;
+        }
+      else if (err < 0)
+        {
+          logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
+          if (err == GNUTLS_E_WARNING_ALERT_RECEIVED ||
+              err == GNUTLS_E_FATAL_ALERT_RECEIVED)
+            {
+              alert = gnutls_alert_get (session);
+              str = gnutls_alert_get_name (alert);
+              if (str == NULL)
+                str = "(unknown)";
+              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%d]: %s\n", alert, str);
+            }
+        }
+    }
+  while (err && gnutls_error_is_fatal (err) == 0);
+
+  if (opt.connect_timeout)
+    {
+#ifdef F_GETFL
+      if (fcntl (fd, F_SETFL, flags) < 0)
+        return -1;
+#else
+      const int zero = 0;
+      if (ioctl (fd, FIONBIO, &zero) < 0)
+        return -1;
+#endif
+    }
+
   if (err < 0)
     {
-      logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
       gnutls_deinit (session);
       return false;
     }
@@ -291,6 +568,14 @@ ssl_connect_wget (int fd)
   return true;
 }
 
+#define _CHECK_CERT(flag,msg) \
+  if (status & (flag))\
+    {\
+      logprintf (LOG_NOTQUIET, (msg),\
+                 severity, quote (host));\
+      success = false;\
+    }
+
 bool
 ssl_check_certificate (int fd, const char *host)
 {
@@ -313,30 +598,19 @@ ssl_check_certificate (int fd, const char *host)
       goto out;
     }
 
-  if (status & GNUTLS_CERT_INVALID)
-    {
-      logprintf (LOG_NOTQUIET, _("%s: The certificate of %s is not trusted.\n"),
-                 severity, quote (host));
-      success = false;
-    }
-  if (status & GNUTLS_CERT_SIGNER_NOT_FOUND)
-    {
-      logprintf (LOG_NOTQUIET, _("%s: The certificate of %s hasn't got a known issuer.\n"),
-                 severity, quote (host));
-      success = false;
-    }
-  if (status & GNUTLS_CERT_REVOKED)
-    {
-      logprintf (LOG_NOTQUIET, _("%s: The certificate of %s has been revoked.\n"),
-                 severity, quote (host));
-      success = false;
-    }
+  _CHECK_CERT (GNUTLS_CERT_INVALID, _("%s: The certificate of %s is not trusted.\n"));
+  _CHECK_CERT (GNUTLS_CERT_SIGNER_NOT_FOUND, _("%s: The certificate of %s hasn't got a known issuer.\n"));
+  _CHECK_CERT (GNUTLS_CERT_REVOKED, _("%s: The certificate of %s has been revoked.\n"));
+  _CHECK_CERT (GNUTLS_CERT_SIGNER_NOT_CA, _("%s: The certificate signer of %s was not a CA.\n"));
+  _CHECK_CERT (GNUTLS_CERT_INSECURE_ALGORITHM, _("%s: The certificate of %s was signed using an insecure algorithm.\n"));
+  _CHECK_CERT (GNUTLS_CERT_NOT_ACTIVATED, _("%s: The certificate of %s is not yet activated.\n"));
+  _CHECK_CERT (GNUTLS_CERT_EXPIRED, _("%s: The certificate of %s has expired.\n"));
 
   if (gnutls_certificate_type_get (ctx->session) == GNUTLS_CRT_X509)
     {
       time_t now = time (NULL);
-      gnutls_x509_crt cert;
-      const gnutls_datum *cert_list;
+      gnutls_x509_crt_t cert;
+      const gnutls_datum_t *cert_list;
       unsigned int cert_list_size;
 
       if ((err = gnutls_x509_crt_init (&cert)) < 0)
@@ -352,7 +626,7 @@ ssl_check_certificate (int fd, const char *host)
         {
           logprintf (LOG_NOTQUIET, _("No certificate found\n"));
           success = false;
-          goto out;
+          goto crt_deinit;
         }
       err = gnutls_x509_crt_import (cert, cert_list, GNUTLS_X509_FMT_DER);
       if (err < 0)
@@ -360,7 +634,7 @@ ssl_check_certificate (int fd, const char *host)
           logprintf (LOG_NOTQUIET, _("Error parsing certificate: %s\n"),
                      gnutls_strerror (err));
           success = false;
-          goto out;
+          goto crt_deinit;
         }
       if (now < gnutls_x509_crt_get_activation_time (cert))
         {
@@ -379,8 +653,14 @@ ssl_check_certificate (int fd, const char *host)
                      quote (host));
           success = false;
         }
+ crt_deinit:
       gnutls_x509_crt_deinit (cert);
-   }
+    }
+  else
+    {
+      logprintf (LOG_NOTQUIET, _("Certificate must be X.509\n"));
+      success = false;
+    }
 
  out:
   return opt.check_cert ? success : true;

src/hash.c

@@ -423,14 +423,14 @@ grow_hash_table (struct hash_table *ht)
    table if necessary.  */
 
 void
-hash_table_put (struct hash_table *ht, const void *key, void *value)
+hash_table_put (struct hash_table *ht, const void *key, const void *value)
 {
   struct cell *c = find_cell (ht, key);
   if (CELL_OCCUPIED (c))
     {
       /* update existing item */
       c->key   = (void *)key; /* const? */
-      c->value = value;
+      c->value = (void *)value;
       return;
     }
 
@@ -445,7 +445,7 @@ hash_table_put (struct hash_table *ht, const void *key, void *value)
   /* add new item */
   ++ht->count;
   c->key   = (void *)key;       /* const? */
-  c->value = value;
+  c->value = (void *)value;
 }
 
 /* Remove KEY->value mapping from HT.  Return 0 if there was no such

src/hash.h

@@ -42,7 +42,7 @@ int hash_table_get_pair (const struct hash_table *, const void *,
                          void *, void *);
 int hash_table_contains (const struct hash_table *, const void *);
 
-void hash_table_put (struct hash_table *, const void *, void *);
+void hash_table_put (struct hash_table *, const void *, const void *);
 int hash_table_remove (struct hash_table *, const void *);
 void hash_table_clear (struct hash_table *);
 

src/host.c

@@ -1,6 +1,6 @@
 /* Host name resolution and matching.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -914,3 +914,18 @@ host_cleanup (void)
       host_name_addresses_map = NULL;
     }
 }
+
+bool
+is_valid_ip_address (const char *name)
+{
+  const char *endp;
+
+  endp = name + strlen(name);
+  if (is_valid_ipv4_address (name, endp))
+    return true;
+#ifdef ENABLE_IPV6
+  if (is_valid_ipv6_address (name, endp))
+    return true;
+#endif
+  return false;
+}

src/host.h

@@ -1,6 +1,6 @@
 /* Declarations for host.c
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -98,6 +98,8 @@ const char *print_address (const ip_address *);
 bool is_valid_ipv6_address (const char *, const char *);
 #endif
 
+bool is_valid_ip_address (const char *name);
+
 bool accept_domain (struct url *);
 bool sufmatch (const char **, const char *);
 

src/html-parse.c

@@ -280,7 +280,7 @@ struct tagstack_item {
   struct tagstack_item *next;
 };
 
-struct tagstack_item *
+static struct tagstack_item *
 tagstack_push (struct tagstack_item **head, struct tagstack_item **tail)
 {
   struct tagstack_item *ts = xmalloc(sizeof(struct tagstack_item));
@@ -301,7 +301,7 @@ tagstack_push (struct tagstack_item **head, struct tagstack_item **tail)
 }
 
 /* remove ts and everything after it from the stack */
-void
+static void
 tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail,
               struct tagstack_item *ts)
 {
@@ -343,7 +343,7 @@ tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail,
     }
 }
 
-struct tagstack_item *
+static struct tagstack_item *
 tagstack_find (struct tagstack_item *tail, const char *tagname_begin,
                const char *tagname_end)
 {

src/html-url.c

@@ -1,6 +1,6 @@
 /* Collect URLs from HTML source.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -79,7 +79,10 @@ enum {
   TAG_SCRIPT,
   TAG_TABLE,
   TAG_TD,
-  TAG_TH
+  TAG_TH,
+  TAG_VIDEO,
+  TAG_AUDIO,
+  TAG_SOURCE
 };
 
 /* The list of known tags and functions used for handling them.  Most
@@ -110,7 +113,10 @@ static struct known_tag {
   { TAG_SCRIPT,  "script",      tag_find_urls },
   { TAG_TABLE,   "table",       tag_find_urls },
   { TAG_TD,      "td",          tag_find_urls },
-  { TAG_TH,      "th",          tag_find_urls }
+  { TAG_TH,      "th",          tag_find_urls },
+  { TAG_VIDEO,   "video",       tag_find_urls },
+  { TAG_AUDIO,   "audio",       tag_find_urls },
+  { TAG_SOURCE,  "source",      tag_find_urls }
 };
 
 /* tag_url_attributes documents which attributes of which tags contain
@@ -157,7 +163,12 @@ static struct {
   { TAG_SCRIPT,         "src",          ATTR_INLINE },
   { TAG_TABLE,          "background",   ATTR_INLINE },
   { TAG_TD,             "background",   ATTR_INLINE },
-  { TAG_TH,             "background",   ATTR_INLINE }
+  { TAG_TH,             "background",   ATTR_INLINE },
+  { TAG_VIDEO,          "src",          ATTR_INLINE },
+  { TAG_VIDEO,          "poster",       ATTR_INLINE },
+  { TAG_AUDIO,          "src",          ATTR_INLINE },
+  { TAG_AUDIO,          "poster",       ATTR_INLINE },
+  { TAG_SOURCE,         "src",          ATTR_INLINE }
 };
 
 /* The lists of interesting tags and attributes are built dynamically,
@@ -273,6 +284,10 @@ append_url (const char *link_uri, int position, int size,
   const char *base = ctx->base ? ctx->base : ctx->parent_base;
   struct url *url;
 
+  struct iri *iri = iri_new ();
+  set_uri_encoding (iri, opt.locale, true);
+  iri->utf8_encode = true;
+
   if (!base)
     {
       DEBUGP (("%s: no base, merge will use \"%s\".\n",
@@ -290,7 +305,7 @@ append_url (const char *link_uri, int position, int size,
           return NULL;
         }
 
-      url = url_parse (link_uri, NULL, NULL, false);
+      url = url_parse (link_uri, NULL, iri, false);
       if (!url)
         {
           DEBUGP (("%s: link \"%s\" doesn't parse.\n",
@@ -312,7 +327,7 @@ append_url (const char *link_uri, int position, int size,
                quote_n (2, link_uri),
                quotearg_n_style (3, escape_quoting_style, complete_uri)));
 
-      url = url_parse (complete_uri, NULL, NULL, false);
+      url = url_parse (complete_uri, NULL, iri, false);
       if (!url)
         {
           DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
@@ -323,6 +338,8 @@ append_url (const char *link_uri, int position, int size,
       xfree (complete_uri);
     }
 
+  iri_free (iri);
+
   DEBUGP (("appending %s to urlpos.\n", quote (url->url)));
 
   newel = xnew0 (struct urlpos);
@@ -675,8 +692,9 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
 
   check_style_attr (tag, ctx);
 
-  if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style")) &&
-      tag->contents_begin && tag->contents_end)
+  if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style"))
+      && tag->contents_begin && tag->contents_end
+      && tag->contents_begin <= tag->contents_end)
   {
     /* parse contents */
     get_urls_css (ctx, tag->contents_begin - ctx->text,
@@ -803,6 +821,13 @@ get_urls_file (const char *file)
           url_text = merged;
         }
 
+      char *new_url = rewrite_shorthand_url (url_text);
+      if (new_url)
+        {
+          xfree (url_text);
+          url_text = new_url;
+        }
+
       url = url_parse (url_text, &up_error_code, NULL, false);
       if (!url)
         {

src/http-ntlm.c

@@ -42,27 +42,33 @@ as that of the covered work.  */
 #include <string.h>
 #include <stdlib.h>
 
-#include <openssl/des.h>
-#include <openssl/md4.h>
-#include <openssl/opensslv.h>
-
 #include "utils.h"
 #include "http-ntlm.h"
 
-#if OPENSSL_VERSION_NUMBER < 0x00907001L
-#define DES_key_schedule des_key_schedule
-#define DES_cblock des_cblock
-#define DES_set_odd_parity des_set_odd_parity
-#define DES_set_key des_set_key
-#define DES_ecb_encrypt des_ecb_encrypt
+#ifdef HAVE_NETTLE
+# include <nettle/md4.h>
+# include <nettle/des.h>
+#else
+# include <openssl/des.h>
+# include <openssl/md4.h>
+# include <openssl/opensslv.h>
+
+# if OPENSSL_VERSION_NUMBER < 0x00907001L
+#  define DES_key_schedule des_key_schedule
+#  define DES_cblock des_cblock
+#  define DES_set_odd_parity des_set_odd_parity
+#  define DES_set_key des_set_key
+#  define DES_ecb_encrypt des_ecb_encrypt
 
 /* This is how things were done in the old days */
-#define DESKEY(x) x
-#define DESKEYARG(x) x
-#else
+#  define DESKEY(x) x
+#  define DESKEYARG(x) x
+# else
 /* Modern version */
-#define DESKEYARG(x) *x
-#define DESKEY(x) &x
+#  define DESKEYARG(x) *x
+#  define DESKEY(x) &x
+# endif
+
 #endif
 
 /* Define this to make the type-3 message include the NT response message */
@@ -176,6 +182,25 @@ ntlm_input (struct ntlmdata *ntlm, const char *header)
  * Turns a 56 bit key into the 64 bit, odd parity key and sets the key.  The
  * key schedule ks is also set.
  */
+#ifdef HAVE_NETTLE
+static void
+setup_des_key(unsigned char *key_56,
+              struct des_ctx *des)
+{
+  unsigned char key[8];
+
+  key[0] = key_56[0];
+  key[1] = ((key_56[0] << 7) & 0xFF) | (key_56[1] >> 1);
+  key[2] = ((key_56[1] << 6) & 0xFF) | (key_56[2] >> 2);
+  key[3] = ((key_56[2] << 5) & 0xFF) | (key_56[3] >> 3);
+  key[4] = ((key_56[3] << 4) & 0xFF) | (key_56[4] >> 4);
+  key[5] = ((key_56[4] << 3) & 0xFF) | (key_56[5] >> 5);
+  key[6] = ((key_56[5] << 2) & 0xFF) | (key_56[6] >> 6);
+  key[7] =  (key_56[6] << 1) & 0xFF;
+
+  nettle_des_set_key(des, key);
+}
+#else
 static void
 setup_des_key(unsigned char *key_56,
               DES_key_schedule DESKEYARG(ks))
@@ -194,6 +219,7 @@ setup_des_key(unsigned char *key_56,
   DES_set_odd_parity(&key);
   DES_set_key(&key, ks);
 }
+#endif
 
  /*
   * takes a 21 byte array and treats it as 3 56-bit DES keys. The
@@ -203,6 +229,18 @@ setup_des_key(unsigned char *key_56,
 static void
 calc_resp(unsigned char *keys, unsigned char *plaintext, unsigned char *results)
 {
+#ifdef HAVE_NETTLE
+  struct des_ctx des;
+
+  setup_des_key(keys, &des);
+  nettle_des_encrypt(&des, 8, results, plaintext);
+
+  setup_des_key(keys + 7, &des);
+  nettle_des_encrypt(&des, 8, results + 8, plaintext);
+
+  setup_des_key(keys + 14, &des);
+  nettle_des_encrypt(&des, 8, results + 16, plaintext);
+#else
   DES_key_schedule ks;
 
   setup_des_key(keys, DESKEY(ks));
@@ -216,6 +254,7 @@ calc_resp(unsigned char *keys, unsigned char *plaintext, unsigned char *results)
   setup_des_key(keys+14, DESKEY(ks));
   DES_ecb_encrypt((DES_cblock*) plaintext, (DES_cblock*) (results+16),
                   DESKEY(ks), DES_ENCRYPT);
+#endif
 }
 
 /*
@@ -255,6 +294,15 @@ mkhash(const char *password,
 
   {
     /* create LanManager hashed password */
+#ifdef HAVE_NETTLE
+    struct des_ctx des;
+
+    setup_des_key(pw, &des);
+    nettle_des_encrypt(&des, 8, lmbuffer, magic);
+
+    setup_des_key(pw + 7, &des);
+    nettle_des_encrypt(&des, 8, lmbuffer + 8, magic);
+#else
     DES_key_schedule ks;
 
     setup_des_key(pw, DESKEY(ks));
@@ -264,6 +312,7 @@ mkhash(const char *password,
     setup_des_key(pw+7, DESKEY(ks));
     DES_ecb_encrypt((DES_cblock *)magic, (DES_cblock *)(lmbuffer+8),
                     DESKEY(ks), DES_ENCRYPT);
+#endif
 
     memset(lmbuffer+16, 0, 5);
   }
@@ -272,8 +321,11 @@ mkhash(const char *password,
 
 #ifdef USE_NTRESPONSES
   {
-    /* create NT hashed password */
+#ifdef HAVE_NETTLE
+    struct md4_ctx MD4;
+#else
     MD4_CTX MD4;
+#endif
 
     len = strlen(password);
 
@@ -282,9 +334,16 @@ mkhash(const char *password,
       pw[2*i+1] = 0;
     }
 
+#ifdef HAVE_NETTLE
+    nettle_md4_init(&MD4);
+    nettle_md4_update(&MD4, 2*len, pw);
+    nettle_md4_digest(&MD4, MD4_DIGEST_SIZE, ntbuffer);
+#else
+    /* create NT hashed password */
     MD4_Init(&MD4);
     MD4_Update(&MD4, pw, 2*len);
     MD4_Final(ntbuffer, &MD4);
+#endif
 
     memset(ntbuffer+16, 0, 5);
   }

src/init.c

@@ -1,6 +1,6 @@
 /* Reading/parsing the initialization file.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -30,6 +30,7 @@ shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
 #include "wget.h"
+#include "exits.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -46,6 +47,10 @@ as that of the covered work.  */
 # endif
 #endif
 
+#include <regex.h>
+#ifdef HAVE_LIBPCRE
+# include <pcre.h>
+#endif
 
 #ifdef HAVE_PWD_H
 # include <pwd.h>
@@ -62,6 +67,7 @@ as that of the covered work.  */
 #include "res.h"                /* for res_cleanup */
 #include "http.h"               /* for http_cleanup */
 #include "retr.h"               /* for output_stream */
+#include "warc.h"               /* for warc_close */
 
 #ifdef TESTING
 #include "test.h"
@@ -81,6 +87,7 @@ CMD_DECLARE (cmd_directory_vector);
 CMD_DECLARE (cmd_number);
 CMD_DECLARE (cmd_number_inf);
 CMD_DECLARE (cmd_string);
+CMD_DECLARE (cmd_string_uppercase);
 CMD_DECLARE (cmd_file);
 CMD_DECLARE (cmd_directory);
 CMD_DECLARE (cmd_time);
@@ -88,12 +95,15 @@ CMD_DECLARE (cmd_vector);
 
 CMD_DECLARE (cmd_spec_dirstruct);
 CMD_DECLARE (cmd_spec_header);
+CMD_DECLARE (cmd_spec_warc_header);
 CMD_DECLARE (cmd_spec_htmlify);
 CMD_DECLARE (cmd_spec_mirror);
 CMD_DECLARE (cmd_spec_prefer_family);
 CMD_DECLARE (cmd_spec_progress);
 CMD_DECLARE (cmd_spec_recursive);
+CMD_DECLARE (cmd_spec_regex_type);
 CMD_DECLARE (cmd_spec_restrict_file_names);
+CMD_DECLARE (cmd_spec_report_speed);
 #ifdef HAVE_SSL
 CMD_DECLARE (cmd_spec_secure_protocol);
 #endif
@@ -115,6 +125,7 @@ static const struct {
 } commands[] = {
   /* KEEP THIS LIST ALPHABETICALLY SORTED */
   { "accept",           &opt.accepts,           cmd_vector },
+  { "acceptregex",      &opt.acceptregex_s,     cmd_string },
   { "addhostdir",       &opt.add_hostdir,       cmd_boolean },
   { "adjustextension",  &opt.adjust_extension,  cmd_boolean },
   { "alwaysrest",       &opt.always_rest,       cmd_boolean }, /* deprecated */
@@ -126,6 +137,8 @@ static const struct {
   { "backups",          &opt.backups,           cmd_number },
   { "base",             &opt.base_href,         cmd_string },
   { "bindaddress",      &opt.bind_address,      cmd_string },
+  { "bodydata",         &opt.body_data,         cmd_string },
+  { "bodyfile",         &opt.body_file,         cmd_string },
 #ifdef HAVE_SSL
   { "cacertificate",    &opt.ca_cert,           cmd_file },
 #endif
@@ -139,6 +152,7 @@ static const struct {
   { "chooseconfig",     &opt.choose_config,	cmd_file },
   { "connecttimeout",   &opt.connect_timeout,   cmd_time },
   { "contentdisposition", &opt.content_disposition, cmd_boolean },
+  { "contentonerror",   &opt.content_on_error,  cmd_boolean },
   { "continue",         &opt.always_rest,       cmd_boolean },
   { "convertlinks",     &opt.convert_links,     cmd_boolean },
   { "cookies",          &opt.cookies,           cmd_boolean },
@@ -146,7 +160,7 @@ static const struct {
 #ifdef ENABLE_DEBUG
   { "debug",            &opt.debug,             cmd_boolean },
 #endif
-  { "defaultpage", 	&opt.default_page,      cmd_string},
+  { "defaultpage", 	&opt.default_page,      cmd_string },
   { "deleteafter",      &opt.delete_after,      cmd_boolean },
   { "dirprefix",        &opt.dir_prefix,        cmd_directory },
   { "dirstruct",        NULL,                   cmd_spec_dirstruct },
@@ -180,6 +194,9 @@ static const struct {
   { "httppasswd",       &opt.http_passwd,       cmd_string }, /* deprecated */
   { "httppassword",     &opt.http_passwd,       cmd_string },
   { "httpproxy",        &opt.http_proxy,        cmd_string },
+#ifdef HAVE_SSL
+  { "httpsonly",        &opt.https_only,        cmd_boolean },
+#endif
   { "httpsproxy",       &opt.https_proxy,       cmd_string },
   { "httpuser",         &opt.http_user,         cmd_string },
   { "ignorecase",       &opt.ignore_case,       cmd_boolean },
@@ -199,6 +216,7 @@ static const struct {
   { "logfile",          &opt.lfilename,         cmd_file },
   { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
   { "maxredirect",      &opt.max_redirect,      cmd_number },
+  { "method",           &opt.method,            cmd_string_uppercase },
   { "mirror",           NULL,                   cmd_spec_mirror },
   { "netrc",            &opt.netrc,             cmd_boolean },
   { "noclobber",        &opt.noclobber,         cmd_boolean },
@@ -213,7 +231,7 @@ static const struct {
   { "postdata",         &opt.post_data,         cmd_string },
   { "postfile",         &opt.post_file_name,    cmd_file },
   { "preferfamily",     NULL,                   cmd_spec_prefer_family },
-  { "preservepermissions", &opt.preserve_perm,  cmd_boolean },/* deprecated */
+  { "preservepermissions", &opt.preserve_perm,  cmd_boolean },
 #ifdef HAVE_SSL
   { "privatekey",       &opt.private_key,       cmd_file },
   { "privatekeytype",   &opt.private_key_type,  cmd_cert_type },
@@ -233,10 +251,13 @@ static const struct {
   { "reclevel",         &opt.reclevel,          cmd_number_inf },
   { "recursive",        NULL,                   cmd_spec_recursive },
   { "referer",          &opt.referer,           cmd_string },
+  { "regextype",        &opt.regex_type,        cmd_spec_regex_type },
   { "reject",           &opt.rejects,           cmd_vector },
+  { "rejectregex",      &opt.rejectregex_s,     cmd_string },
   { "relativeonly",     &opt.relative_only,     cmd_boolean },
   { "remoteencoding",   &opt.encoding_remote,   cmd_string },
   { "removelisting",    &opt.remove_listing,    cmd_boolean },
+  { "reportspeed",             &opt.report_bps, cmd_spec_report_speed},
   { "restrictfilenames", NULL,                  cmd_spec_restrict_file_names },
   { "retrsymlinks",     &opt.retr_symlinks,     cmd_boolean },
   { "retryconnrefused", &opt.retry_connrefused, cmd_boolean },
@@ -263,6 +284,17 @@ static const struct {
   { "verbose",          NULL,                   cmd_spec_verbose },
   { "wait",             &opt.wait,              cmd_time },
   { "waitretry",        &opt.waitretry,         cmd_time },
+  { "warccdx",          &opt.warc_cdx_enabled,  cmd_boolean },
+  { "warccdxdedup",     &opt.warc_cdx_dedup_filename,  cmd_file },
+#ifdef HAVE_LIBZ
+  { "warccompression",  &opt.warc_compression_enabled, cmd_boolean },
+#endif
+  { "warcdigests",      &opt.warc_digests_enabled, cmd_boolean },
+  { "warcfile",         &opt.warc_filename,     cmd_file },
+  { "warcheader",       NULL,                   cmd_spec_warc_header },
+  { "warckeeplog",      &opt.warc_keep_log,     cmd_boolean },
+  { "warcmaxsize",      &opt.warc_maxsize,      cmd_bytes },
+  { "warctempdir",      &opt.warc_tempdir,      cmd_directory },
 #ifdef USE_WATT32
   { "wdebug",           &opt.wdebug,            cmd_boolean },
 #endif
@@ -347,6 +379,8 @@ defaults (void)
   opt.restrict_files_nonascii = false;
   opt.restrict_files_case = restrict_no_case_restriction;
 
+  opt.regex_type = regex_type_posix;
+
   opt.max_redirect = 20;
 
   opt.waitretry = 10;
@@ -361,6 +395,18 @@ defaults (void)
 
   opt.useservertimestamps = true;
   opt.show_all_dns_entries = false;
+
+  opt.warc_maxsize = 0; /* 1024 * 1024 * 1024; */
+#ifdef HAVE_LIBZ
+  opt.warc_compression_enabled = true;
+#else
+  opt.warc_compression_enabled = false;
+#endif
+  opt.warc_digests_enabled = true;
+  opt.warc_cdx_enabled = false;
+  opt.warc_cdx_dedup_filename = NULL;
+  opt.warc_tempdir = NULL;
+  opt.warc_keep_log = true;
 }
 
 /* Return the user's home directory (strdup-ed), or NULL if none is
@@ -370,7 +416,6 @@ home_dir (void)
 {
   static char *buf = NULL;
   static char *home, *ret;
-  int len;
 
   if (!home)
     {
@@ -378,6 +423,8 @@ home_dir (void)
       if (!home)
         {
 #if defined(MSDOS)
+          int len;
+
           /* Under MSDOS, if $HOME isn't defined, use the directory where
              `wget.exe' resides.  */
           const char *_w32_get_argv0 (void); /* in libwatt.a/pcconfig.c */
@@ -415,8 +462,7 @@ home_dir (void)
     }
 
   ret = home ? xstrdup (home) : NULL;
-  if (buf)
-    free (buf);
+  free (buf);
 
   return ret;
 }
@@ -442,7 +488,7 @@ wgetrc_env_file_name (void)
   return NULL;
 }
 
-/* Check for the existance of '$HOME/.wgetrc' and return it's path
+/* Check for the existance of '$HOME/.wgetrc' and return its path
    if it exists and is set.  */
 char *
 wgetrc_user_file_name (void)
@@ -530,7 +576,8 @@ bool
 run_wgetrc (const char *file)
 {
   FILE *fp;
-  char *line;
+  char *line = NULL;
+  size_t bufsize = 0;
   int ln;
   int errcnt = 0;
 
@@ -542,7 +589,7 @@ run_wgetrc (const char *file)
       return true;                      /* not a fatal error */
     }
   ln = 1;
-  while ((line = read_whole_line (fp)) != NULL)
+  while (getline (&line, &bufsize, fp) > 0)
     {
       char *com = NULL, *val = NULL;
       int comind;
@@ -576,9 +623,9 @@ run_wgetrc (const char *file)
         }
       xfree_null (com);
       xfree_null (val);
-      xfree (line);
       ++ln;
     }
+  xfree (line);
   fclose (fp);
 
   return errcnt == 0;
@@ -596,21 +643,34 @@ initialize (void)
      variable has been set. For internal testing purposes only!  */
   env_sysrc = getenv ("SYSTEM_WGETRC");
   if (env_sysrc && file_exists_p (env_sysrc))
-    ok &= run_wgetrc (env_sysrc);
+    {
+      ok &= run_wgetrc (env_sysrc);
+      /* If there are any problems parsing the system wgetrc file, tell
+         the user and exit */
+      if (! ok)
+        {
+          fprintf (stderr, _("\
+Parsing system wgetrc file (env SYSTEM_WGETRC) failed.  Please check\n\
+'%s',\n\
+or specify a different file using --config.\n"), env_sysrc);
+          exit (2);
+        }
+    }
   /* Otherwise, if SYSTEM_WGETRC is defined, use it.  */
 #ifdef SYSTEM_WGETRC
   else if (file_exists_p (SYSTEM_WGETRC))
     ok &= run_wgetrc (SYSTEM_WGETRC);
-#endif
   /* If there are any problems parsing the system wgetrc file, tell
      the user and exit */
   if (! ok)
     {
       fprintf (stderr, _("\
-Parsing system wgetrc file failed, please check '%s'.           \
-Or specify a different file using --config\n"), SYSTEM_WGETRC);
+Parsing system wgetrc file failed.  Please check\n\
+'%s',\n\
+or specify a different file using --config.\n"), SYSTEM_WGETRC);
       exit (2);
     }
+#endif
   /* Override it with your own, if one exists.  */
   file = wgetrc_file_name ();
   if (!file)
@@ -903,8 +963,25 @@ cmd_string (const char *com, const char *val, void *place)
   return true;
 }
 
+/* Like cmd_string but ensure the string is upper case.  */
+static bool
+cmd_string_uppercase (const char *com, const char *val, void *place)
+{
+  char *q, **pstring;
+  pstring = (char **)place;
+  xfree_null (*pstring);
 
-/* Like the above, but handles tilde-expansion when reading a user's
+  *pstring = xmalloc (strlen (val) + 1);
+
+  for (q = *pstring; *val; val++, q++)
+    *q = c_toupper (*val);
+
+  *q = '\0';
+  return true;
+}
+
+
+/* Like cmd_string, but handles tilde-expansion when reading a user's
    `.wgetrc'.  In that case, and if VAL begins with `~', the tilde
    gets expanded to the user's home directory.  */
 static bool
@@ -1220,6 +1297,27 @@ cmd_spec_header (const char *com, const char *val, void *place_ignored)
   return true;
 }
 
+static bool
+cmd_spec_warc_header (const char *com, const char *val, void *place_ignored)
+{
+  /* Empty value means reset the list of headers. */
+  if (*val == '\0')
+    {
+      free_vec (opt.warc_user_headers);
+      opt.warc_user_headers = NULL;
+      return true;
+    }
+
+  if (!check_user_specified_header (val))
+    {
+      fprintf (stderr, _("%s: %s: Invalid WARC header %s.\n"),
+               exec_name, com, quote (val));
+      return false;
+    }
+  opt.warc_user_headers = vec_append (opt.warc_user_headers, val);
+  return true;
+}
+
 static bool
 cmd_spec_htmlify (const char *com, const char *val, void *place_ignored)
 {
@@ -1307,6 +1405,25 @@ cmd_spec_recursive (const char *com, const char *val, void *place_ignored)
   return true;
 }
 
+/* Validate --regex-type and set the choice.  */
+
+static bool
+cmd_spec_regex_type (const char *com, const char *val, void *place_ignored)
+{
+  static const struct decode_item choices[] = {
+    { "posix", regex_type_posix },
+#ifdef HAVE_LIBPCRE
+    { "pcre",  regex_type_pcre },
+#endif
+  };
+  int regex_type = regex_type_posix;
+  int ok = decode_string (val, choices, countof (choices), &regex_type);
+  if (!ok)
+    fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val));
+  opt.regex_type = regex_type;
+  return ok;
+}
+
 static bool
 cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored)
 {
@@ -1361,6 +1478,15 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
   return true;
 }
 
+static bool
+cmd_spec_report_speed (const char *com, const char *val, void *place_ignored)
+{
+  opt.report_bps = strcasecmp (val, "bits") == 0;
+  if (!opt.report_bps)
+    fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val));
+  return opt.report_bps;
+}
+
 #ifdef HAVE_SSL
 static bool
 cmd_spec_secure_protocol (const char *com, const char *val, void *place)
@@ -1370,6 +1496,7 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
     { "sslv2", secure_protocol_sslv2 },
     { "sslv3", secure_protocol_sslv3 },
     { "tlsv1", secure_protocol_tlsv1 },
+    { "pfs", secure_protocol_pfs },
   };
   int ok = decode_string (val, choices, countof (choices), place);
   if (!ok)
@@ -1567,6 +1694,7 @@ decode_string (const char *val, const struct decode_item *items, int itemcount,
 
 
 void cleanup_html_url (void);
+void spider_cleanup (void);
 
 
 /* Free the memory allocated by global variables.  */
@@ -1575,8 +1703,16 @@ cleanup (void)
 {
   /* Free external resources, close files, etc. */
 
+  /* Close WARC file. */
+  if (opt.warc_filename != 0)
+    warc_close ();
+
+  log_close ();
+
   if (output_stream)
-    fclose (output_stream);
+    if (fclose (output_stream) == EOF)
+      inform_exit_status (CLOSEFAILED);
+
   /* No need to check for error because Wget flushes its output (and
      checks for errors) after any data arrives.  */
 
@@ -1593,9 +1729,13 @@ cleanup (void)
   res_cleanup ();
   http_cleanup ();
   cleanup_html_url ();
+  spider_cleanup ();
   host_cleanup ();
   log_cleanup ();
 
+  for (i = 0; i < nurl; i++)
+    xfree (url[i]);
+
   {
     extern acc_t *netrc_list;
     free_netrc (netrc_list);
@@ -1624,6 +1764,7 @@ cleanup (void)
   xfree_null (opt.http_user);
   xfree_null (opt.http_passwd);
   free_vec (opt.user_headers);
+  free_vec (opt.warc_user_headers);
 # ifdef HAVE_SSL
   xfree_null (opt.cert_file);
   xfree_null (opt.private_key);
@@ -1638,6 +1779,7 @@ cleanup (void)
   xfree_null (opt.user);
   xfree_null (opt.passwd);
   xfree_null (opt.base_href);
+  xfree_null (opt.method);
 
 #endif /* DEBUG_MALLOC */
 }

src/log.c

@@ -79,6 +79,10 @@ as that of the covered work.  */
    logging is inhibited, logfp is set back to NULL. */
 static FILE *logfp;
 
+/* A second file descriptor pointing to the temporary log file for the
+   WARC writer.  If WARC writing is disabled, this is NULL.  */
+static FILE *warclogfp;
+
 /* If true, it means logging is inhibited, i.e. nothing is printed or
    stored.  */
 static bool inhibit_logging;
@@ -304,6 +308,31 @@ get_log_fp (void)
     return logfp;
   return stderr;
 }
+
+/* Returns the file descriptor for the secondary log file. This is
+   WARCLOGFP, except if called before log_init, in which case it
+   returns stderr.  This is useful in case someone calls a logging
+   function before log_init.
+
+   If logging is inhibited, return NULL.  */
+
+static FILE *
+get_warc_log_fp (void)
+{
+  if (inhibit_logging)
+    return NULL;
+  if (warclogfp)
+    return warclogfp;
+  return NULL;
+}
+
+/* Sets the file descriptor for the secondary log file.  */
+
+void
+log_set_warc_log_fp (FILE * fp)
+{
+  warclogfp = fp;
+}
 
 /* Log a literal string S.  The string is logged as-is, without a
    newline appended.  */
@@ -312,13 +341,17 @@ void
 logputs (enum log_options o, const char *s)
 {
   FILE *fp;
+  FILE *warcfp;
 
   check_redirect_output ();
   if ((fp = get_log_fp ()) == NULL)
     return;
+  warcfp = get_warc_log_fp ();
   CHECK_VERBOSE (o);
 
   FPUTS (s, fp);
+  if (warcfp != NULL)
+    FPUTS (s, warcfp);
   if (save_context_p)
     saved_append (s);
   if (flush_log_p)
@@ -356,8 +389,9 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
   int available_size = sizeof (smallmsg);
   int numwritten;
   FILE *fp = get_log_fp ();
+  FILE *warcfp = get_warc_log_fp ();
 
-  if (!save_context_p)
+  if (!save_context_p && warcfp == NULL)
     {
       /* In the simple case just call vfprintf(), to avoid needless
          allocation and games with vsnprintf(). */
@@ -407,8 +441,11 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
     }
 
   /* Writing succeeded. */
-  saved_append (write_ptr);
+  if (save_context_p)
+    saved_append (write_ptr);
   FPUTS (write_ptr, fp);
+  if (warcfp != NULL)
+    FPUTS (write_ptr, warcfp);
   if (state->bigmsg)
     xfree (state->bigmsg);
 
@@ -426,6 +463,7 @@ void
 logflush (void)
 {
   FILE *fp = get_log_fp ();
+  FILE *warcfp = get_warc_log_fp ();
   if (fp)
     {
 /* 2005-10-25 SMS.
@@ -440,6 +478,10 @@ logflush (void)
       fflush (fp);
 #endif /* def __VMS [else] */
     }
+
+  if (warcfp != NULL)
+    fflush (warcfp);
+
   needs_flushing = false;
 }
 
@@ -573,14 +615,14 @@ log_init (const char *file, bool appendp)
     }
 }
 
-/* Close LOGFP, inhibit further logging and free the memory associated
-   with it.  */
+/* Close LOGFP (only if we opened it, not if it's stderr), inhibit
+   further logging and free the memory associated with it.  */
 void
 log_close (void)
 {
   int i;
 
-  if (logfp)
+  if (logfp && (logfp != stderr))
     fclose (logfp);
   logfp = NULL;
   inhibit_logging = true;
@@ -598,6 +640,7 @@ log_dump_context (void)
 {
   int num = log_line_current;
   FILE *fp = get_log_fp ();
+  FILE *warcfp = get_warc_log_fp ();
   if (!fp)
     return;
 
@@ -609,14 +652,23 @@ log_dump_context (void)
     {
       struct log_ln *ln = log_lines + num;
       if (ln->content)
-        FPUTS (ln->content, fp);
+        {
+          FPUTS (ln->content, fp);
+          if (warcfp != NULL)
+            FPUTS (ln->content, warcfp);
+        }
       ROT_ADVANCE (num);
     }
   while (num != log_line_current);
   if (trailing_line)
     if (log_lines[log_line_current].content)
-      FPUTS (log_lines[log_line_current].content, fp);
+      {
+        FPUTS (log_lines[log_line_current].content, fp);
+        if (warcfp != NULL)
+          FPUTS (log_lines[log_line_current].content, warcfp);
+      }
   fflush (fp);
+  fflush (warcfp);
 }
 
 /* String escape functions. */
@@ -819,7 +871,7 @@ redirect_output (void)
          can do but disable printing completely. */
       fprintf (stderr, _("\n%s received.\n"), redirect_request_signal_name);
       fprintf (stderr, _("%s: %s; disabling logging.\n"),
-               logfile, strerror (errno));
+               (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
       inhibit_logging = true;
     }
   save_context_p = false;

src/log.h

@@ -34,8 +34,12 @@ as that of the covered work.  */
 /* The log file to which Wget writes to after HUP.  */
 #define DEFAULT_LOGFILE "wget-log"
 
+#include <stdio.h>
+
 enum log_options { LOG_VERBOSE, LOG_NOTQUIET, LOG_NONVERBOSE, LOG_ALWAYS };
 
+void log_set_warc_log_fp (FILE *);
+
 void logprintf (enum log_options, const char *, ...)
      GCC_FORMAT_ATTR (2, 3);
 void debug_logprintf (const char *, ...) GCC_FORMAT_ATTR (1, 2);

src/main.c

@@ -1,6 +1,6 @@
 /* Command line parsing.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -55,7 +55,7 @@ as that of the covered work.  */
 #include "spider.h"
 #include "http.h"               /* for save_cookies */
 #include "ptimer.h"
-
+#include "warc.h"
 #include <getopt.h>
 #include <getpass.h>
 #include <quote.h>
@@ -157,6 +157,7 @@ struct cmdline_option {
 static struct cmdline_option option_data[] =
   {
     { "accept", 'A', OPT_VALUE, "accept", -1 },
+    { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 },
     { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 },
     { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument },
     { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 },
@@ -166,6 +167,8 @@ static struct cmdline_option option_data[] =
     { "backups", 0, OPT_BOOLEAN, "backups", -1 },
     { "base", 'B', OPT_VALUE, "base", -1 },
     { "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
+    { "body-data", 0, OPT_VALUE, "bodydata", -1 },
+    { "body-file", 0, OPT_VALUE, "bodyfile", -1 },
     { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 },
     { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 },
     { "cache", 0, OPT_BOOLEAN, "cache", -1 },
@@ -178,6 +181,7 @@ static struct cmdline_option option_data[] =
     { "continue", 'c', OPT_BOOLEAN, "continue", -1 },
     { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 },
     { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 },
+    { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 },
     { "cookies", 0, OPT_BOOLEAN, "cookies", -1 },
     { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 },
     { WHEN_DEBUG ("debug"), 'd', OPT_BOOLEAN, "debug", -1 },
@@ -213,6 +217,7 @@ static struct cmdline_option option_data[] =
     { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */
     { "http-password", 0, OPT_VALUE, "httppassword", -1 },
     { "http-user", 0, OPT_VALUE, "httpuser", -1 },
+    { IF_SSL ("https-only"), 0, OPT_BOOLEAN, "httpsonly", -1 },
     { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 },
     { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 },
     { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 },
@@ -229,6 +234,7 @@ static struct cmdline_option option_data[] =
     { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
     { "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
     { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
+    { "method", 0, OPT_VALUE, "method", -1 },
     { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
     { "no", 'n', OPT__NO, NULL, required_argument },
     { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
@@ -242,7 +248,7 @@ static struct cmdline_option option_data[] =
     { "post-data", 0, OPT_VALUE, "postdata", -1 },
     { "post-file", 0, OPT_VALUE, "postfile", -1 },
     { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 },
-    { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, /* deprecated */
+    { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 },
     { IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 },
     { IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 },
     { "progress", 0, OPT_VALUE, "progress", -1 },
@@ -259,10 +265,13 @@ static struct cmdline_option option_data[] =
     { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 },
     { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 },
     { "referer", 0, OPT_VALUE, "referer", -1 },
+    { "regex-type", 0, OPT_VALUE, "regextype", -1 },
     { "reject", 'R', OPT_VALUE, "reject", -1 },
+    { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 },
     { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 },
     { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 },
     { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 },
+    { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 },
     { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
     { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
     { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
@@ -286,6 +295,17 @@ static struct cmdline_option option_data[] =
     { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument },
     { "wait", 'w', OPT_VALUE, "wait", -1 },
     { "waitretry", 0, OPT_VALUE, "waitretry", -1 },
+    { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 },
+#ifdef HAVE_LIBZ
+    { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 },
+#endif
+    { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 },
+    { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 },
+    { "warc-file", 0, OPT_VALUE, "warcfile", -1 },
+    { "warc-header", 0, OPT_VALUE, "warcheader", -1 },
+    { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 },
+    { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 },
+    { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 },
 #ifdef USE_WATT32
     { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 },
 #endif
@@ -393,11 +413,11 @@ init_switches (void)
 }
 
 /* Print the usage message.  */
-static void
+static int
 print_usage (int error)
 {
-  fprintf (error ? stderr : stdout, _("Usage: %s [OPTION]... [URL]...\n"),
-           exec_name);
+  return fprintf (error ? stderr : stdout,
+                  _("Usage: %s [OPTION]... [URL]...\n"), exec_name);
 }
 
 /* Print the help message, describing all the available options.  If
@@ -443,6 +463,8 @@ Logging and input file:\n"),
   -v,  --verbose             be verbose (this is the default).\n"),
     N_("\
   -nv, --no-verbose          turn off verboseness, without being quiet.\n"),
+    N_("\
+       --report-speed=TYPE   Output bandwidth as TYPE.  TYPE can be bits.\n"),
     N_("\
   -i,  --input-file=FILE     download URLs found in local or external FILE.\n"),
     N_("\
@@ -591,9 +613,17 @@ HTTP options:\n"),
        --post-data=STRING      use the POST method; send STRING as the data.\n"),
     N_("\
        --post-file=FILE        use the POST method; send contents of FILE.\n"),
+    N_("\
+       --method=HTTPMethod     use method \"HTTPMethod\" in the header.\n"),
+    N_("\
+       --body-data=STRING      Send STRING as data. --method MUST be set.\n"),
+    N_("\
+       --body-file=FILE        Send contents of FILE. --method MUST be set.\n"),
     N_("\
        --content-disposition   honor the Content-Disposition header when\n\
                                choosing local file names (EXPERIMENTAL).\n"),
+    N_("\
+       --content-on-error      output the received content on server errors.\n"),
     N_("\
        --auth-no-challenge     send Basic HTTP authentication information\n\
                                without first waiting for the server's\n\
@@ -605,7 +635,9 @@ HTTP options:\n"),
 HTTPS (SSL/TLS) options:\n"),
     N_("\
        --secure-protocol=PR     choose secure protocol, one of auto, SSLv2,\n\
-                                SSLv3, and TLSv1.\n"),
+                                SSLv3, TLSv1 and PFS.\n"),
+    N_("\
+       --https-only             only follow secure HTTPS links\n"),
     N_("\
        --no-check-certificate   don't validate the server's certificate.\n"),
     N_("\
@@ -643,10 +675,37 @@ FTP options:\n"),
        --no-glob               turn off FTP file name globbing.\n"),
     N_("\
        --no-passive-ftp        disable the \"passive\" transfer mode.\n"),
+    N_("\
+       --preserve-permissions  preserve remote file permissions.\n"),
     N_("\
        --retr-symlinks         when recursing, get linked-to files (not dir).\n"),
     "\n",
 
+    N_("\
+WARC options:\n"),
+    N_("\
+       --warc-file=FILENAME      save request/response data to a .warc.gz file.\n"),
+    N_("\
+       --warc-header=STRING      insert STRING into the warcinfo record.\n"),
+    N_("\
+       --warc-max-size=NUMBER    set maximum size of WARC files to NUMBER.\n"),
+    N_("\
+       --warc-cdx                write CDX index files.\n"),
+    N_("\
+       --warc-dedup=FILENAME     do not store records listed in this CDX file.\n"),
+#ifdef HAVE_LIBZ
+    N_("\
+       --no-warc-compression     do not compress WARC files with GZIP.\n"),
+#endif
+    N_("\
+       --no-warc-digests         do not calculate SHA1 digests.\n"),
+    N_("\
+       --no-warc-keep-log        do not store the log file in a WARC record.\n"),
+    N_("\
+       --warc-tempdir=DIRECTORY  location for temporary files created by the\n\
+                                 WARC writer.\n"),
+    "\n",
+
     N_("\
 Recursive download:\n"),
     N_("\
@@ -658,6 +717,9 @@ Recursive download:\n"),
     N_("\
   -k,  --convert-links      make links in downloaded HTML or CSS point to\n\
                             local files.\n"),
+    N_("\
+  --backups=N   before writing file X, rotate up to N backup files.\n"),
+
 #ifdef __VMS
     N_("\
   -K,  --backup-converted   before converting file X, back up as X_orig.\n"),
@@ -679,6 +741,17 @@ Recursive accept/reject:\n"),
   -A,  --accept=LIST               comma-separated list of accepted extensions.\n"),
     N_("\
   -R,  --reject=LIST               comma-separated list of rejected extensions.\n"),
+    N_("\
+       --accept-regex=REGEX        regex matching accepted URLs.\n"),
+    N_("\
+       --reject-regex=REGEX        regex matching rejected URLs.\n"),
+#ifdef HAVE_LIBPCRE
+    N_("\
+       --regex-type=TYPE           regex type (posix|pcre).\n"),
+#else
+    N_("\
+       --regex-type=TYPE           regex type (posix).\n"),
+#endif
     N_("\
   -D,  --domains=LIST              comma-separated list of accepted domains.\n"),
     N_("\
@@ -703,18 +776,20 @@ Recursive accept/reject:\n"),
     N_("\
   -np, --no-parent                 don't ascend to the parent directory.\n"),
     "\n",
-
     N_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n")
   };
 
   size_t i;
 
-  printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
-          version_string);
-  print_usage (0);
+  if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"),
+              version_string) < 0)
+    exit (3);
+  if (print_usage (0) < 0)
+    exit (3);
 
   for (i = 0; i < countof (help); i++)
-    fputs (_(help[i]), stdout);
+    if (fputs (_(help[i]), stdout) < 0)
+      exit (3);
 
   exit (0);
 }
@@ -759,7 +834,7 @@ prompt_for_password (void)
    to at most line_length. prefix is printed on the first line
    and an appropriate number of spaces are added on subsequent
    lines.*/
-static void
+static int
 format_and_print_line (const char *prefix, const char *line,
                        int line_length)
 {
@@ -768,14 +843,16 @@ format_and_print_line (const char *prefix, const char *line,
 
   assert (prefix != NULL);
   assert (line != NULL);
+  assert (line_length > TABULATION);
 
   line_dup = xstrdup (line);
 
-  if (line_length <= 0)
-    line_length = MAX_CHARS_PER_LINE - TABULATION;
+  if (printf ("%s", prefix) < 0)
+    return -1;
+
+  /* Wrap to new line after prefix. */
+  remaining_chars = 0;
 
-  printf ("%s", prefix);
-  remaining_chars = line_length;
   /* We break on spaces. */
   token = strtok (line_dup, " ");
   while (token != NULL)
@@ -783,19 +860,23 @@ format_and_print_line (const char *prefix, const char *line,
       /* If however a token is much larger than the maximum
          line length, all bets are off and we simply print the
          token on the next line. */
-      if (remaining_chars <= strlen (token))
+      if (remaining_chars <= (int) strlen (token))
         {
-          printf ("\n%*c", TABULATION, ' ');
+          if (printf ("\n%*c", TABULATION, ' ') < 0)
+            return -1;
           remaining_chars = line_length - TABULATION;
         }
-      printf ("%s ", token);
+      if (printf ("%s ", token) < 0)
+        return -1;
       remaining_chars -= strlen (token) + 1;  /* account for " " */
       token = strtok (NULL, " ");
     }
 
-  printf ("\n");
+  if (printf ("\n") < 0)
+    return -1;
 
   xfree (line_dup);
+  return 0;
 }
 
 static void
@@ -808,76 +889,96 @@ print_version (void)
   char *env_wgetrc, *user_wgetrc;
   int i;
 
-  printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE);
+  if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0)
+    exit (3);
 
   for (i = 0; compiled_features[i] != NULL; )
     {
       int line_length = MAX_CHARS_PER_LINE;
       while ((line_length > 0) && (compiled_features[i] != NULL))
         {
-          printf ("%s ", compiled_features[i]);
+          if (printf ("%s ", compiled_features[i]) < 0)
+            exit (3);
           line_length -= strlen (compiled_features[i]) + 2;
           i++;
         }
-      printf ("\n");
+      if (printf ("\n") < 0)
+        exit (3);
     }
-  printf ("\n");
+  if (printf ("\n") < 0)
+    exit (3);
 
   /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is
      absent. */
-  printf ("%s\n", wgetrc_title);
+  if (printf ("%s\n", wgetrc_title) < 0)
+    exit (3);
+
   env_wgetrc = wgetrc_env_file_name ();
   if (env_wgetrc && *env_wgetrc)
     {
-      printf (_("    %s (env)\n"), env_wgetrc);
+      if (printf (_("    %s (env)\n"), env_wgetrc) < 0)
+        exit (3);
       xfree (env_wgetrc);
     }
   user_wgetrc = wgetrc_user_file_name ();
   if (user_wgetrc)
     {
-      printf (_("    %s (user)\n"), user_wgetrc);
+      if (printf (_("    %s (user)\n"), user_wgetrc) < 0)
+        exit (3);
       xfree (user_wgetrc);
     }
 #ifdef SYSTEM_WGETRC
-  printf (_("    %s (system)\n"), SYSTEM_WGETRC);
+  if (printf (_("    %s (system)\n"), SYSTEM_WGETRC) < 0)
+    exit (3);
 #endif
 
 #ifdef ENABLE_NLS
-  format_and_print_line (locale_title,
-                        LOCALEDIR,
-                        MAX_CHARS_PER_LINE);
+  if (format_and_print_line (locale_title,
+                             LOCALEDIR,
+                             MAX_CHARS_PER_LINE) < 0)
+    exit (3);
 #endif /* def ENABLE_NLS */
 
   if (compilation_string != NULL)
-    format_and_print_line (compile_title,
-                           compilation_string,
-                           MAX_CHARS_PER_LINE);
+    if (format_and_print_line (compile_title,
+                               compilation_string,
+                               MAX_CHARS_PER_LINE) < 0)
+      exit (3);
 
   if (link_string != NULL)
-    format_and_print_line (link_title,
-                           link_string,
-                           MAX_CHARS_PER_LINE);
+    if (format_and_print_line (link_title,
+                               link_string,
+                               MAX_CHARS_PER_LINE) < 0)
+      exit (3);
+
+  if (printf ("\n") < 0)
+    exit (3);
 
-  printf ("\n");
   /* TRANSLATORS: When available, an actual copyright character
-     (cirle-c) should be used in preference to "(C)". */
-  fputs (_("\
-Copyright (C) 2009 Free Software Foundation, Inc.\n"), stdout);
-  fputs (_("\
+     (circle-c) should be used in preference to "(C)". */
+  if (fputs (_("\
+Copyright (C) 2011 Free Software Foundation, Inc.\n"), stdout) < 0)
+    exit (3);
+  if (fputs (_("\
 License GPLv3+: GNU GPL version 3 or later\n\
 <http://www.gnu.org/licenses/gpl.html>.\n\
 This is free software: you are free to change and redistribute it.\n\
-There is NO WARRANTY, to the extent permitted by law.\n"), stdout);
+There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0)
+    exit (3);
   /* TRANSLATORS: When available, please use the proper diacritics for
      names such as this one. See en_US.po for reference. */
-  fputs (_("\nOriginally written by Hrvoje Niksic <hniksic@xemacs.org>.\n"),
-         stdout);
-  fputs (_("Please send bug reports and questions to <bug-wget@gnu.org>.\n"),
-         stdout);
+  if (fputs (_("\nOriginally written by Hrvoje Niksic <hniksic@xemacs.org>.\n"),
+             stdout) < 0)
+    exit (3);
+  if (fputs (_("Please send bug reports and questions to <bug-wget@gnu.org>.\n"),
+             stdout) < 0)
+    exit (3);
+
   exit (0);
 }
 
 char *program_name; /* Needed by lib/error.c. */
+char *program_argstring; /* Needed by wget_warc.c. */
 
 int
 main (int argc, char **argv)
@@ -913,13 +1014,34 @@ main (int argc, char **argv)
   windows_main ((char **) &exec_name);
 #endif
 
+  /* Construct the arguments string. */
+  int argstring_length = 1;
+  for (i = 1; i < argc; i++)
+    argstring_length += strlen (argv[i]) + 2 + 1;
+  char *p = program_argstring = malloc (argstring_length * sizeof (char));
+  if (p == NULL)
+    {
+      fprintf (stderr, _("Memory allocation problem\n"));
+      exit (2);
+    }
+  for (i = 1; i < argc; i++)
+    {
+      *p++ = '"';
+      int arglen = strlen (argv[i]);
+      memcpy (p, argv[i], arglen);
+      p += arglen;
+      *p++ = '"';
+      *p++ = ' ';
+    }
+  *p = '\0';
+
   /* Load the hard-coded defaults.  */
   defaults ();
 
   init_switches ();
 
-  /* This seperate getopt_long is needed to find the user config
-     and parse it before the other user options. */
+  /* This separate getopt_long is needed to find the user config file
+     option ("--config") and parse it before the other user options. */
   longindex = -1;
   int retconf;
   bool use_userconfig = false;
@@ -928,22 +1050,27 @@ main (int argc, char **argv)
                                 short_options, long_options, &longindex)) != -1)
     {
       int confval;
-      bool userrc_ret = true;
       struct cmdline_option *config_opt;
-      confval = long_options[longindex].val;
-      config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER];
-      if (strcmp (config_opt->long_name, "config") == 0)
+
+      /* There is no short option for "--config". */
+      if (longindex >= 0)
         {
-          userrc_ret &= run_wgetrc (optarg);
-          use_userconfig = true;
+          confval = long_options[longindex].val;
+          config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER];
+          if (strcmp (config_opt->long_name, "config") == 0)
+            {
+              bool userrc_ret = true;
+              userrc_ret &= run_wgetrc (optarg);
+              use_userconfig = true;
+              if (userrc_ret)
+                break;
+              else
+                {
+                  fprintf (stderr, _("Exiting due to error in %s\n"), optarg);
+                  exit (2);
+                }
+            }
         }
-      if (!userrc_ret)
-        {
-          printf ("Exiting due to error in %s\n", optarg);
-          exit (2);
-        }
-      else
-        break;
     }
 
   /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */
@@ -966,9 +1093,10 @@ main (int argc, char **argv)
         {
           if (ret == '?')
             {
-              print_usage (0);
-              printf ("\n");
-              printf (_("Try `%s --help' for more options.\n"), exec_name);
+              print_usage (1);
+              fprintf (stderr, "\n");
+              fprintf (stderr, _("Try `%s --help' for more options.\n"),
+		       exec_name);
               exit (2);
             }
           /* Find the short option character in the mapping.  */
@@ -1076,7 +1204,7 @@ main (int argc, char **argv)
     {
       fprintf (stderr,
                _("Both --no-clobber and --convert-links were specified,"
-                 "only --convert-links will be used.\n"));
+                 " only --convert-links will be used.\n"));
       opt.noclobber = false;
     }
 
@@ -1099,6 +1227,7 @@ main (int argc, char **argv)
   if (opt.verbose == -1)
     opt.verbose = !opt.quiet;
 
+
   /* Sanity checks.  */
   if (opt.verbose && opt.quiet)
     {
@@ -1157,6 +1286,47 @@ for details.\n\n"));
            }
     }
 
+  if (opt.warc_filename != 0)
+    {
+      if (opt.noclobber)
+        {
+          fprintf (stderr,
+                   _("WARC output does not work with --no-clobber, "
+                     "--no-clobber will be disabled.\n"));
+          opt.noclobber = false;
+        }
+      if (opt.timestamping)
+        {
+          fprintf (stderr,
+                   _("WARC output does not work with timestamping, "
+                     "timestamping will be disabled.\n"));
+          opt.timestamping = false;
+        }
+      if (opt.spider)
+        {
+          fprintf (stderr,
+                   _("WARC output does not work with --spider.\n"));
+          exit (1);
+        }
+      if (opt.always_rest)
+        {
+          fprintf (stderr,
+                   _("WARC output does not work with --continue, "
+                     "--continue will be disabled.\n"));
+          opt.always_rest = false;
+        }
+      if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled)
+        {
+          fprintf (stderr,
+                   _("Digests are disabled; WARC deduplication will "
+                     "not find duplicate records.\n"));
+        }
+      if (opt.warc_keep_log)
+        {
+          opt.progress_type = xstrdup ("dot");
+        }
+    }
+
   if (opt.ask_passwd && opt.passwd)
     {
       fprintf (stderr,
@@ -1170,13 +1340,100 @@ for details.\n\n"));
       /* No URL specified.  */
       fprintf (stderr, _("%s: missing URL\n"), exec_name);
       print_usage (1);
-      printf ("\n");
+      fprintf (stderr, "\n");
       /* #### Something nicer should be printed here -- similar to the
          pre-1.5 `--help' page.  */
       fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name);
       exit (1);
     }
 
+  /* Compile the regular expressions.  */
+  switch (opt.regex_type)
+    {
+#ifdef HAVE_LIBPCRE
+      case regex_type_pcre:
+        opt.regex_compile_fun = compile_pcre_regex;
+        opt.regex_match_fun = match_pcre_regex;
+        break;
+#endif
+
+      case regex_type_posix:
+      default:
+        opt.regex_compile_fun = compile_posix_regex;
+        opt.regex_match_fun = match_posix_regex;
+        break;
+    }
+  if (opt.acceptregex_s)
+    {
+      opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s);
+      if (!opt.acceptregex)
+        exit (1);
+    }
+  if (opt.rejectregex_s)
+    {
+      opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s);
+      if (!opt.rejectregex)
+        exit (1);
+    }
+  if (opt.post_data || opt.post_file_name)
+    {
+      if (opt.post_data && opt.post_file_name)
+        {
+          fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n"));
+          exit (1);
+        }
+      else if (opt.method)
+        {
+          fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
+                             "--method expects data through --body-data and --body-file options"));
+          exit (1);
+        }
+    }
+  if (opt.body_data || opt.body_file)
+    {
+      if (!opt.method)
+        {
+          fprintf (stderr, _("You must specify a method through --method=HTTPMethod "
+                              "to use with --body-data or --body-file.\n"));
+          exit (1);
+        }
+      else if (opt.body_data && opt.body_file)
+        {
+          fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n"));
+          exit (1);
+        }
+    }
+
+  /* Set various options as required for opt.method.  */
+
+  /* When user specifies HEAD as the method, we do not wish to download any
+     files. Hence, set wget to run in spider mode.  */
+  if (opt.method && strcasecmp (opt.method, "HEAD") == 0)
+    setoptval ("spider", "1", "spider");
+
+  /* Convert post_data to body-data and post_file_name to body-file options.
+     This is required so as to remove redundant code later on in gethttp().
+     The --post-data and --post-file options may also be removed in
+     the future hence it makes sense to convert them to aliases for
+     the more generic --method options.
+     This MUST occur only after the sanity checks so as to prevent the
+     user from setting both post and body options simultaneously.
+  */
+  if (opt.post_data || opt.post_file_name)
+    {
+        setoptval ("method", "POST", "method");
+        if (opt.post_data)
+          {
+            setoptval ("bodydata", opt.post_data, "body-data");
+            opt.post_data = NULL;
+          }
+        else
+          {
+            setoptval ("bodyfile", opt.post_file_name, "body-file");
+            opt.post_file_name = NULL;
+          }
+    }
+
 #ifdef ENABLE_IRI
   if (opt.enable_iri)
     {
@@ -1223,6 +1480,11 @@ for details.\n\n"));
 
   /* Fill in the arguments.  */
   url = alloca_array (char *, nurl + 1);
+  if (url == NULL)
+    {
+      fprintf (stderr, _("Memory allocation problem\n"));
+      exit (2);
+    }
   for (i = 0; i < nurl; i++, optind++)
     {
       char *rewritten = rewrite_shorthand_url (argv[optind]);
@@ -1236,6 +1498,10 @@ for details.\n\n"));
   /* Initialize logging.  */
   log_init (opt.lfilename, append_to_log);
 
+  /* Open WARC file. */
+  if (opt.warc_filename != 0)
+    warc_init ();
+
   DEBUGP (("DEBUG output created by Wget %s on %s.\n\n",
            version_string, OS_TYPE));
 
@@ -1368,7 +1634,7 @@ outputting to a regular file.\n"));
                           &dt, opt.recursive, iri, true);
           }
 
-          if (opt.delete_after && file_exists_p(filename))
+          if (opt.delete_after && filename != NULL && file_exists_p (filename))
             {
               DEBUGP (("Removing file due to --delete-after in main():\n"));
               logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
@@ -1435,12 +1701,9 @@ outputting to a regular file.\n"));
   if (opt.convert_links && !opt.delete_after)
     convert_all_links ();
 
-  log_close ();
-  for (i = 0; i < nurl; i++)
-    xfree (url[i]);
   cleanup ();
 
-  return get_exit_status ();
+  exit (get_exit_status ());
 }
 #endif /* TESTING */
 

src/mswindows.c

@@ -323,7 +323,7 @@ fork_to_background (void)
   rv = fake_fork_child ();
   if (rv < 0)
     {
-      fprintf (stderr, "fake_fork_child() failed\n");
+      fprintf (stderr, _("fake_fork_child() failed\n"));
       abort ();
     }
   else if (rv == 0)
@@ -331,7 +331,7 @@ fork_to_background (void)
       /* We're the parent.  */
       fake_fork ();
       /* If fake_fork() returns, it failed.  */
-      fprintf (stderr, "fake_fork() failed\n");
+      fprintf (stderr, _("fake_fork() failed\n"));
       abort ();
     }
   /* If we get here, we're the child.  */
@@ -646,7 +646,7 @@ set_windows_fd_as_blocking_socket (int fd)
 	if(ret)
     {
       fprintf (stderr,
-               "ioctl() failed.  The socket could not be set as blocking.\n" );
+               _("ioctl() failed.  The socket could not be set as blocking.\n") );
       DEBUGP (("Winsock error: %d\n", WSAGetLastError ()));
       abort ();
     }

src/mswindows.h

@@ -74,10 +74,6 @@ as that of the covered work.  */
 
 #include <stdio.h>
 
-/* The same for snprintf() and vsnprintf().  */
-#define snprintf _snprintf
-#define vsnprintf _vsnprintf
-
 /* Define a wgint type under Windows. */
 typedef __int64 wgint;
 #define SIZEOF_WGINT 8

src/netrc.c

@@ -164,48 +164,6 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 
 # define xrealloc realloc
 
-/* Read a line from FP.  The function reallocs the storage as needed
-   to accomodate for any length of the line.  Reallocs are done
-   storage exponentially, doubling the storage after each overflow to
-   minimize the number of calls to realloc() and fgets().  The newline
-   character at the end of line is retained.
-
-   After end-of-file is encountered without anything being read, NULL
-   is returned.  NULL is also returned on error.  To distinguish
-   between these two cases, use the stdio function ferror().  */
-
-char *
-read_whole_line (FILE *fp)
-{
-  int length = 0;
-  int bufsize = 81;
-  char *line = xmalloc (bufsize);
-
-  while (fgets (line + length, bufsize - length, fp))
-    {
-      length += strlen (line + length);
-      assert (length > 0);
-      if (line[length - 1] == '\n')
-        break;
-      /* fgets() guarantees to read the whole line, or to use up the
-         space we've given it.  We can double the buffer
-         unconditionally.  */
-      bufsize <<= 1;
-      line = xrealloc (line, bufsize);
-    }
-  if (length == 0 || ferror (fp))
-    {
-      xfree (line);
-      return NULL;
-    }
-  if (length + 1 < bufsize)
-    /* Relieve the memory from our exponential greediness.  We say
-       `length + 1' because the terminating \0 is not included in
-       LENGTH.  We don't need to zero-terminate the string ourselves,
-       though, because fgets() does that.  */
-    line = xrealloc (line, length + 1);
-  return line;
-}
 #endif /* STANDALONE */
 
 /* Maybe add NEWENTRY to the account information list, LIST.  NEWENTRY is
@@ -264,10 +222,11 @@ static acc_t *
 parse_netrc (const char *path)
 {
   FILE *fp;
-  char *line, *p, *tok;
+  char *line = NULL, *p, *tok;
   const char *premature_token;
   acc_t *current, *retval;
   int ln, qmark;
+  size_t bufsize = 0;
 
   /* The latest token we've seen in the file.  */
   enum
@@ -290,7 +249,7 @@ parse_netrc (const char *path)
   premature_token = NULL;
 
   /* While there are lines in the file...  */
-  while ((line = read_whole_line (fp)) != NULL)
+  while (getline (&line, &bufsize, fp) > 0)
     {
       ln ++;
 
@@ -423,10 +382,9 @@ parse_netrc (const char *path)
                          exec_name, path, ln, tok);
             }
         }
-
-      xfree (line);
     }
 
+  xfree (line);
   fclose (fp);
 
   /* Finalize the last machine entry we found.  */

src/openssl.c

@@ -1,6 +1,6 @@
 /* SSL support via OpenSSL library.
    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011 Free Software Foundation, Inc.
+   2009, 2010, 2011, 2012 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.
@@ -159,9 +159,9 @@ key_type_to_ssl_type (enum keyfile_type type)
    Returns true on success, false otherwise.  */
 
 bool
-ssl_init ()
+ssl_init (void)
 {
-  SSL_METHOD *meth;
+  SSL_METHOD const *meth;
 
   if (ssl_ctx)
     /* The SSL has already been initialized. */
@@ -194,6 +194,7 @@ ssl_init ()
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
+    case secure_protocol_pfs:
     case secure_protocol_tlsv1:
       meth = TLSv1_client_method ();
       break;
@@ -201,10 +202,18 @@ ssl_init ()
       abort ();
     }
 
-  ssl_ctx = SSL_CTX_new (meth);
+  /* The type cast below accommodates older OpenSSL versions (0.9.8)
+     where SSL_CTX_new() is declared without a "const" argument. */
+  ssl_ctx = SSL_CTX_new ((SSL_METHOD *)meth);
   if (!ssl_ctx)
     goto error;
 
+  /* OpenSSL ciphers: https://www.openssl.org/docs/apps/ciphers.html
+   * Since we want a good protection, we also use HIGH (that excludes MD4 ciphers and some more)
+   */
+  if (opt.secure_protocol == secure_protocol_pfs)
+    SSL_CTX_set_cipher_list (ssl_ctx, "HIGH:MEDIUM:!RC4:!SRP:!PSK:!RSA:!aNULL@STRENGTH");
+
   SSL_CTX_set_default_verify_paths (ssl_ctx);
   SSL_CTX_load_verify_locations (ssl_ctx, opt.ca_cert, opt.ca_directory);
 
@@ -249,24 +258,50 @@ ssl_init ()
   return false;
 }
 
-struct openssl_transport_context {
+struct openssl_transport_context
+{
   SSL *conn;                    /* SSL connection handle */
   char *last_error;             /* last error printed with openssl_errstr */
 };
 
+struct openssl_read_args
+{
+  int fd;
+  struct openssl_transport_context *ctx;
+  char *buf;
+  int bufsize;
+  int retval;
+};
+
+static void openssl_read_callback(void *arg)
+{
+  struct openssl_read_args *args = (struct openssl_read_args *) arg;
+  struct openssl_transport_context *ctx = args->ctx;
+  SSL *conn = ctx->conn;
+  char *buf = args->buf;
+  int bufsize = args->bufsize;
+  int ret;
+
+  do
+    ret = SSL_read (conn, buf, bufsize);
+  while (ret == -1 && SSL_get_error (conn, ret) == SSL_ERROR_SYSCALL
+         && errno == EINTR);
+  args->retval = ret;
+}
+
 static int
 openssl_read (int fd, char *buf, int bufsize, void *arg)
 {
-  int ret;
-  struct openssl_transport_context *ctx = arg;
-  SSL *conn = ctx->conn;
-  do
-    ret = SSL_read (conn, buf, bufsize);
-  while (ret == -1
-         && SSL_get_error (conn, ret) == SSL_ERROR_SYSCALL
-         && errno == EINTR);
+  struct openssl_read_args args;
+  args.fd = fd;
+  args.buf = buf;
+  args.bufsize = bufsize;
+  args.ctx = (struct openssl_transport_context*) arg;
 
-  return ret;
+  if (run_with_timeout(opt.read_timeout, openssl_read_callback, &args)) {
+    return -1;
+  }
+  return args.retval;
 }
 
 static int
@@ -384,6 +419,19 @@ static struct transport_implementation openssl_transport = {
   openssl_peek, openssl_errstr, openssl_close
 };
 
+struct scwt_context
+{
+  SSL *ssl;
+  int result;
+};
+
+static void
+ssl_connect_with_timeout_callback(void *arg)
+{
+  struct scwt_context *ctx = (struct scwt_context *)arg;
+  ctx->result = SSL_connect(ctx->ssl);
+}
+
 /* Perform the SSL handshake on file descriptor FD, which is assumed
    to be connected to an SSL server.  The SSL handle provided by
    OpenSSL is registered with the file descriptor FD using
@@ -393,9 +441,10 @@ static struct transport_implementation openssl_transport = {
    Returns true on success, false on failure.  */
 
 bool
-ssl_connect_wget (int fd)
+ssl_connect_wget (int fd, const char *hostname)
 {
   SSL *conn;
+  struct scwt_context scwt_ctx;
   struct openssl_transport_context *ctx;
 
   DEBUGP (("Initiating SSL handshake.\n"));
@@ -404,13 +453,33 @@ ssl_connect_wget (int fd)
   conn = SSL_new (ssl_ctx);
   if (!conn)
     goto error;
+#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
+  /* If the SSL library was build with support for ServerNameIndication
+     then use it whenever we have a hostname.  If not, don't, ever. */
+  if (! is_valid_ip_address (hostname))
+    {
+      if (! SSL_set_tlsext_host_name (conn, hostname))
+	{
+	DEBUGP (("Failed to set TLS server-name indication."));
+	goto error;
+	}
+    }
+#endif
+
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
 #endif
   if (!SSL_set_fd (conn, FD_TO_SOCKET (fd)))
     goto error;
   SSL_set_connect_state (conn);
-  if (SSL_connect (conn) <= 0 || conn->state != SSL_ST_OK)
+
+  scwt_ctx.ssl = conn;
+  if (run_with_timeout(opt.read_timeout, ssl_connect_with_timeout_callback,
+                       &scwt_ctx)) {
+    DEBUGP (("SSL handshake timed out.\n"));
+    goto timeout;
+  }
+  if (scwt_ctx.result <= 0 || conn->state != SSL_ST_OK)
     goto error;
 
   ctx = xnew0 (struct openssl_transport_context);
@@ -426,6 +495,7 @@ ssl_connect_wget (int fd)
  error:
   DEBUGP (("SSL handshake failed.\n"));
   print_errors ();
+ timeout:
   if (conn)
     SSL_free (conn);
   return false;

src/options.h

@@ -74,6 +74,19 @@ struct options
   bool ignore_case;		/* Whether to ignore case when
 				   matching dirs and files */
 
+  char *acceptregex_s;		/* Patterns to accept (a regex string). */
+  char *rejectregex_s;		/* Patterns to reject (a regex string). */
+  void *acceptregex;		/* Patterns to accept (a regex struct). */
+  void *rejectregex;		/* Patterns to reject (a regex struct). */
+  enum {
+#ifdef HAVE_LIBPCRE
+    regex_type_pcre,
+#endif
+    regex_type_posix
+  } regex_type;   /* The regex library. */
+  void *(*regex_compile_fun)(const char *);  /* Function to compile a regex. */
+  bool (*regex_match_fun)(const void *, const char *);  /* Function to match a string to a regex. */
+
   char **domains;		/* See host.c */
   char **exclude_domains;
   bool dns_cache;		/* whether we cache DNS lookups. */
@@ -87,6 +100,15 @@ struct options
 				   FTP. */
   char *output_document;	/* The output file to which the
 				   documents will be printed.  */
+  char *warc_filename;		/* WARC output filename */
+  char *warc_tempdir;	/* WARC temp dir */
+  char *warc_cdx_dedup_filename;	/* CDX file to be used for deduplication. */
+  wgint warc_maxsize;           /* WARC max archive size */
+  bool warc_compression_enabled;  /* For GZIP compression. */
+  bool warc_digests_enabled;  /* For SHA1 digests. */
+  bool warc_cdx_enabled;      /* Create CDX files? */
+  bool warc_keep_log;         /* Store the log file in a WARC record. */
+  char **warc_user_headers;		/* User-defined WARC header(s). */
 
   char *user;			/* Generic username */
   char *passwd;			/* Generic password */
@@ -130,6 +152,8 @@ struct options
   bool server_response;		/* Do we print server response? */
   bool save_headers;		/* Do we save headers together with
 				   file? */
+  bool content_on_error;	/* Do we output the content when the HTTP
+				   status code indicates a server error */
 
 #ifdef ENABLE_DEBUG
   bool debug;			/* Debugging on/off */
@@ -142,7 +166,7 @@ struct options
   bool timestamping;		/* Whether to use time-stamping. */
 
   bool backup_converted;	/* Do we save pre-converted files as *.orig? */
-  bool backups;			/* Are numeric backups made? */
+  int backups;			/* Are numeric backups made? */
 
   char *useragent;		/* User-Agent string, which can be set
 				   to something other than Wget. */
@@ -176,7 +200,8 @@ struct options
     secure_protocol_auto,
     secure_protocol_sslv2,
     secure_protocol_sslv3,
-    secure_protocol_tlsv1
+    secure_protocol_tlsv1,
+    secure_protocol_pfs
   } secure_protocol;		/* type of secure protocol to use. */
   bool check_cert;		/* whether to validate the server's cert */
   char *cert_file;		/* external client certificate to use. */
@@ -191,9 +216,9 @@ struct options
   char *ca_directory;		/* CA directory (hash files) */
   char *ca_cert;		/* CA certificate file to use */
 
-
   char *random_file;		/* file with random data to seed the PRNG */
   char *egd_file;		/* file name of the egd daemon socket */
+  bool https_only;		/* whether to follow HTTPS only */
 #endif /* HAVE_SSL */
 
   bool cookies;			/* whether cookies are used. */
@@ -204,6 +229,9 @@ struct options
 
   char *post_data;		/* POST query string */
   char *post_file_name;		/* File to post */
+  char *method;                 /* HTTP Method to use in Header */
+  char *body_data;              /* HTTP Method Data String */
+  char *body_file;              /* HTTP Method File */
 
   enum {
     restrict_unix,
@@ -255,6 +283,7 @@ struct options
 
   bool show_all_dns_entries; /* Show all the DNS entries when resolving a
                                 name. */
+  bool report_bps;              /*Output bandwidth in bits format*/
 };
 
 extern struct options opt;

src/progress.c

@@ -766,7 +766,7 @@ update_speed_ring (struct bar_progress *bp, wgint howmuch, double dltime)
 }
 
 #if USE_NLS_PROGRESS_BAR
-int
+static int
 count_cols (const char *mbs)
 {
   wchar_t wc;
@@ -795,7 +795,7 @@ count_cols (const char *mbs)
 # define count_cols(mbs) ((int)(strlen(mbs)))
 #endif
 
-const char *
+static const char *
 get_eta (int *bcd)
 {
   /* TRANSLATORS: "ETA" is English-centric, but this must
@@ -861,7 +861,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   struct bar_progress_hist *hist = &bp->hist;
 
   /* The progress bar should look like this:
-     xx% [=======>             ] nn,nnn 12.34K/s  eta 36m 51s
+     xx% [=======>             ] nn,nnn 12.34KB/s  eta 36m 51s
 
      Calculate the geometry.  The idea is to assign as much room as
      possible to the progress bar.  The other idea is to never let
@@ -873,7 +873,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      "xx% " or "100%"  - percentage               - 4 chars
      "[]"              - progress bar decorations - 2 chars
      " nnn,nnn,nnn"    - downloaded bytes         - 12 chars or very rarely more
-     " 12.5K/s"        - download rate             - 8 chars
+     " 12.5KB/s"        - download rate           - 9 chars
      "  eta 36m 51s"   - ETA                      - 14 chars
 
      "=====>..."       - progress bar             - the rest
@@ -977,10 +977,11 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       *p++ = ' ';
     }
 
-  /* " 12.52K/s" */
+  /* " 12.52Kb/s or 12.52KB/s" */
   if (hist->total_time > 0 && hist->total_bytes)
     {
-      static const char *short_units[] = { "B/s", "K/s", "M/s", "G/s" };
+      static const char *short_units[] = { "B/s", "KB/s", "MB/s", "GB/s" };
+      static const char *short_units_bits[] = { "b/s", "Kb/s", "Mb/s", "Gb/s" };
       int units = 0;
       /* Calculate the download speed using the history ring and
          recent data that hasn't made it to the ring yet.  */
@@ -988,7 +989,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       double dltime = hist->total_time + (dl_total_time - bp->recent_start);
       double dlspeed = calc_rate (dlquant, dltime, &units);
       sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
-               dlspeed, short_units[units]);
+               dlspeed,  !opt.report_bps ? short_units[units] : short_units_bits[units]);
       move_to_end (p);
     }
   else

src/ptimer.c

@@ -59,9 +59,7 @@ as that of the covered work.  */
 #include <errno.h>
 #include <unistd.h>
 #include <time.h>
-#ifdef HAVE_SYS_TIME_H
-# include <sys/time.h>
-#endif
+#include <sys/time.h>
 
 /* Cygwin currently (as of 2005-04-08, Cygwin 1.5.14) lacks clock_getres,
    but still defines _POSIX_TIMERS!  Because of that we simply use the

src/recur.c

@@ -1,6 +1,6 @@
 /* Handling of recursive HTTP retrieving.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -253,26 +253,23 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
          the second time.  */
       if (dl_url_file_map && hash_table_contains (dl_url_file_map, url))
         {
+	  bool is_css_bool;
+
           file = xstrdup (hash_table_get (dl_url_file_map, url));
 
           DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n",
                    url, file));
 
-          /* this sucks, needs to be combined! */
-          if (html_allowed
-              && downloaded_html_set
-              && string_set_contains (downloaded_html_set, file))
-            {
-              descend = true;
-              is_css = false;
-            }
-          if (css_allowed
-              && downloaded_css_set
-              && string_set_contains (downloaded_css_set, file))
-            {
-              descend = true;
-              is_css = true;
-            }
+	  if ((is_css_bool = (css_allowed
+			      && downloaded_css_set
+			      && string_set_contains (downloaded_css_set, file)))
+	      || (html_allowed
+		  && downloaded_html_set
+		  && string_set_contains (downloaded_html_set, file)))
+	    {
+	      descend = true;
+	      is_css = is_css_bool;
+	    }
         }
       else
         {
@@ -508,15 +505,16 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
     }
 
   /* Several things to check for:
-     1. if scheme is not http, and we don't load it
-     2. check for relative links (if relative_only is set)
-     3. check for domain
-     4. check for no-parent
-     5. check for excludes && includes
-     6. check for suffix
-     7. check for same host (if spanhost is unset), with possible
+     1. if scheme is not https and https_only requested
+     2. if scheme is not http, and we don't load it
+     3. check for relative links (if relative_only is set)
+     4. check for domain
+     5. check for no-parent
+     6. check for excludes && includes
+     7. check for suffix
+     8. check for same host (if spanhost is unset), with possible
      gethostbyname baggage
-     8. check for robots.txt
+     9. check for robots.txt
 
      Addendum: If the URL is FTP, and it is to be loaded, only the
      domain and suffix settings are "stronger".
@@ -528,6 +526,14 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
      More time- and memory- consuming tests should be put later on
      the list.  */
 
+#ifdef HAVE_SSL
+  if (opt.https_only && u->scheme != SCHEME_HTTPS)
+    {
+      DEBUGP (("Not following non-HTTPS links.\n"));
+      goto out;
+    }
+#endif
+
   /* Determine whether URL under consideration has a HTTP-like scheme. */
   u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
 
@@ -586,6 +592,11 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
           goto out;
         }
     }
+  if (!accept_url (url))
+    {
+      DEBUGP (("%s is excluded/not-included through regex.\n", url));
+      goto out;
+    }
 
   /* 6. Check for acceptance/rejection rules.  We ignore these rules
      for directories (no file name to match) and for non-leaf HTMLs,
@@ -636,7 +647,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
                  files after downloading or we're just running a spider. */
               if (opt.delete_after || opt.spider)
                 {
-                  logprintf (LOG_VERBOSE, "Removing %s.\n", rfile);
+                  logprintf (LOG_VERBOSE, _("Removing %s.\n"), rfile);
                   if (unlink (rfile))
                       logprintf (LOG_NOTQUIET, "unlink: %s\n",
                                  strerror (errno));

src/retr.c

@@ -37,6 +37,9 @@ as that of the covered work.  */
 #include <errno.h>
 #include <string.h>
 #include <assert.h>
+#ifdef VMS
+# include <unixio.h>            /* For delete(). */
+#endif
 
 #include "exits.h"
 #include "utils.h"
@@ -139,13 +142,16 @@ limit_bandwidth (wgint bytes, struct ptimer *timer)
 
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount
-   of data written.  */
+   of data written.  If OUT2 is not NULL, also write BUF to OUT2.
+   In case of error writing to OUT, -1 is returned.  In case of error
+   writing to OUT2, -2 is returned.  Return 1 if the whole BUF was
+   skipped.  */
 
 static int
-write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
-            wgint *written)
+write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
+            wgint *skip, wgint *written)
 {
-  if (!out)
+  if (out == NULL && out2 == NULL)
     return 1;
   if (*skip > bufsize)
     {
@@ -161,7 +167,10 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
         return 1;
     }
 
-  fwrite (buf, 1, bufsize, out);
+  if (out != NULL)
+    fwrite (buf, 1, bufsize, out);
+  if (out2 != NULL)
+    fwrite (buf, 1, bufsize, out2);
   *written += bufsize;
 
   /* Immediately flush the downloaded data.  This should not hinder
@@ -178,9 +187,17 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
      actual justification.  (Also, why 16K?  Anyone test other values?)
   */
 #ifndef __VMS
-  fflush (out);
+  if (out != NULL)
+    fflush (out);
+  if (out2 != NULL)
+    fflush (out2);
 #endif /* ndef __VMS */
-  return !ferror (out);
+  if (out != NULL && ferror (out))
+    return -1;
+  else if (out2 != NULL && ferror (out2))
+    return -2;
+  else
+    return 0;
 }
 
 /* Read the contents of file descriptor FD until it the connection
@@ -198,15 +215,23 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
    the amount of data written to disk.  The time it took to download
    the data is stored to ELAPSED.
 
+   If OUT2 is non-NULL, the contents is also written to OUT2.
+   OUT2 will get an exact copy of the response: if this is a chunked
+   response, everything -- including the chunk headers -- is written
+   to OUT2.  (OUT will only get the unchunked response.)
+
    The function exits and returns the amount of data read.  In case of
    error while reading data, -1 is returned.  In case of error while
-   writing data, -2 is returned.  */
+   writing data to OUT, -2 is returned.  In case of error while writing
+   data to OUT2, -3 is returned.  */
 
 int
 fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
-              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
+              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
+              FILE *out2)
 {
   int ret = 0;
+#undef max
 #define max(a,b) ((a) > (b) ? (a) : (b))
   int dlbufsize = max (BUFSIZ, 8 * 1024);
   char *dlbuf = xmalloc (dlbufsize);
@@ -286,13 +311,24 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
                   ret = -1;
                   break;
                 }
+              else if (out2 != NULL)
+                fwrite (line, 1, strlen (line), out2);
 
               remaining_chunk_size = strtol (line, &endl, 16);
+              xfree (line);
+
               if (remaining_chunk_size == 0)
                 {
                   ret = 0;
-                  if (fd_read_line (fd) == NULL)
+                  line = fd_read_line (fd);
+                  if (line == NULL)
                     ret = -1;
+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
                   break;
                 }
             }
@@ -342,20 +378,30 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
       if (ret > 0)
         {
           sum_read += ret;
-          if (!write_data (out, dlbuf, ret, &skip, &sum_written))
+          int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
+          if (write_res < 0)
             {
-              ret = -2;
+              ret = (write_res == -3) ? -3 : -2;
               goto out;
             }
           if (chunked)
             {
               remaining_chunk_size -= ret;
               if (remaining_chunk_size == 0)
-                if (fd_read_line (fd) == NULL)
-                  {
-                    ret = -1;
-                    break;
-                  }
+                {
+                  char *line = fd_read_line (fd);
+                  if (line == NULL)
+                    {
+                      ret = -1;
+                      break;
+                    }
+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
+                }
             }
         }
 
@@ -577,6 +623,7 @@ retr_rate (wgint bytes, double secs)
 {
   static char res[20];
   static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
+  static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" };
   int units;
 
   double dlrate = calc_rate (bytes, secs, &units);
@@ -584,7 +631,7 @@ retr_rate (wgint bytes, double secs)
      e.g. "1022", "247", "12.5", "2.38".  */
   sprintf (res, "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
-           dlrate, rate_names[units]);
+           dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);
 
   return res;
 }
@@ -601,6 +648,11 @@ double
 calc_rate (wgint bytes, double secs, int *units)
 {
   double dlrate;
+  double bibyte = 1000.0;
+ 
+  if (!opt.report_bps)
+    bibyte = 1024.0;
+
 
   assert (secs >= 0);
   assert (bytes >= 0);
@@ -612,35 +664,39 @@ calc_rate (wgint bytes, double secs, int *units)
        0 and the timer's resolution, assume half the resolution.  */
     secs = ptimer_resolution () / 2.0;
 
-  dlrate = bytes / secs;
-  if (dlrate < 1024.0)
+  dlrate = convert_to_bits (bytes) / secs;
+  if (dlrate < bibyte)
     *units = 0;
-  else if (dlrate < 1024.0 * 1024.0)
-    *units = 1, dlrate /= 1024.0;
-  else if (dlrate < 1024.0 * 1024.0 * 1024.0)
-    *units = 2, dlrate /= (1024.0 * 1024.0);
+  else if (dlrate < (bibyte * bibyte))
+    *units = 1, dlrate /= bibyte;
+  else if (dlrate < (bibyte * bibyte * bibyte))
+    *units = 2, dlrate /= (bibyte * bibyte);
+
   else
     /* Maybe someone will need this, one day. */
-    *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0);
+    *units = 3, dlrate /= (bibyte * bibyte * bibyte);
 
   return dlrate;
 }
 
 
-#define SUSPEND_POST_DATA do {                  \
-  post_data_suspended = true;                   \
-  saved_post_data = opt.post_data;              \
-  saved_post_file_name = opt.post_file_name;    \
-  opt.post_data = NULL;                         \
-  opt.post_file_name = NULL;                    \
+#define SUSPEND_METHOD do {                     \
+  method_suspended = true;                      \
+  saved_body_data = opt.body_data;              \
+  saved_body_file_name = opt.body_file;         \
+  saved_method = opt.method;                    \
+  opt.body_data = NULL;                         \
+  opt.body_file = NULL;                         \
+  opt.method = NULL;                            \
 } while (0)
 
-#define RESTORE_POST_DATA do {                          \
-  if (post_data_suspended)                              \
+#define RESTORE_METHOD do {                             \
+  if (method_suspended)                                 \
     {                                                   \
-      opt.post_data = saved_post_data;                  \
-      opt.post_file_name = saved_post_file_name;        \
-      post_data_suspended = false;                      \
+      opt.body_data = saved_body_data;                  \
+      opt.body_file = saved_body_file_name;             \
+      opt.method = saved_method;                        \
+      method_suspended = false;                         \
     }                                                   \
 } while (0)
 
@@ -668,9 +724,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
   char *local_file;
   int redirection_count = 0;
 
-  bool post_data_suspended = false;
-  char *saved_post_data = NULL;
-  char *saved_post_file_name = NULL;
+  bool method_suspended = false;
+  char *saved_body_data = NULL;
+  char *saved_method = NULL;
+  char *saved_body_file_name = NULL;
 
   /* If dt is NULL, use local storage.  */
   if (!dt)
@@ -711,7 +768,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
                      proxy, error);
           xfree (url);
           xfree (error);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           result = PROXERR;
           goto bail;
         }
@@ -720,7 +777,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
           url_free (proxy_url);
           xfree (url);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           result = PROXERR;
           goto bail;
         }
@@ -788,6 +845,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       iri->utf8_encode = opt.enable_iri;
       set_content_encoding (iri, NULL);
       xfree_null (iri->orig_url);
+      iri->orig_url = NULL;
 
       /* Now, see if this new location makes sense. */
       newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
@@ -803,7 +861,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           xfree (url);
           xfree (mynewloc);
           xfree (error);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           goto bail;
         }
 
@@ -825,7 +883,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
             }
           xfree (url);
           xfree (mynewloc);
-          RESTORE_POST_DATA;
+          RESTORE_METHOD;
           result = WRONGCODE;
           goto bail;
         }
@@ -848,8 +906,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 	 RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
 	 specifically to preserve the method of the request.
 	 */
-      if (result != NEWLOCATION_KEEP_POST && !post_data_suspended)
-        SUSPEND_POST_DATA;
+      if (result != NEWLOCATION_KEEP_POST && !method_suspended)
+        SUSPEND_METHOD;
 
       goto redirected;
     }
@@ -882,10 +940,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
         register_redirection (origurl, u->url);
 
       if (*dt & TEXTHTML)
-        register_html (u->url, local_file);
+        register_html (local_file);
 
       if (*dt & TEXTCSS)
-        register_css (u->url, local_file);
+        register_css (local_file);
     }
 
   if (file)
@@ -912,7 +970,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       xfree (url);
     }
 
-  RESTORE_POST_DATA;
+  RESTORE_METHOD;
 
 bail:
   if (register_status)
@@ -1127,7 +1185,16 @@ free_urlpos (struct urlpos *l)
 void
 rotate_backups(const char *fname)
 {
-  int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
+#ifdef __VMS
+# define SEP "_"
+# define AVS ";*"                       /* All-version suffix. */
+# define AVSL (sizeof (AVS) - 1)
+#else
+# define SEP "."
+# define AVSL 0
+#endif
+
+  int maxlen = strlen (fname) + sizeof (SEP) + numdigit (opt.backups) + AVSL;
   char *from = (char *)alloca (maxlen);
   char *to = (char *)alloca (maxlen);
   struct_stat sb;
@@ -1139,12 +1206,24 @@ rotate_backups(const char *fname)
 
   for (i = opt.backups; i > 1; i--)
     {
-      sprintf (from, "%s.%d", fname, i - 1);
-      sprintf (to, "%s.%d", fname, i);
+#ifdef VMS
+      /* Delete (all versions of) any existing max-suffix file, to avoid
+       * creating multiple versions of it.  (On VMS, rename() will
+       * create a new version of an existing destination file, not
+       * destroy/overwrite it.)
+       */
+      if (i == opt.backups)
+        {
+          sprintf (to, "%s%s%d%s", fname, SEP, i, AVS);
+          delete (to);
+        }
+#endif
+      sprintf (to, "%s%s%d", fname, SEP, i);
+      sprintf (from, "%s%s%d", fname, SEP, i - 1);
       rename (from, to);
     }
 
-  sprintf (to, "%s.%d", fname, 1);
+  sprintf (to, "%s%s%d", fname, SEP, 1);
   rename(fname, to);
 }
 

src/retr.h

@@ -50,7 +50,7 @@ enum {
   rb_chunked_transfer_encoding = 4
 };
 
-int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int);
+int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int, FILE *);
 
 typedef const char *(*hunk_terminator_t) (const char *, const char *, int);
 

src/ssl.h

@@ -1,6 +1,6 @@
 /* SSL support.
    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011 Free Software Foundation, Inc.
+   2009, 2010, 2011, 2012 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.
@@ -33,7 +33,7 @@ as that of the covered work.  */
 #define GEN_SSLFUNC_H
 
 bool ssl_init (void);
-bool ssl_connect_wget (int);
+bool ssl_connect_wget (int, const char *);
 bool ssl_check_certificate (int, const char *);
 
 #endif /* GEN_SSLFUNC_H */

src/test.c

@@ -28,6 +28,8 @@ Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
+#include "wget.h"
+
 #include <stdio.h>
 
 #include "test.h"
@@ -46,6 +48,8 @@ const char *test_append_uri_pathel();
 const char *test_are_urls_equal();
 const char *test_is_robots_txt_url();
 
+const char *program_argstring = "TEST";
+
 int tests_run;
 
 static const char *

src/url.c

@@ -701,7 +701,10 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       if (!iri->utf8_encode)
         new_url = NULL;
       else
-        iri->orig_url = xstrdup (url);
+        {
+          iri->orig_url = xstrdup (url);
+          percent_encode = true;
+        }
     }
 
   /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
@@ -1245,12 +1248,16 @@ mkalldirs (const char *path)
    The idea is to have a convenient and efficient way to construct a
    string by having various functions append data to it.  Instead of
    passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the
-   functions in questions, we pass the pointer to this struct.  */
+   functions in questions, we pass the pointer to this struct.
+
+   Functions that write to the members in this struct must make sure
+   that base remains null terminated by calling append_null().
+   */
 
 struct growable {
   char *base;
-  int size;
-  int tail;
+  int size;   /* memory allocated */
+  int tail;   /* string length */
 };
 
 /* Ensure that the string can accept APPEND_COUNT more characters past
@@ -1268,28 +1275,55 @@ struct growable {
 /* Move the tail position by APPEND_COUNT characters. */
 #define TAIL_INCR(r, append_count) ((r)->tail += append_count)
 
-/* Append the string STR to DEST.  NOTICE: the string in DEST is not
-   terminated.  */
 
+/* Append NULL to DEST. */
+static void
+append_null (struct growable *dest)
+{
+  GROW (dest, 1);
+  *TAIL (dest) = 0;
+}
+
+/* Shorten DEST to LENGTH. */
+static void
+shorten_length (size_t length, struct growable *dest)
+{
+  if (length < dest->tail)
+    dest->tail = length;
+
+  append_null (dest);
+}
+
+/* Append CH to DEST. */
+static void
+append_char (char ch, struct growable *dest)
+{
+  if (ch)
+    {
+      GROW (dest, 1);
+      *TAIL (dest) = ch;
+      TAIL_INCR (dest, 1);
+    }
+
+  append_null (dest);
+}
+
+/* Append the string STR to DEST. */
 static void
 append_string (const char *str, struct growable *dest)
 {
   int l = strlen (str);
-  GROW (dest, l);
-  memcpy (TAIL (dest), str, l);
-  TAIL_INCR (dest, l);
+
+  if (l)
+    {
+      GROW (dest, l);
+      memcpy (TAIL (dest), str, l);
+      TAIL_INCR (dest, l);
+    }
+
+  append_null (dest);
 }
 
-/* Append CH to DEST.  For example, append_char (0, DEST)
-   zero-terminates DEST.  */
-
-static void
-append_char (char ch, struct growable *dest)
-{
-  GROW (dest, 1);
-  *TAIL (dest) = ch;
-  TAIL_INCR (dest, 1);
-}
 
 enum {
   filechr_not_unix    = 1,      /* unusable on Unix, / and \0 */
@@ -1361,6 +1395,7 @@ UWC,  C,  C,  C,   C,  C,  C,  C,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
    query, normally '?'.  Since Windows cannot handle '?' as part of
    file name, we use '@' instead there.  */
 #define FN_QUERY_SEP (opt.restrict_files_os != restrict_windows ? '?' : '@')
+#define FN_QUERY_SEP_STR (opt.restrict_files_os != restrict_windows ? "?" : "@")
 
 /* Quote path element, characters in [b, e), as file name, and append
    the quoted string to DEST.  Each character is quoted as per
@@ -1454,6 +1489,7 @@ append_uri_pathel (const char *b, const char *e, bool escaped,
     }
 
   TAIL_INCR (dest, outlen);
+  append_null (dest);
 }
 
 /* Append to DEST the directory structure that corresponds the
@@ -1494,22 +1530,28 @@ append_dir_structure (const struct url *u, struct growable *dest)
     }
 }
 
-/* Return a unique file name that matches the given URL as good as
+/* Return a unique file name that matches the given URL as well as
    possible.  Does not create directories on the file system.  */
 
 char *
 url_file_name (const struct url *u, char *replaced_filename)
 {
   struct growable fnres;        /* stands for "file name result" */
+  struct growable temp_fnres;
 
-  const char *u_file, *u_query;
-  char *fname, *unique;
-  char *index_filename = "index.html"; /* The default index file is index.html */
+  const char *u_file;
+  char *fname, *unique, *fname_len_check;
+  const char *index_filename = "index.html"; /* The default index file is index.html */
+  size_t max_length;
 
   fnres.base = NULL;
   fnres.size = 0;
   fnres.tail = 0;
 
+  temp_fnres.base = NULL;
+  temp_fnres.size = 0;
+  temp_fnres.tail = 0;
+
   /* If an alternative index file was defined, change index_filename */
   if (opt.default_page)
     index_filename = opt.default_page;
@@ -1555,44 +1597,87 @@ url_file_name (const struct url *u, char *replaced_filename)
 
   if (!replaced_filename)
     {
-      /* Add the file name. */
-      if (fnres.tail)
-	append_char ('/', &fnres);
+      /* Create the filename. */
       u_file = *u->file ? u->file : index_filename;
-      append_uri_pathel (u_file, u_file + strlen (u_file), false, &fnres);
 
-      /* Append "?query" to the file name. */
-      u_query = u->query && *u->query ? u->query : NULL;
-      if (u_query)
-	{
-	  append_char (FN_QUERY_SEP, &fnres);
-	  append_uri_pathel (u_query, u_query + strlen (u_query),
-			     true, &fnres);
-	}
+      /* Append "?query" to the file name, even if empty,
+       * and create fname_len_check. */
+      if (u->query)
+        fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);
+      else
+        fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
     }
   else
     {
-      if (fnres.tail)
-	append_char ('/', &fnres);
       u_file = replaced_filename;
-      append_uri_pathel (u_file, u_file + strlen (u_file), false, &fnres);
+      fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
     }
 
-  /* Zero-terminate the file name. */
-  append_char ('\0', &fnres);
+  append_uri_pathel (fname_len_check,
+    fname_len_check + strlen (fname_len_check), false, &temp_fnres);
+
+  /* Zero-terminate the temporary file name. */
+  append_char ('\0', &temp_fnres);
+
+  /* Check that the length of the file name is acceptable. */
+#ifdef WINDOWS
+  if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
+    {
+      max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2);
+      /* FIXME: In Windows a filename is usually limited to 255 characters.
+      To really be accurate you could call GetVolumeInformation() to get
+      lpMaximumComponentLength
+      */
+      if (max_length > 255)
+        {
+          max_length = 255;
+        }
+    }
+  else
+    {
+      max_length = 0;
+    }
+#else
+  max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER;
+#endif
+  if (max_length > 0 && strlen (temp_fnres.base) > max_length)
+    {
+      logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n",
+          (unsigned long) strlen (temp_fnres.base));
+      logprintf (LOG_NOTQUIET, "Trying to shorten...\n");
+
+      /* Shorten the file name. */
+      temp_fnres.base[max_length] = '\0';
+
+      logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
+    }
+
+  free (fname_len_check);
+
+  /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
+   * just append it. */
+  if (fnres.tail)
+    append_char ('/', &fnres);
+  append_string (temp_fnres.base, &fnres);
 
   fname = fnres.base;
 
+  /* Make a final check that the path length is acceptable? */
+  /* TODO: check fnres.base for path length problem */
+
+  free (temp_fnres.base);
+
   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
      2) Retrieval with regetting.
      3) Timestamping is used.
      4) Hierarchy is built.
+     5) Backups are specified.
 
      The exception is the case when file does exist and is a
      directory (see `mkalldirs' for explanation).  */
 
-  if ((opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct)
+  if (ALLOW_CLOBBER
       && !(file_exists_p (fname) && !file_non_directory_p (fname)))
     {
       unique = fname;
@@ -2229,7 +2314,6 @@ test_append_uri_pathel()
 
       append_string (test_array[i].original_url, &dest);
       append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);
-      append_char ('\0', &dest);
 
       mu_assert ("test_append_uri_pathel: wrong result",
                  strcmp (dest.base, test_array[i].expected_result) == 0);

src/url.h

@@ -37,6 +37,22 @@ as that of the covered work.  */
 #define DEFAULT_FTP_PORT 21
 #define DEFAULT_HTTPS_PORT 443
 
+/* This represents how many characters less than the OS max name length a file
+ * should be.  More precisely, a file name should be at most
+ * (NAME_MAX - CHOMP_BUFFER) characters in length.  This number was arrived at
+ * by adding the lengths of all possible strings that could be appended to a
+ * file name later in the code (e.g. ".orig", ".html", etc.).  This is
+ * hopefully plenty of extra characters, but I am not guaranteeing that a file
+ * name will be of the proper length by the time the code wants to open a
+ * file descriptor. */
+#define CHOMP_BUFFER 19
+
+/* The flags that allow clobbering the file (opening with "wb").
+   Defined here to avoid repetition later.  #### This will require
+   rework.  */
+#define ALLOW_CLOBBER (opt.noclobber || opt.always_rest || opt.timestamping \
+                  || opt.dirstruct || opt.output_document || opt.backups > 0)
+
 /* Specifies how, or whether, user auth information should be included
  * in URLs regenerated from URL parse structures. */
 enum url_auth_mode {

src/utils.c

@@ -42,21 +42,29 @@ as that of the covered work.  */
 #ifdef HAVE_PROCESS_H
 # include <process.h>  /* getpid() */
 #endif
-#ifdef HAVE_UTIME_H
-# include <utime.h>
-#endif
 #include <errno.h>
 #include <fcntl.h>
 #include <assert.h>
 #include <stdarg.h>
 #include <locale.h>
 
+#if HAVE_UTIME
+# include <sys/types.h>
+# ifdef HAVE_UTIME_H
+#  include <utime.h>
+# endif
+
+# ifdef HAVE_SYS_UTIME_H
+#  include <sys/utime.h>
+# endif
+#endif
+
+#include <sys/time.h>
+
 #include <sys/stat.h>
 
 /* For TIOCGWINSZ and friends: */
-#ifdef HAVE_SYS_IOCTL_H
-# include <sys/ioctl.h>
-#endif
+#include <sys/ioctl.h>
 #ifdef HAVE_TERMIOS_H
 # include <termios.h>
 #endif
@@ -65,6 +73,11 @@ as that of the covered work.  */
 #include <signal.h>
 #include <setjmp.h>
 
+#include <regex.h>
+#ifdef HAVE_LIBPCRE
+# include <pcre.h>
+#endif
+
 #ifndef HAVE_SIGSETJMP
 /* If sigsetjmp is a macro, configure won't pick it up. */
 # ifdef sigsetjmp
@@ -487,6 +500,20 @@ fork_to_background (void)
 void
 touch (const char *file, time_t tm)
 {
+#if HAVE_UTIME
+# ifdef HAVE_STRUCT_UTIMBUF
+  struct utimbuf times;
+# else
+  struct {
+    time_t actime;
+    time_t modtime;
+  } times;
+# endif
+  times.modtime = tm;
+  times.actime = time (NULL);
+  if (utime (file, &times) == -1)
+    logprintf (LOG_NOTQUIET, "utime(%s): %s\n", file, strerror (errno));
+#else
   struct timespec timespecs[2];
   int fd;
 
@@ -506,6 +533,7 @@ touch (const char *file, time_t tm)
     logprintf (LOG_NOTQUIET, "futimens(%s): %s\n", file, strerror (errno));
 
   close (fd);
+#endif
 }
 
 /* Checks if FILE is a symbolic link, and removes it if it is.  Does
@@ -675,7 +703,7 @@ unique_create (const char *name, bool binary, char **opened_name)
       xfree (uname);
       uname = unique_name (name, false);
     }
-  if (opened_name && fp != NULL)
+  if (opened_name)
     {
       if (fp)
         *opened_name = uname;
@@ -746,8 +774,7 @@ fopen_excl (const char *fname, int binary)
       open_id = 13;
       fd = open( fname,                 /* File name. */
        flags,                           /* Flags. */
-       0777,                            /* Mode for default protection.
-*/
+       0777,                            /* Mode for default protection. */
        "rfm=stmlf",                     /* Stream_LF. */
        OPEN_OPT_ARGS);                  /* Access callback. */
     }
@@ -873,15 +900,14 @@ static bool in_acclist (const char *const *, const char *, bool);
 bool
 acceptable (const char *s)
 {
-  int l = strlen (s);
+  const char *p;
 
   if (opt.output_document && strcmp (s, opt.output_document) == 0)
     return true;
 
-  while (l && s[l] != '/')
-    --l;
-  if (s[l] == '/')
-    s += (l + 1);
+  if ((p = strrchr (s, '/')))
+    s = p + 1;
+
   if (opt.accepts)
     {
       if (opt.rejects)
@@ -892,6 +918,20 @@ acceptable (const char *s)
     }
   else if (opt.rejects)
     return !in_acclist ((const char *const *)opt.rejects, s, true);
+
+  return true;
+}
+
+/* Determine whether an URL is acceptable to be followed, according to
+   regex patterns to accept/reject.  */
+bool
+accept_url (const char *s)
+{
+  if (opt.acceptregex && !opt.regex_match_fun (opt.acceptregex, s))
+    return false;
+  if (opt.rejectregex && opt.regex_match_fun (opt.rejectregex, s))
+    return false;
+
   return true;
 }
 
@@ -978,29 +1018,15 @@ accdir (const char *directory)
 bool
 match_tail (const char *string, const char *tail, bool fold_case)
 {
-  int i, j;
+  int pos = strlen (string) - strlen (tail);
 
-  /* We want this to be fast, so we code two loops, one with
-     case-folding, one without. */
+  if (pos < 0)
+    return false;  /* tail is longer than string.  */
 
   if (!fold_case)
-    {
-      for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--)
-        if (string[i] != tail[j])
-          break;
-    }
+    return !strcmp (string + pos, tail);
   else
-    {
-      for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--)
-        if (c_tolower (string[i]) != c_tolower (tail[j]))
-          break;
-    }
-
-  /* If the tail was exhausted, the match was succesful.  */
-  if (j == -1)
-    return true;
-  else
-    return false;
+    return !strcasecmp (string + pos, tail);
 }
 
 /* Checks whether string S matches each element of ACCEPTS.  A list
@@ -1049,15 +1075,12 @@ in_acclist (const char *const *accepts, const char *s, bool backward)
 char *
 suffix (const char *str)
 {
-  int i;
+  char *p;
 
-  for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--)
-    ;
+  if ((p = strrchr (str, '.')) && !strchr (p + 1, '/'))
+    return p + 1;
 
-  if (str[i++] == '.')
-    return (char *)str + i;
-  else
-    return NULL;
+  return NULL;
 }
 
 /* Return true if S contains globbing wildcards (`*', `?', `[' or
@@ -1066,10 +1089,7 @@ suffix (const char *str)
 bool
 has_wildcards_p (const char *s)
 {
-  for (; *s; s++)
-    if (*s == '*' || *s == '?' || *s == '[' || *s == ']')
-      return true;
-  return false;
+  return !!strpbrk (s, "*?[]");
 }
 
 /* Return true if FNAME ends with a typical HTML suffix.  The
@@ -1098,56 +1118,6 @@ has_html_suffix_p (const char *fname)
   return false;
 }
 
-/* Read a line from FP and return the pointer to freshly allocated
-   storage.  The storage space is obtained through malloc() and should
-   be freed with free() when it is no longer needed.
-
-   The length of the line is not limited, except by available memory.
-   The newline character at the end of line is retained.  The line is
-   terminated with a zero character.
-
-   After end-of-file is encountered without anything being read, NULL
-   is returned.  NULL is also returned on error.  To distinguish
-   between these two cases, use the stdio function ferror().  */
-
-char *
-read_whole_line (FILE *fp)
-{
-  int length = 0;
-  int bufsize = 82;
-  char *line = xmalloc (bufsize);
-
-  while (fgets (line + length, bufsize - length, fp))
-    {
-      length += strlen (line + length);
-      if (length == 0)
-        /* Possible for example when reading from a binary file where
-           a line begins with \0.  */
-        continue;
-
-      if (line[length - 1] == '\n')
-        break;
-
-      /* fgets() guarantees to read the whole line, or to use up the
-         space we've given it.  We can double the buffer
-         unconditionally.  */
-      bufsize <<= 1;
-      line = xrealloc (line, bufsize);
-    }
-  if (length == 0 || ferror (fp))
-    {
-      xfree (line);
-      return NULL;
-    }
-  if (length + 1 < bufsize)
-    /* Relieve the memory from our exponential greediness.  We say
-       `length + 1' because the terminating \0 is not included in
-       LENGTH.  We don't need to zero-terminate the string ourselves,
-       though, because fgets() does that.  */
-    line = xrealloc (line, length + 1);
-  return line;
-}
-
 /* Read FILE into memory.  A pointer to `struct file_memory' are
    returned; use struct element `content' to access file contents, and
    the element `length' to know the file length.  `content' is *not*
@@ -1803,6 +1773,17 @@ number_to_static_string (wgint number)
   ringpos = (ringpos + 1) % RING_SIZE;
   return buf;
 }
+
+/* Converts the byte to bits format if --report-bps option is enabled
+ */
+wgint
+convert_to_bits (wgint num)
+{
+  if (opt.report_bps)
+    return num * 8;
+  return num;
+}
+
 
 /* Determine the width of the terminal we're running on.  If that's
    not possible, return 0.  */
@@ -1930,9 +1911,10 @@ abort_run_with_timeout (int sig)
   /* We don't have siglongjmp to preserve the set of blocked signals;
      if we longjumped out of the handler at this point, SIGALRM would
      remain blocked.  We must unblock it manually. */
-  int mask = siggetmask ();
-  mask &= ~sigmask (SIGALRM);
-  sigsetmask (mask);
+  sigset_t set;
+  sigemptyset (&set);
+  sigaddset (&set, SIGALRM);
+  sigprocmask (SIG_BLOCK, &set, NULL);
 
   /* Now it's safe to longjump. */
   longjmp (run_with_timeout_env, -1);
@@ -2275,6 +2257,89 @@ base64_decode (const char *base64, void *dest)
   return q - (char *) dest;
 }
 
+#ifdef HAVE_LIBPCRE
+/* Compiles the PCRE regex. */
+void *
+compile_pcre_regex (const char *str)
+{
+  const char *errbuf;
+  int erroffset;
+  pcre *regex = pcre_compile (str, 0, &errbuf, &erroffset, 0);
+  if (! regex)
+    {
+      fprintf (stderr, _("Invalid regular expression %s, %s\n"),
+               quote (str), errbuf);
+      return false;
+    }
+  return regex;
+}
+#endif
+
+/* Compiles the POSIX regex. */
+void *
+compile_posix_regex (const char *str)
+{
+  regex_t *regex = xmalloc (sizeof (regex_t));
+  int errcode = regcomp ((regex_t *) regex, str, REG_EXTENDED | REG_NOSUB);
+  if (errcode != 0)
+    {
+      int errbuf_size = regerror (errcode, (regex_t *) regex, NULL, 0);
+      char *errbuf = xmalloc (errbuf_size);
+      regerror (errcode, (regex_t *) regex, errbuf, errbuf_size);
+      fprintf (stderr, _("Invalid regular expression %s, %s\n"),
+               quote (str), errbuf);
+      xfree (errbuf);
+      return NULL;
+    }
+
+  return regex;
+}
+
+#ifdef HAVE_LIBPCRE
+#define OVECCOUNT 30
+/* Matches a PCRE regex.  */
+bool
+match_pcre_regex (const void *regex, const char *str)
+{
+  int l = strlen (str);
+  int ovector[OVECCOUNT];
+
+  int rc = pcre_exec ((pcre *) regex, 0, str, l, 0, 0, ovector, OVECCOUNT);
+  if (rc == PCRE_ERROR_NOMATCH)
+    return false;
+  else if (rc < 0)
+    {
+      logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"),
+                 quote (str), rc);
+      return false;
+    }
+  else
+    return true;
+}
+#undef OVECCOUNT
+#endif
+
+/* Matches a POSIX regex.  */
+bool
+match_posix_regex (const void *regex, const char *str)
+{
+  int rc = regexec ((regex_t *) regex, str, 0, NULL, 0);
+  if (rc == REG_NOMATCH)
+    return false;
+  else if (rc == 0)
+    return true;
+  else
+    {
+      int errbuf_size = regerror (rc, opt.acceptregex, NULL, 0);
+      char *errbuf = xmalloc (errbuf_size);
+      regerror (rc, opt.acceptregex, errbuf, errbuf_size);
+      logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"),
+                 quote (str), rc);
+      xfree (errbuf);
+      return false;
+    }
+}
+
 #undef IS_ASCII
 #undef NEXT_CHAR
 
@@ -2359,21 +2424,75 @@ print_decimal (double number)
   return buf;
 }
 
+/* Get the maximum name length for the given path. */
+/* Return 0 if length is unknown. */
+size_t
+get_max_length (const char *path, int length, int name)
+{
+  long ret;
+  char *p, *d;
+
+  /* Make a copy of the path that we can modify. */
+  p = path ? strdupdelim (path, path + length) : strdup ("");
+
+  for (;;)
+    {
+      errno = 0;
+      /* For an empty path query the current directory. */
+#if HAVE_PATHCONF
+      ret = pathconf (*p ? p : ".", name);
+      if (!(ret < 0 && errno == ENOENT))
+        break;
+#else
+      ret = PATH_MAX;
+#endif
+
+      /* The path does not exist yet, but may be created. */
+      /* Already at current or root directory, give up. */
+      if (!*p || strcmp (p, "/") == 0)
+        break;
+
+      /* Remove one directory level and try again. */
+      d = strrchr (p, '/');
+      if (d == p)
+        p[1] = '\0';  /* check root directory */
+      else if (d)
+        *d = '\0';  /* remove last directory part */
+      else
+        *p = '\0';  /* check current directory */
+    }
+
+  xfree (p);
+
+  if (ret < 0)
+    {
+      /* pathconf() has a message for us. */
+      if (errno != 0)
+          perror ("pathconf");
+
+      /* If (errno == 0) then there is no max length.
+         Even on error return 0 so the caller can continue. */
+      return 0;
+    }
+
+  return ret;
+}
+
 #ifdef TESTING
 
 const char *
 test_subdir_p()
 {
-  int i;
-  struct {
-    char *d1;
-    char *d2;
+  static struct {
+    const char *d1;
+    const char *d2;
     bool result;
   } test_array[] = {
     { "/somedir", "/somedir", true },
     { "/somedir", "/somedir/d2", true },
     { "/somedir/d1", "/somedir", false },
   };
+  unsigned i;
 
   for (i = 0; i < countof(test_array); ++i)
     {
@@ -2389,10 +2508,9 @@ test_subdir_p()
 const char *
 test_dir_matches_p()
 {
-  int i;
-  struct {
-    char *dirlist[3];
-    char *dir;
+  static struct {
+    const char *dirlist[3];
+    const char *dir;
     bool result;
   } test_array[] = {
     { { "/somedir", "/someotherdir", NULL }, "somedir", true },
@@ -2411,6 +2529,7 @@ test_dir_matches_p()
     { { "/Tmp/has", NULL, NULL }, "/Tmp/has space", false },
     { { "/Tmp/has", NULL, NULL }, "/Tmp/has,comma", false },
   };
+  unsigned i;
 
   for (i = 0; i < countof(test_array); ++i)
     {

src/utils.h

@@ -90,6 +90,7 @@ char *file_merge (const char *, const char *);
 
 int fnmatch_nocase (const char *, const char *, int);
 bool acceptable (const char *);
+bool accept_url (const char *);
 bool accdir (const char *s);
 char *suffix (const char *s);
 bool match_tail (const char *, const char *, bool);
@@ -97,7 +98,6 @@ bool has_wildcards_p (const char *);
 
 bool has_html_suffix_p (const char *);
 
-char *read_whole_line (FILE *);
 struct file_memory *wget_read_file (const char *);
 void wget_read_file_free (struct file_memory *);
 
@@ -127,6 +127,7 @@ char *human_readable (HR_NUMTYPE);
 int numdigit (wgint);
 char *number_to_string (char *, wgint);
 char *number_to_static_string (wgint);
+wgint convert_to_bits (wgint);
 
 int determine_screen_width (void);
 int random_number (int);
@@ -141,10 +142,20 @@ void xsleep (double);
 int base64_encode (const void *, int, char *);
 int base64_decode (const char *, void *);
 
+#ifdef HAVE_LIBPCRE
+void *compile_pcre_regex (const char *);
+bool match_pcre_regex (const void *, const char *);
+#endif
+
+void *compile_posix_regex (const char *);
+bool match_posix_regex (const void *, const char *);
+
 void stable_sort (void *, size_t, size_t, int (*) (const void *, const void *));
 
 const char *print_decimal (double);
 
+size_t get_max_length (const char *path, int length, int name);
+
 extern unsigned char char_prop[];
 
 #endif /* UTILS_H */

src/warc.h

@@ -0,0 +1,26 @@
+/* Declarations of WARC helper methods. */
+#ifndef WARC_H
+#define WARC_H
+
+#include "host.h"
+
+void warc_init (void);
+void warc_close (void);
+void warc_timestamp (char *timestamp);
+void warc_uuid_str (char *id_str);
+
+FILE * warc_tempfile (void);
+
+bool warc_write_request_record (char *url, char *timestamp_str,
+  char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset);
+bool warc_write_response_record (char *url, char *timestamp_str,
+  char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset,
+  char *mime_type, int response_code, char *redirect_location);
+bool warc_write_resource_record (char *resource_uuid, const char *url,
+  const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip,
+  const char *content_type, FILE *body, off_t payload_offset);
+bool warc_write_metadata_record (char *record_uuid, const char *url,
+  const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip,
+  const char *content_type, FILE *body, off_t payload_offset);
+
+#endif /* WARC_H */

src/wget.h

@@ -314,6 +314,10 @@ typedef double SUM_SIZE_INT;
    0-pad the address.)  */
 #define PTR_FORMAT(p) (int) (2 * sizeof (void *)), (unsigned long) (p)
 
+/* Find the maximum buffer length needed to print an integer of type `x'
+   in base 10. 24082 / 10000 = 8*log_{10}(2).  */
+#define MAX_INT_TO_STRING_LEN(x) ((sizeof(x) * 24082 / 10000) + 2)
+
 extern const char *exec_name;
 
 /* Document type ("dt") flags */
@@ -353,7 +357,9 @@ typedef enum
   PROXERR,
   /* 50  */
   AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR,
-  UNLINKERR, NEWLOCATION_KEEP_POST
+  UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, ATTRMISSING, UNKNOWNATTR,
+
+  WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR
 } uerr_t;
 
 /* 2005-02-19 SMS.

tests/ChangeLog

@@ -1,3 +1,71 @@
+2013-11-04  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am: Add new tests introduced in last commit to
+          EXTRA_DIST.
+        Reported by: Andrea Urbani  <matfanjol@mail.com>
+
+2013-10-17  Andrea Urbani  <matfanjol@mail.com>
+
+	* FTPServer.pm (GetBehavior): new routine.
+	* FTPServer.pm (get_list): new parameter to skip hidden files
+	* Test-ftp-list-Multinet.px: Test LIST on a "UNIX MultiNet
+	  Unix Emulation" system that returns an empty content when
+	  "LIST -a" is requested (probably because no "-a" files
+	  exist)
+	* Test-ftp-list-Unknown.px: Test LIST on a "Unknown ftp
+	  service" system that returns an empty content when
+	  "LIST -a" is requested (probably because no "-a" files
+	  exist)
+	* Test-ftp-list-Unknown-a.px: Test LIST on a "Unknown ftp
+	  service" system that recognises "LIST -a" as "give me the
+	  -a file" and there is a "-a" file + other two files.
+	  "LIST -a" will return only "-a", "LIST" all the three files.
+	* Test-ftp-list-Unknown-hidden.px: Test LIST on a "Unknown ftp
+	  service" system that recognises "LIST -a" as an "UNIX Type:
+	  L8" system (show me also the hidden files) and there is an
+	  hidden file.
+	* Test-ftp-list-Unknown-list-a-fails.px: Test LIST on a
+	  "Unknown ftp service" system that raises an error on
+	  "LIST -a" command.
+	* Test-ftp-list-UNIX-hidden.px: Test LIST on a "UNIX Type:
+	  L8" system that recognises "LIST -a" as "show me also the
+	  hidden files" and there is an hidden file.
+
+2013-10-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Test-idn-robots-utf8.px: Remove -H.
+	* Test-idn-cmd.px: Likewise.
+	* Test-idn-cmd-utf8.px: Likewise.
+	Suggested by: Tim Ruehsen <tim.ruehsen@gmx.de>
+
+2013-10-07  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* Test-idn-robots.px: added punycoded and escaped URLs to follow
+	  removed -H
+
+2013-08-22  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* Makefile.am (EXTRA_DIST): Add Test--httpsonly-r.px.
+	* run-px (tests): Likewise.
+	* Test--httpsonly-r.px: New file.
+
+2013-03-12  Darshit Shah <darnir@gmail.com>
+
+	* Makefile.am (EXTRA_DIST): Add Test--post-file.px.
+	* run-px (tests): Likewise.
+	* Test--post-file.px: New file.
+
+2012-11-09  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* HTTPServer.pm: added check for must-not-match request-header
+	* Test-cookies.px: check cookie deletion and cookie domain matching
+
+2012-06-16  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* Makefile.am (EXTRA_DIST): Add Test-stdouterr.px.
+	* run-px (tests): Likewise.
+	* Test-stdouterr.px: New file.
+
 2011-06-03  Merinov Nikolay  <kim.roader@gmail.com>
 
 	* Test-idn-cmd-utf8.px: Added test for idn with utf-8 local encoding.

tests/FTPServer.pm

@@ -81,10 +81,25 @@ sub _LIST_command
     my ($conn, $cmd, $path) = @_;
     my $paths = $conn->{'paths'};
 
-    # This is something of a hack. Some clients expect a Unix server
-    # to respond to flags on the 'ls command line'. Remove these flags
-    # and ignore them. This is particularly an issue with ncftp 2.4.3.
-    $path =~ s/^-[a-zA-Z0-9]+\s?//;
+    my $ReturnEmptyList = ( $paths->GetBehavior('list_empty_if_list_a') &&
+                            $path eq '-a');
+    my $SkipHiddenFiles = ( $paths->GetBehavior('list_no_hidden_if_list') &&
+                            ( ! $path ) );
+
+    if ($paths->GetBehavior('list_fails_if_list_a') && $path eq '-a')
+      {
+            print {$conn->{socket}} "500 Unknown command\r\n";
+            return;
+      }
+
+
+    if (!$paths->GetBehavior('list_dont_clean_path'))
+      {
+        # This is something of a hack. Some clients expect a Unix server
+        # to respond to flags on the 'ls command line'. Remove these flags
+        # and ignore them. This is particularly an issue with ncftp 2.4.3.
+        $path =~ s/^-[a-zA-Z0-9]+\s?//;
+      }
 
     my $dir = $conn->{'dir'};
 
@@ -94,12 +109,16 @@ sub _LIST_command
     # working directory.
     local $_;
 
-    $dir = FTPPaths::path_merge($dir, $path);
-    my $listing = $paths->get_list($dir);
-    unless ($listing) {
-        print {$conn->{socket}} "550 File or directory not found.\r\n";
-        return;
-    }
+    my $listing;
+    if (!$ReturnEmptyList)
+      {
+        $dir = FTPPaths::path_merge($dir, $path);
+        $listing = $paths->get_list($dir,$SkipHiddenFiles);
+        unless ($listing) {
+            print {$conn->{socket}} "550 File or directory not found.\r\n";
+            return;
+        }
+      }
 
     print STDERR "_LIST_command - dir is: $dir\n" if $log;
 
@@ -112,9 +131,12 @@ sub _LIST_command
         return;
     }
 
-    for my $item (@$listing) {
-        print $sock "$item\r\n";
-    }
+    if (!$ReturnEmptyList)
+      {
+        for my $item (@$listing) {
+            print $sock "$item\r\n";
+        }
+      }
 
     unless ($sock->close) {
         print {$conn->{socket}} "550 Error closing data connection: $!\r\n";
@@ -372,7 +394,14 @@ sub _SYST_command
 {
     my ($conn, $cmd, $dummy) = @_;
 
-    print {$conn->{socket}} "215 UNIX Type: L8\r\n";
+    if ($conn->{'paths'}->GetBehavior('syst_response'))
+      {
+        print {$conn->{socket}} $conn->{'paths'}->GetBehavior('syst_response') . "\r\n";
+      }
+    else
+      {
+        print {$conn->{socket}} "215 UNIX Type: L8\r\n";
+      }
 }
 
 sub _TYPE_command
@@ -780,7 +809,7 @@ sub _format_for_list {
 }
 
 sub get_list {
-    my ($self, $path) = @_;
+    my ($self, $path, $no_hidden) = @_;
     my $info = $self->get_info($path);
     return undef unless defined $info;
     my $list = [];
@@ -788,7 +817,19 @@ sub get_list {
     if ($info->{'_type'} eq 'd') {
         for my $item (keys %$info) {
             next if $item =~ /^_/;
-            push @$list, $self->_format_for_list($item, $info->{$item});
+            # 2013-10-17 Andrea Urbani (matfanjol)
+            #            I skip the hidden files if requested
+            if (($no_hidden) &&
+                (defined($info->{$item}->{'attr'})) &&
+                (index($info->{$item}->{'attr'}, "H")>=0))
+              {
+                # This is an hidden file and I don't want to see it!
+                print STDERR "get_list: Skipped hidden file [$item]\n";
+              }
+            else
+              {
+                push @$list, $self->_format_for_list($item, $info->{$item});
+              }
         }
     } else {
         push @$list, $self->_format_for_list(final_component($path), $info);
@@ -797,6 +838,29 @@ sub get_list {
     return $list;
 }
 
+# 2013-10-17 Andrea Urbani (matfanjol)
+# It returns the behavior of the given name.
+# In this file I handle also the following behaviors:
+#  list_dont_clean_path  : if defined, the command
+#                           $path =~ s/^-[a-zA-Z0-9]+\s?//;
+#                          is not runt and the given path
+#                          remains the original one
+#  list_empty_if_list_a  : if defined, "LIST -a" returns an
+#                          empty content
+#  list_fails_if_list_a  : if defined, "LIST -a" returns an
+#                          error
+#  list_no_hidden_if_list: if defined, "LIST" doesn't return
+#                          hidden files.
+#                          To define an hidden file add
+#                            attr => "H"
+#                          to the url files
+#  syst_response         : if defined, its content is printed
+#                          out as SYST response
+sub GetBehavior {
+  my ($self, $name) = @_;
+  return $self->{'_behavior'}{$name};
+}
+
 1;
 
 # vim: et ts=4 sw=4

tests/HTTPServer.pm

@@ -218,12 +218,24 @@ sub verify_request_headers {
 
     return 1 unless exists $url_rec->{'request_headers'};
     for my $hdrname (keys %{$url_rec->{'request_headers'}}) {
-        my $rhdr = $req->header ($hdrname);
+        my $must_not_match;
         my $ehdr = $url_rec->{'request_headers'}{$hdrname};
-        unless (defined $rhdr && $rhdr =~ $ehdr) {
-            $rhdr = '' unless defined $rhdr;
-            print STDERR "\n*** Mismatch on $hdrname: $rhdr =~ $ehdr\n";
-            return undef;
+        if ($must_not_match = ($hdrname =~ /^!(\w+)/)) {
+            $hdrname = $1;
+        }
+        my $rhdr = $req->header ($hdrname);
+        if ($must_not_match) {
+            if (defined $rhdr && $rhdr =~ $ehdr) {
+                $rhdr = '' unless defined $rhdr;
+                print STDERR "\n*** Match forbidden $hdrname: $rhdr =~ $ehdr\n";
+                return undef;
+            }
+        } else {
+            unless (defined $rhdr && $rhdr =~ $ehdr) {
+                $rhdr = '' unless defined $rhdr;
+                print STDERR "\n*** Mismatch on $hdrname: $rhdr =~ $ehdr\n";
+                return undef;
+            }
         }
     }
 

tests/Makefile.am

@@ -82,6 +82,12 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
              Test-ftp-iri-fallback.px \
              Test-ftp-iri-recursive.px \
              Test-ftp-iri-disabled.px \
+             Test-ftp-list-Multinet.px \
+             Test-ftp-list-Unknown.px \
+             Test-ftp-list-Unknown-a.px \
+             Test-ftp-list-Unknown-hidden.px \
+             Test-ftp-list-Unknown-list-a-fails.px \
+             Test-ftp-list-UNIX-hidden.px \
              Test-HTTP-Content-Disposition-1.px \
              Test-HTTP-Content-Disposition-2.px \
              Test-HTTP-Content-Disposition.px \
@@ -119,17 +125,20 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
              Test-O--no-content-disposition-trivial.px \
              Test-O-nonexisting.px \
              Test-O.px \
+             Test--post-file.px \
              Test-proxied-https-auth.px \
              Test-proxy-auth-basic.px \
              Test-restrict-ascii.px \
              Test-Restrict-Lowercase.px \
              Test-Restrict-Uppercase.px \
+	     Test-stdouterr.px \
              Test--spider-fail.px \
              Test--spider.px \
              Test--spider-r-HTTP-Content-Disposition.px \
              Test--spider-r--no-content-disposition.px \
              Test--spider-r--no-content-disposition-trivial.px \
              Test--spider-r.px \
+             Test--httpsonly-r.px \
              run-px certs
 
 check_PROGRAMS = unit-tests

tests/Test--httpsonly-r.px

@@ -0,0 +1,79 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use HTTPTest;
+
+
+###############################################################################
+
+my $mainpage = <<EOF;
+<html>
+<head>
+  <title>Main Page</title>
+</head>
+<body>
+  <p>
+    Some text and a link to a <a href="http://localhost:{{port}}/secondpage.html">second page</a>.
+  </p>
+</body>
+</html>
+EOF
+
+my $secondpage = <<EOF;
+<html>
+<head>
+  <title>Second Page</title>
+</head>
+<body>
+  <p>
+    Anything.
+  </p>
+</body>
+</html>
+EOF
+
+# code, msg, headers, content
+my %urls = (
+    '/index.html' => {
+        code => "200",
+        msg => "Dontcare",
+        headers => {
+            "Content-type" => "text/html",
+        },
+        content => $mainpage,
+    },
+    '/secondpage.html' => {
+        code => "200",
+        msg => "Dontcare",
+        headers => {
+            "Content-type" => "text/html",
+        },
+        content => $secondpage,
+    }
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --https-only -r -nH http://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'index.html' => {
+        content => $mainpage,
+    },
+);
+
+###############################################################################
+
+my $the_test = HTTPTest->new (name => "Test--httpsonly-r",
+                              input => \%urls,
+                              cmdline => $cmdline,
+                              errcode => $expected_error_code,
+                              output => \%expected_downloaded_files);
+print $expected_error_code."\n";
+
+exit $the_test->run();
+
+# vim: et ts=4 sw=4
+

tests/Test--post-file.px

@@ -0,0 +1,23 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use HTTPTest;
+
+
+###############################################################################
+
+my $cmdline = $WgetTest::WGETPATH . " -d --post-file=nofile http://localhost:{{port}}/";
+
+my $expected_error_code = 3;
+
+
+###############################################################################
+
+my $the_test = HTTPTest->new (name => "Test-missing-file",
+                              cmdline => $cmdline,
+                              errcode => $expected_error_code);
+exit $the_test->run();
+
+# vim: et ts=4 sw=4

tests/Test-cookies.px

@@ -10,6 +10,10 @@ use HTTPTest;
 
 my $page1 = "Hello, world!\n";
 my $page2 = "Goodbye, Sam.\n";
+my $page3 = "Page three.\n";
+my $page4 = "Page four.\n";
+my $page5 = "Page five.\n";
+my $page6 = "Page six.\n";
 
 # code, msg, headers, content
 my %urls = (
@@ -30,10 +34,49 @@ my %urls = (
             "Cookie" => qr|foo=bar|,
         },
     },
+# remove the cookie 'foo'
+    '/three.txt' => {
+        code => "200",
+        msg => "Ok",
+        headers => {
+            "Content-type" => "text/plain",
+            "Set-Cookie" => "foo=; Expires=Sun, 06 Nov 1994 08:49:37 GMT",
+        },
+        content => $page3,
+    },
+    '/four.txt' => {
+        code => "200",
+        msg => "Ok",
+        content => $page4,
+        request_headers => {
+            "!Cookie" => qr|foo=|,
+        },
+    },
+# try to set a cookie 'foo' with mismatching domain
+# see RFC 6265 5.3.6: ignore the cookie if it doesn't domain-match
+    '/five.txt' => {
+        code => "200",
+        msg => "Ok",
+        headers => {
+            "Content-type" => "text/plain",
+            "Set-Cookie" => "foo=bar; domain=.example.com",
+        },
+        content => $page5,
+    },
+    '/six.txt' => {
+        code => "200",
+        msg => "Ok",
+        content => $page6,
+        request_headers => {
+            "!Cookie" => qr|foo=bar|,
+        },
+    },
 );
 
 my $cmdline = $WgetTest::WGETPATH . " http://localhost:{{port}}/one.txt"
-    . " http://localhost:{{port}}/two.txt";
+    . " http://localhost:{{port}}/two.txt" . " http://localhost:{{port}}/three.txt"
+    . " http://localhost:{{port}}/four.txt" . " http://localhost:{{port}}/five.txt"
+	 . " http://localhost:{{port}}/six.txt";
 
 my $expected_error_code = 0;
 
@@ -44,6 +87,18 @@ my %expected_downloaded_files = (
     'two.txt' => {
         content => $page2,
     },
+    'three.txt' => {
+        content => $page3,
+    },
+    'four.txt' => {
+        content => $page4,
+    },
+    'five.txt' => {
+        content => $page5,
+    },
+    'six.txt' => {
+        content => $page6,
+    },
 );
 
 ###############################################################################

tests/Test-ftp-list-Multinet.px

@@ -0,0 +1,67 @@
+#!/usr/bin/env perl
+
+
+# 2013-10-17 Andrea Urbani (matfanjol)
+# In this ftp test:
+# - the response of SYST command is
+#   215 UNIX MultiNet Unix Emulation V5.3(93)
+# - the response of "LIST -a" command is an empty
+#   directory.
+# wget should use directly the "LIST" command to get
+# the right content, but it will be ok also "LIST -a"
+# if followed by "LIST" (in the case of future changes).
+
+
+use strict;
+use warnings;
+
+use FTPTest;
+
+
+###############################################################################
+
+my $afile = <<EOF;
+Some text.
+EOF
+
+my $bfile = <<EOF;
+Some more text.
+EOF
+
+$afile =~ s/\n/\r\n/g;
+$bfile =~ s/\n/\r\n/g;
+
+# code, msg, headers, content
+my %urls = (
+    '/afile.txt' => {
+        content => $afile,
+    },
+    '/bfile.txt' => {
+        content => $bfile,
+    },
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --no-directories --recursive --level=1 --accept \"?file.txt\" ftp://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'afile.txt' => {
+        content => $afile,
+    },
+    'bfile.txt' => {
+        content => $bfile,
+    },
+);
+
+###############################################################################
+
+my $the_test = FTPTest->new (name => "Test-ftp-list-Multinet",
+                             input => \%urls,
+                             cmdline => $cmdline,
+                             errcode => $expected_error_code,
+                             output => \%expected_downloaded_files,
+                             server_behavior => {list_empty_if_list_a => 1,
+                                                 syst_response => "215 UNIX MultiNet Unix Emulation V5.3(93)"});
+exit $the_test->run();
+

tests/Test-ftp-list-UNIX-hidden.px

@@ -0,0 +1,65 @@
+#!/usr/bin/env perl
+
+# 2013-10-17 Andrea Urbani (matfanjol)
+# In this ftp test:
+# - the response of "LIST -a" command contains
+#   all the files
+# - the response of "LIST" command contains
+#   the normal files (hidden files are not present)
+# wget should use only "LIST -a" because it recognise
+# the system as "UNIX Type: L8" and so it should see
+# and download the hidden file too.
+
+use strict;
+use warnings;
+
+use FTPTest;
+
+
+###############################################################################
+
+my $normalfile = <<EOF;
+I'm a normal file
+EOF
+
+my $hiddenfile = <<EOF;
+I'm an hidden file
+EOF
+
+$normalfile =~ s/\n/\r\n/g;
+$hiddenfile =~ s/\n/\r\n/g;
+
+# code, msg, headers, content
+my %urls = (
+    '/normalfile.txt' => {
+        content => $normalfile,
+    },
+    '/hiddenfile.txt' => {
+        content => $hiddenfile,
+        attr => "H",
+    },
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --no-directories --recursive --level=1 ftp://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'normalfile.txt' => {
+        content => $normalfile,
+    },
+    'hiddenfile.txt' => {
+        content => $hiddenfile,
+    },
+);
+
+###############################################################################
+
+my $the_test = FTPTest->new (name => "Test-ftp-list-UNIX-hidden",
+                             input => \%urls,
+                             cmdline => $cmdline,
+                             errcode => $expected_error_code,
+                             output => \%expected_downloaded_files,
+                             server_behavior => {list_no_hidden_if_list => 1});
+exit $the_test->run();
+

tests/Test-ftp-list-Unknown-a.px

@@ -0,0 +1,77 @@
+#!/usr/bin/env perl
+
+# 2013-10-17 Andrea Urbani (matfanjol)
+# In this ftp test:
+# - the response of SYST command is
+#   215 Unknown ftp service
+# - the response of "LIST -a" command is a file
+#   called "-a".
+# wget should use "LIST -a", but also "LIST".
+# After "LIST", wget will see more data is available.
+# (See also Test-ftp-list-Unknown-b.px)
+
+use strict;
+use warnings;
+
+use FTPTest;
+
+
+###############################################################################
+
+my $afile = <<EOF;
+Some text.
+EOF
+
+my $bfile = <<EOF;
+Some more text.
+EOF
+
+
+my $minusafile = <<EOF;
+The strange situation.
+EOF
+
+$afile =~ s/\n/\r\n/g;
+$bfile =~ s/\n/\r\n/g;
+$minusafile =~ s/\n/\r\n/g;
+
+# code, msg, headers, content
+my %urls = (
+    '/afile.txt' => {
+        content => $afile,
+    },
+    '/bfile.txt' => {
+        content => $bfile,
+    },
+    '/-a' => {
+        content => $minusafile,
+    },
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --no-directories --recursive --level=1 ftp://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'afile.txt' => {
+        content => $afile,
+    },
+    'bfile.txt' => {
+        content => $bfile,
+    },
+    '-a' => {
+        content => $minusafile,
+    },
+);
+
+###############################################################################
+
+my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown-a",
+                             input => \%urls,
+                             cmdline => $cmdline,
+                             errcode => $expected_error_code,
+                             output => \%expected_downloaded_files,
+                             server_behavior => {list_dont_clean_path => 1,
+                                                 syst_response => "215 Unknown ftp service"});
+exit $the_test->run();
+

tests/Test-ftp-list-Unknown-hidden.px

@@ -0,0 +1,69 @@
+#!/usr/bin/env perl
+
+# 2013-10-17 Andrea Urbani (matfanjol)
+# In this ftp test:
+# - the response of SYST command is
+#   215 Unknown ftp service
+# - the response of "LIST -a" command contains
+#   all the files
+# - the response of "LIST" command contains
+#   the normal files (hidden files are not present)
+# wget should use "LIST -a", but also "LIST".
+# After "LIST", wget will see more data is available
+# on "LIST -a", so it should go back to "LIST -a".
+# (See also Test-ftp-list-Unknown-a.px)
+
+use strict;
+use warnings;
+
+use FTPTest;
+
+
+###############################################################################
+
+my $normalfile = <<EOF;
+I'm a normal file
+EOF
+
+my $hiddenfile = <<EOF;
+I'm an hidden file
+EOF
+
+$normalfile =~ s/\n/\r\n/g;
+$hiddenfile =~ s/\n/\r\n/g;
+
+# code, msg, headers, content
+my %urls = (
+    '/normalfile.txt' => {
+        content => $normalfile,
+    },
+    '/hiddenfile.txt' => {
+        content => $hiddenfile,
+        attr => "H",
+    },
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --no-directories --recursive --level=1 ftp://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'normalfile.txt' => {
+        content => $normalfile,
+    },
+    'hiddenfile.txt' => {
+        content => $hiddenfile,
+    },
+);
+
+###############################################################################
+
+my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown-hidden",
+                             input => \%urls,
+                             cmdline => $cmdline,
+                             errcode => $expected_error_code,
+                             output => \%expected_downloaded_files,
+                             server_behavior => {list_no_hidden_if_list => 1,
+                                                 syst_response => "215 Unknown ftp service"});
+exit $the_test->run();
+

tests/Test-ftp-list-Unknown-list-a-fails.px

@@ -0,0 +1,62 @@
+#!/usr/bin/env perl
+
+
+# 2013-10-17 Andrea Urbani (matfanjol)
+# In this ftp test:
+# - the response of "LIST -a" command is a failure
+# wget should use "LIST -a" then "LIST" to get the right
+# content.
+
+use strict;
+use warnings;
+
+use FTPTest;
+
+
+###############################################################################
+
+my $afile = <<EOF;
+Some text.
+EOF
+
+my $bfile = <<EOF;
+Some more text.
+EOF
+
+$afile =~ s/\n/\r\n/g;
+$bfile =~ s/\n/\r\n/g;
+
+# code, msg, headers, content
+my %urls = (
+    '/afile.txt' => {
+        content => $afile,
+    },
+    '/bfile.txt' => {
+        content => $bfile,
+    },
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --no-directories --recursive --level=1 --accept \"?file.txt\" ftp://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'afile.txt' => {
+        content => $afile,
+    },
+    'bfile.txt' => {
+        content => $bfile,
+    },
+);
+
+###############################################################################
+
+my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown-list-a-fails",
+                             input => \%urls,
+                             cmdline => $cmdline,
+                             errcode => $expected_error_code,
+                             output => \%expected_downloaded_files,
+                             server_behavior => {list_fails_if_list_a => 1,
+                                                 syst_response => "215 Unknown ftp service"});
+exit $the_test->run();
+

tests/Test-ftp-list-Unknown.px

@@ -0,0 +1,65 @@
+#!/usr/bin/env perl
+
+
+# 2013-10-17 Andrea Urbani (matfanjol)
+# In this ftp test:
+# - the response of SYST command is
+#   215 Unknown ftp service
+# - the response of "LIST -a" command is an empty
+#   directory.
+# wget should use "LIST -a" then "LIST" to get the right
+# content.
+
+use strict;
+use warnings;
+
+use FTPTest;
+
+
+###############################################################################
+
+my $afile = <<EOF;
+Some text.
+EOF
+
+my $bfile = <<EOF;
+Some more text.
+EOF
+
+$afile =~ s/\n/\r\n/g;
+$bfile =~ s/\n/\r\n/g;
+
+# code, msg, headers, content
+my %urls = (
+    '/afile.txt' => {
+        content => $afile,
+    },
+    '/bfile.txt' => {
+        content => $bfile,
+    },
+);
+
+my $cmdline = $WgetTest::WGETPATH . " --no-directories --recursive --level=1 --accept \"?file.txt\" ftp://localhost:{{port}}/";
+
+my $expected_error_code = 0;
+
+my %expected_downloaded_files = (
+    'afile.txt' => {
+        content => $afile,
+    },
+    'bfile.txt' => {
+        content => $bfile,
+    },
+);
+
+###############################################################################
+
+my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown",
+                             input => \%urls,
+                             cmdline => $cmdline,
+                             errcode => $expected_error_code,
+                             output => \%expected_downloaded_files,
+                             server_behavior => {list_empty_if_list_a => 1,
+                                                 syst_response => "215 Unknown ftp service"});
+exit $the_test->run();
+

tests/Test-idn-cmd-utf8.px

@@ -28,7 +28,7 @@ my %urls = (
     },
 );
 
-my $cmdline = $WgetTest::WGETPATH . " --iri -rH"
+my $cmdline = $WgetTest::WGETPATH . " --iri -r"
     . " -e http_proxy=localhost:{{port}} --local-encoding=UTF-8 $utf8_hostname";
 
 my $expected_error_code = 0;

tests/Test-idn-cmd.px

@@ -28,7 +28,7 @@ my %urls = (
     },
 );
 
-my $cmdline = $WgetTest::WGETPATH . " --iri -rH"
+my $cmdline = $WgetTest::WGETPATH . " --iri -r"
     . " -e http_proxy=localhost:{{port}} --local-encoding=EUC-JP $euc_jp_hostname";
 
 my $expected_error_code = 0;

tests/Test-idn-robots-utf8.px

@@ -48,7 +48,7 @@ my %urls = (
     },
 );
 
-my $cmdline = $WgetTest::WGETPATH . " --iri -rH"
+my $cmdline = $WgetTest::WGETPATH . " --iri -r"
     . " -e http_proxy=localhost:{{port}} --local-encoding=UTF-8"
     . " http://$utf8_hostname/";
 

tests/Test-idn-robots.px

@@ -9,11 +9,14 @@ use HTTPTest;
 # " Kon'nichiwa <dot> Japan
 my $euc_jp_hostname = "\272\243\306\374\244\317.\306\374\313\334";
 my $punycoded_hostname = 'xn--v9ju72g90p.xn--wgv71a';
+my $escaped_hostname = "%ba%a3%c6%fc%a4%cf.%c6%fc%cb%dc";
 
 ###############################################################################
 
 my $starter_file = <<EOF;
 <a href="http://$euc_jp_hostname/foo.txt">The link</a>
+<a href="http://$punycoded_hostname/foo2.txt">The second link</a>
+<a href="http://$escaped_hostname/foo3.txt">The third link</a>
 EOF
 
 my $result_file = <<EOF;
@@ -38,6 +41,22 @@ my %urls = (
         },
         content => $result_file,
     },
+    "http://$punycoded_hostname/foo2.txt" => {
+        code => "200",
+        msg => "Uh-huh2",
+        headers => {
+            'Content-Type' => 'text/plain',
+        },
+        content => $result_file,
+    },
+    "http://$punycoded_hostname/foo3.txt" => {
+        code => "200",
+        msg => "Uh-huh3",
+        headers => {
+            'Content-Type' => 'text/plain',
+        },
+        content => $result_file,
+    },
     "http://$punycoded_hostname/robots.txt" => {
         code => "200",
         msg => "Uh-huh",
@@ -48,7 +67,7 @@ my %urls = (
     },
 );
 
-my $cmdline = $WgetTest::WGETPATH . " --iri -rH"
+my $cmdline = $WgetTest::WGETPATH . " --iri -r"
     . " -e http_proxy=localhost:{{port}} --local-encoding=EUC-JP"
     . " http://$euc_jp_hostname/";
 
@@ -61,6 +80,12 @@ my %expected_downloaded_files = (
     "$punycoded_hostname/foo.txt" => {
         content => $result_file,
     },
+    "$punycoded_hostname/foo2.txt" => {
+        content => $result_file,
+    },
+    "$punycoded_hostname/foo3.txt" => {
+        content => $result_file,
+    },
     "$punycoded_hostname/robots.txt" => {
         content => '',
     },

tests/Test-stdouterr.px

@@ -0,0 +1,48 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use HTTPTest;
+
+
+###############################################################################
+
+# code, msg, headers, content
+my %urls = (
+    '/somefile.txt' => {
+        code => "200",
+        msg => "Dontcare",
+        headers => {
+            "Content-type" => "text/plain",
+        },
+        content => "blabla",
+    },
+);
+
+unless(-e "/dev/full") {
+    exit(2); # skip
+}
+
+my $cmdline = $WgetTest::WGETPATH . " -c http://localhost:{{port}}/somefile.txt -O /dev/full";
+
+my $expected_error_code = 3;
+
+my %existing_files = (
+);
+
+my %expected_downloaded_files = (
+);
+
+###############################################################################
+
+my $the_test = HTTPTest->new (name => "Test-stdouterr",
+                              input => \%urls,
+                              cmdline => $cmdline,
+                              errcode => $expected_error_code,
+                              existing => \%existing_files,
+                              output => \%expected_downloaded_files);
+exit $the_test->run();
+
+# vim: et ts=4 sw=4
+

tests/run-px

@@ -35,6 +35,12 @@ my @tests = (
     'Test-ftp-iri-fallback.px',
     'Test-ftp-iri-recursive.px',
     'Test-ftp-iri-disabled.px',
+    'Test-ftp-list-Multinet.px',
+    'Test-ftp-list-Unknown.px',
+    'Test-ftp-list-Unknown-a.px',
+    'Test-ftp-list-Unknown-hidden.px',
+    'Test-ftp-list-Unknown-list-a-fails.px',
+    'Test-ftp-list-UNIX-hidden.px',
     'Test-HTTP-Content-Disposition-1.px',
     'Test-HTTP-Content-Disposition-2.px',
     'Test-HTTP-Content-Disposition.px',
@@ -70,15 +76,18 @@ my @tests = (
     'Test-O--no-content-disposition-trivial.px',
     'Test-O-nonexisting.px',
     'Test-O.px',
+    'Test--post-file.px',
     'Test-O-nc.px',
     'Test-restrict-ascii.px',
     'Test-Restrict-Lowercase.px',
     'Test-Restrict-Uppercase.px',
+    'Test-stdouterr.px',
     'Test--spider-fail.px',
     'Test--spider-r-HTTP-Content-Disposition.px',
     'Test--spider-r--no-content-disposition.px',
     'Test--spider-r--no-content-disposition-trivial.px',
     'Test--spider-r.px',
+    'Test--httpsonly-r.px',
 );
 
 foreach my $var (qw(SYSTEM_WGETRC WGETRC)) {

util/paramcheck.pl

@@ -33,11 +33,11 @@ my $tex_content  = read_file($tex_file);
 
 my @args = ([
     $main_content,
-    qr/static \s+? struct \s+? cmdline_option \s+? option_data\[\] \s+? = \s+? \{ (.*?) \}\;/sx,
+    qr/static \s+? struct \s+? cmdline_option \s+? option_data\[\] \s+? = \s+? \{ (.+?) \}\;/sx,
     [ qw(long_name short_name type data argtype) ],
 ], [
     $init_content,
-    qr/commands\[\] \s+? = \s+? \{ (.*?) \}\;/sx,
+    qr/commands\[\] \s+? = \s+? \{ (.+?) \}\;/sx,
     [ qw(name place action) ],
 ]);
 
@@ -78,18 +78,18 @@ sub extract_entries
     my (@entries, %index, $i);
 
     foreach my $chunk (@$chunks) {
-        my ($args) = $chunk =~ /\{ \s+? (.*?) \s+? \}/sx;
+        my ($args) = $chunk =~ /\{ \s+? (.+?) \s+? \}/sx;
         next unless defined $args;
 
         my @args = map {
           tr/'"//d; $_
         } map {
-          /\((.*?)\)/ ? $1 : $_
+          /\((.+?)\)/ ? $1 : $_
         } split /\,\s+/, $args;
 
         my $entry = { map { $_ => shift @args } @$names };
 
-        ($entry->{line}) = $chunk =~ /^ \s+? (\{.*)/mx;
+        ($entry->{line}) = $chunk =~ /^ \s+? (\{.+)/mx;
         if ($chunk =~ /deprecated/i) {
             $entries[-1]->{deprecated} = true;
         }
@@ -103,9 +103,9 @@ sub extract_entries
         push @entries, $entry;
     }
 
-    push @entries, \%index;
+    push @entries, { %index };
 
-    return \@entries;
+    return [ @entries ];
 }
 
 sub output_results
@@ -281,7 +281,7 @@ sub emit_undocumented_opts
     while ($tex =~ /^\@item\w*? \s+? --([-a-z0-9]+)/gmx) {
         $tex_items{$1} = true;
     }
-    my ($help) = $main =~ /\n print_help .*? \{\n (.*) \n\} \n/sx;
+    my ($help) = $main =~ /\n print_help .*? \{\n (.+) \n\} \n/sx;
     while ($help =~ /--([-a-z0-9]+)/g) {
         $main_items{$1} = true;
     }

util/trunc.c

@@ -128,5 +128,11 @@ main (int argc, char *argv[])
       exit (EXIT_FAILURE);
     }
 
+  if (close (fd) < 0)
+    {
+      perror (PROGRAM_NAME ": close");
+      exit (EXIT_FAILURE);
+    }
+
   return 0;
 }