* NEWS: Update NEWS for new release

* src/iri.c(do_conversion): Reallocate the output buffer to a larger
  size if it is already full

.gitlab-ci.yml

@@ -111,6 +111,32 @@ GnuTLS/Debian:
       - tests/*.log
       - testenv/*.log
 
+Minimal/Debian:
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
+  script:
+    - cp -a /builds/common/gnulib .
+    - export CFLAGS=$CFLAGS_DEFAULT
+    - ./bootstrap --skip-po
+    - ./configure $CONFIGURE_BASE_FLAGS --cache-file ../cache/config.cache
+      --disable-nls --without-ssl --enable-ipv6 --without-zlib --without-libiconv-prefix
+      --disable-iri --disable-ntlm --disable-pcre --without-libpsl --without-libuuid
+      --without-libintl-prefix
+    - make -j$(nproc)
+    - make -j$(nproc) check
+    - make check-valgrind
+  tags:
+    - shared
+    - docker
+  artifacts:
+    expire_in: 2 weeks
+    when: on_failure
+    paths:
+      - ./*.log
+      - fuzz/*.log
+      - tests/*.log
+      - testenv/*.log
+
+
 #Centos7 build:
 #  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$CENTOS7_BUILD
 #  script:

.travis.yml

@@ -10,7 +10,7 @@ notifications:
     email:
         recipients:
             - darnir@gmail.com
-            - bug-wget@gnu.org
+            - wget-dev@gnu.org
         on_success: change
         on_failure: always
 

Makefile.am

@@ -1,6 +1,5 @@
 # Makefile for `Wget' utility
-# Copyright (C) 1995-1997, 2006-2018 Free Software Foundation,
-# Inc.
+# Copyright (C) 1995-1997, 2006-2019 Free Software Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

NEWS

@@ -1,10 +1,31 @@
 GNU Wget NEWS -- history of user-visible changes.
 
-Copyright (C) 1997-2018 Free Software Foundation, Inc.
+Copyright (C) 1997-2019 Free Software Foundation, Inc.
 See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
+* Changes in Wget 1.20.3
+
+** Fixed a buffer overflow vulnerability
+
+
+* Changes in Wget 1.20.2
+
+** NTLM authentication will retry under certain cases
+
+
+* Changes in Wget 1.20.1
+
+** --xattr is no longer default since it introduces privacy issues.
+
+** --xattr saves the Referer as scheme/host/port, user/pw/path/query/fragment
+   are no longer saved to prevent privacy issues.
+
+** --xattr saves the Original URL without user/password to prevent
+   privacy issues.
+
+
 * Changes in Wget 1.20
 
 ** Add new option `--retry-on-host-error` to treat local errors as transient

README.checkout

@@ -161,8 +161,8 @@ Testing and development
  that bug, which might be tedious. Tests should only be run through GDB for that purpose.
 
 
- Copyright (C) 2008, 2010, 2014-2015, 2018 Free Software Foundation,
- Inc.
+ Copyright (C) 2008, 2010, 2014-2015, 2018-2019 Free Software
+ Foundation, Inc.
  License GPLv3+: GNU GPL version 3 or later
  <https://www.gnu.org/licenses/gpl.html>.
  This is free software: you are free to change and redistribute it.

bootstrap

@@ -4,7 +4,7 @@ scriptversion=2018-10-13.05; # UTC
 
 # Bootstrap this package from checked-out sources.
 
-# Copyright (C) 2003-2018 Free Software Foundation, Inc.
+# Copyright (C) 2003-2019 Free Software Foundation, Inc.
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

bootstrap.conf

@@ -1,5 +1,5 @@
 # bootstrap.conf - Bootstrap configuration.
-# Copyright (C) 2007-2012, 2018 Free Software Foundation, Inc.
+# Copyright (C) 2007-2012, 2018-2019 Free Software Foundation, Inc.
 #
 # This file is part of GNU Wget.
 #

build-aux/build_info.pl

@@ -2,7 +2,7 @@
 
 # Generate build_info.c.
 
-# Copyright (C) 2009-2011, 2018 Free Software Foundation, Inc.
+# Copyright (C) 2009-2011, 2018-2019 Free Software Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

configure.ac

@@ -1,5 +1,5 @@
 dnl Template file for GNU Autoconf
-dnl Copyright (C) 1995-1997, 2001, 2007-2014, 2018 Free Software
+dnl Copyright (C) 1995-1997, 2001, 2007-2014, 2018-2019 Free Software
 dnl Foundation, Inc.
 
 dnl This program is free software; you can redistribute it and/or modify
@@ -51,7 +51,7 @@ dnl
 AM_INIT_AUTOMAKE([1.9])
 
 dnl
-dnl Get cannonical host
+dnl Get canonical host
 dnl
 AC_CANONICAL_HOST
 AC_DEFINE_UNQUOTED([OS_TYPE], "$host_os",
@@ -103,6 +103,15 @@ AC_ARG_ENABLE([fuzzing],
 #AC_SUBST([FUZZ_LIBS])
 AM_CONDITIONAL([FUZZING], [test "$enable_fuzzing" = "yes"])
 
+dnl needed for some fuzzers
+OLD_LIBS="$LIBS"
+AC_SEARCH_LIBS([dlopen], [dl dld])
+if [[ "${ac_cv_search_dlopen#-l}" != "${ac_cv_search_dlopen}" ]]; then
+  FUZZ_LIBS="${ac_cv_search_dlopen}"
+  AC_SUBST([FUZZ_LIBS])
+fi
+LIBS="$OLD_LIBS"
+
 dnl Opie: Support for opie s/key FTP logins
 AC_ARG_ENABLE([opie],
   [AS_HELP_STRING([--disable-opie], [disable support for opie or s/key FTP login])],
@@ -281,7 +290,8 @@ AC_CHECK_TYPES(sig_atomic_t, [], [], [
 #include <signal.h>
 ])
 
-# gnulib
+# gnulib, AM_ICONV should be placed before gl_INIT
+AM_ICONV
 gl_INIT
 
 dnl
@@ -618,8 +628,6 @@ AC_ARG_ENABLE(iri,
 )
 
 AS_IF([test "X$iri" != "Xno"],[
-  AM_ICONV
-
   if test "X$am_cv_func_iconv" != "Xyes"; then
     iri=no
     if test "X$force_iri" = "Xyes"; then

contrib/make-release

@@ -102,7 +102,14 @@ echo ""
 build-aux/announce-gen --release-type "$REL_TYPE" --package-name "$APP_LOWER" --previous-version="$PREV_VER" --current-version "$CUR_VER" --gpg-key-id="$GPG_KEY" --url-directory "https://${REL_LOC}/gnu/${APP_LOWER}" --news=NEWS > "$APP_NAME-$APP_VER.announce"
 
 echo ""
-echo "Announce email template placed at $APP_NAME-$APP_VER.announce"
+echo "Remaining Release Checklist:
+  1. Send a mail to coordinator@translationproject.org with the subject
+	 \"Requesting Translations for $APP_NAME-$APP_VER.POT\"
+  2. Announce email template placed at $APP_NAME-$APP_VER.announce.
+     Send an announce mail to info-gnu@gnu.org
+"
+
+echo ""
 
 prev_tag=$(git tag | tail -2 | head -1)
 echo ""

contrib/spell-checker

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+#
+# see https://github.com/codespell-project/codespell
+#
+# in short:
+#  - pip install codespell
+#  - add ~/.local/bin to PATH
+
+codespell $* -L "tim,gonna,cas,tage,lightening,wither" `git ls-files|egrep -v '_fuzzer.in|_fuzzer.repro|\.der$|gnulib|ChangeLog'`

doc/Makefile.am

@@ -1,5 +1,5 @@
 # Makefile for `wget' utility
-# Copyright (C) 1995-1997, 2007-2011, 2015, 2018 Free Software
+# Copyright (C) 1995-1997, 2007-2011, 2015, 2018-2019 Free Software
 # Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify

doc/fdl.texi

@@ -5,8 +5,8 @@
 @c hence no sectioning command or @node.
 
 @display
-Copyright @copyright{} 2000-2002, 2007-2008, 2015, 2018 Free Software
-Foundation, Inc.
+Copyright @copyright{} 2000-2002, 2007-2008, 2015, 2018-2019 Free
+Software Foundation, Inc.
 @uref{http://fsf.org/}
 
 Everyone is permitted to copy and distribute verbatim copies

doc/metalink-standard.txt

@@ -106,7 +106,7 @@ found unsafe too, the file is not downloaded.
 
 See [1 #section-1.1].
 
-cat > bugus.meta4 << EOF
+cat > bogus.meta4 << EOF
 <?xml version="1.0" encoding="UTF-8"?>
 <metalink xmlns="urn:ietf:params:xml:ns:metalink">
   <file name="/dir/A/File1">

doc/texi2pod.pl

@@ -1,6 +1,6 @@
 #! /usr/bin/env perl
 
-#   Copyright (C) 1999-2001, 2003, 2007, 2009-2011, 2015, 2018 Free
+#   Copyright (C) 1999-2001, 2003, 2007, 2009-2011, 2015, 2018-2019 Free
 #   Software Foundation, Inc.
 
 # This file is part of GCC.

doc/wget.texi

@@ -31,8 +31,8 @@ This file documents the GNU Wget utility for downloading network
 data.
 
 @c man begin COPYRIGHT
-Copyright @copyright{} 1996-2011, 2015, 2018 Free Software Foundation,
-Inc.
+Copyright @copyright{} 1996-2011, 2015, 2018-2019 Free Software
+Foundation, Inc.
 
 @iftex
 Permission is granted to make and distribute verbatim copies of
@@ -540,6 +540,14 @@ right NUMBER.
 Set preferred location for Metalink resources. This has effect if multiple
 resources with same priority are available.
 
+@cindex xattr
+@item --xattr
+Enable use of file system's extended attributes to save the
+original URL and the Referer HTTP header value if used.
+
+Be aware that the URL might contain private information like
+access tokens or credentials.
+
 
 @cindex force html
 @item -F
@@ -855,7 +863,7 @@ files.  In such a case, wget could simply be invoked with this parameter to get
 a much cleaner output on the screen.
 
 This option will also force the progress bar to be printed to @file{stderr} when
-used alongside the @samp{--logfile} option.
+used alongside the @samp{--output-file} option.
 
 @item -N
 @itemx --timestamping
@@ -1394,10 +1402,10 @@ to the inability of server-side scripts to cope with the connections.
 @cindex cache
 @item --no-cache
 Disable server-side cache.  In this case, Wget will send the remote
-server an appropriate directive (@samp{Pragma: no-cache}) to get the
-file from the remote service, rather than returning the cached version.
-This is especially useful for retrieving and flushing out-of-date
-documents on proxy servers.
+server appropriate directives (@samp{Cache-Control: no-cache} and
+@samp{Pragma: no-cache}) to get the file from the remote service,
+rather than returning the cached version. This is especially useful
+for retrieving and flushing out-of-date documents on proxy servers.
 
 Caching is allowed by default.
 
@@ -1809,7 +1817,7 @@ Set the cipher list string. Typically this string sets the
 cipher suites and other SSL/TLS options that the user wish should be used, in a
 set order of preference (GnuTLS calls it 'priority string'). This string
 will be fed verbatim to the SSL/TLS engine (OpenSSL or GnuTLS) and hence
-its format and syntax is dependant on that. Wget will not process or manipulate it
+its format and syntax is dependent on that. Wget will not process or manipulate it
 in any way. Refer to the OpenSSL or GnuTLS documentation for more information.
 
 @cindex SSL certificate, check
@@ -1997,7 +2005,7 @@ the same time over the HSTS database. Before dumping the updated HSTS entries
 on the file, Wget will re-read it and merge the changes.
 
 Using a custom HSTS database and/or modifying an existing one is discouraged.
-For more information about the potential security threats arised from such practice,
+For more information about the potential security threats arose from such practice,
 see section 14 "Security Considerations" of RFC 6797, specially section 14.9
 "Creative Manipulation of HSTS Policy Store".
 @end table
@@ -4190,7 +4198,7 @@ from the debug log before sending it to the bug address.  The
 @code{-d} won't go out of its way to collect sensitive information,
 but the log @emph{will} contain a fairly complete transcript of Wget's
 communication with the server, which may include passwords and pieces
-of downloaded data.  Since the bug address is publically archived, you
+of downloaded data.  Since the bug address is publicly archived, you
 may assume that all bug reports are visible to the public.
 
 @item
@@ -4439,7 +4447,7 @@ Tomislav Petrović, Mario Mikočević---many bug reports and
 suggestions.
 
 @item
-François Pinard---many thorough bug reports and discussions.
+Françis Pinard---many thorough bug reports and discussions.
 
 @item
 Karl Eichwalder---lots of help with internationalization, Makefile

fuzz/Makefile.am

@@ -1,4 +1,4 @@
-AM_CFLAGS = $(WERROR_CFLAGS) $(WARN_CFLAGS) -Wno-unused-parameter -Wno-pedantic
+AM_CFLAGS = $(WERROR_CFLAGS) $(WARN_CFLAGS)
 AM_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_builddir)/lib -I$(top_srcdir)/lib \
  -DSRCDIR=\"$(abs_srcdir)\"
 LDADD = ../lib/libgnu.a \
@@ -6,7 +6,7 @@ LDADD = ../lib/libgnu.a \
  $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO) $(LIB_GETLOGIN) $(LIB_NANOSLEEP) $(LIB_POLL) \
  $(LIB_POSIX_SPAWN) $(LIB_PTHREAD_SIGMASK) $(LIB_SELECT) $(LIBICONV) $(LIBINTL) \
  $(LIBMULTITHREAD) $(LIBTHREAD) $(SERVENT_LIB) @INTL_MACOSX_LIBS@ \
- -ldl
+ $(FUZZ_LIBS)
 
 WGET_TESTS = \
  wget_css_fuzzer$(EXEEXT) \
@@ -16,7 +16,8 @@ WGET_TESTS = \
  wget_netrc_fuzzer$(EXEEXT) \
  wget_options_fuzzer$(EXEEXT) \
  wget_robots_fuzzer$(EXEEXT) \
- wget_url_fuzzer$(EXEEXT)
+ wget_url_fuzzer$(EXEEXT) \
+ wget_read_hunk_fuzzer$(EXEEXT)
 
 if FUZZING
   bin_PROGRAMS = $(WGET_TESTS)
@@ -60,6 +61,9 @@ wget_robots_fuzzer_LDADD = ../src/libunittest.a $(LDADD)
 wget_url_fuzzer_SOURCES = wget_url_fuzzer.c $(MAIN)
 wget_url_fuzzer_LDADD = ../src/libunittest.a $(LDADD)
 
+wget_read_hunk_fuzzer_SOURCES = wget_read_hunk_fuzzer.c $(MAIN)
+wget_read_hunk_fuzzer_LDADD = ../src/libunittest.a $(LDADD)
+
 #EXTRA_DIST = $(wildcard *.options) $(wildcard *.dict) \
 # $(wildcard *.in) $(wildcard *.repro)
 
@@ -74,7 +78,7 @@ clean-local:
 
 oss-fuzz:
 	if test "$$OUT" != ""; then \
-	XLIBS="-lpsl -lgnutls -lnettle -lhogweed -lidn2 -lunistring"; \
+	XLIBS="-lpsl -lgnutls -lhogweed -lnettle -lidn2 -lunistring"; \
 	for ccfile in wget*_fuzzer.c; do \
 	fuzzer=$$(basename $$ccfile .c); \
 	$$CXX $$CXXFLAGS -I$(top_srcdir)/src -I$(top_srcdir) -I$(top_srcdir)/lib \

fuzz/fuzzer.h

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -20,6 +20,16 @@
 #include <stddef.h> // size_t
 #include <stdint.h> // uint8_t
 
+#define CLOSE_STDERR \
+	int bak = dup(STDERR_FILENO); \
+	int fd = open("/dev/null", O_WRONLY); \
+	dup2(fd, STDERR_FILENO); \
+	close(fd);
+
+#define RESTORE_STDERR \
+	dup2(bak, STDERR_FILENO); \
+	close(bak);
+
 #ifdef __cplusplus
 extern "C"
 #endif

fuzz/glob_crash.c

@@ -1,4 +1,22 @@
 /*
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
+ *
+ * This file is part of GNU Wget.
+ *
+ * GNU Wget is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GNU Wget is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Wget.  If not, see <https://www.gnu.org/licenses/>.
+
+
  * Created 19.10.2017 by Tim Rühsen
  *
  * Call glob() using data from fuzzer crash file

fuzz/main.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -51,11 +51,12 @@ static int test_all_from(const char *dirname)
 			char fname[strlen(dirname) + strlen(dp->d_name) + 2];
 			snprintf(fname, sizeof(fname), "%s/%s", dirname, dp->d_name);
 
-                        struct file_memory *fmem;
+			struct file_memory *fmem;
 			if ((fmem = wget_read_file(fname))) {
 				printf("testing %ld bytes from '%s'\n", fmem->length, fname);
+				fflush(stdout);
 				LLVMFuzzerTestOneInput((uint8_t *)fmem->content, fmem->length);
-                                wget_read_file_free(fmem);
+				wget_read_file_free(fmem);
 			}
 		}
 		closedir(dirp);

fuzz/run-afl.sh

@@ -1,6 +1,6 @@
 #!/bin/sh -eu
 
-# Copyright(c) 2017 Free Software Foundation, Inc.
+# Copyright (c) 2017, 2019 Free Software Foundation, Inc.
 #
 # This file is part of GNU Wget.
 #

fuzz/run-clang.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 #
-# Copyright(c) 2017 Free Software Foundation, Inc.
+# Copyright (c) 2017, 2019 Free Software Foundation, Inc.
 #
 # This file is part of GNU Wget.
 #

fuzz/wget_cookie_fuzzer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -25,6 +25,8 @@
 #include <stdio.h>  // fmemopen
 #include <string.h>  // strncmp
 #include <stdlib.h>  // free
+#include <fcntl.h>  // open flags
+#include <unistd.h>  // close
 
 #include "wget.h"
 #undef fopen_wgetrc
@@ -60,15 +62,10 @@ FILE *fopen_wgetrc(const char *pathname, const char *mode)
 void exit_wget(int status)
 {
 }
-#else
-void exit(int status)
-{
-}
 #endif
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-	FILE *bak;
 	struct cookie_jar *cookie_jar;
 	char *set_cookie;
 
@@ -79,8 +76,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	memcpy(set_cookie, data, size);
 	set_cookie[size] = 0;
 
-	bak = stderr;
-	stderr = fopen("/dev/null", "w");
+	CLOSE_STDERR
 
 	cookie_jar = cookie_jar_new();
 	cookie_handle_set_cookie(cookie_jar, "x", 81, "p", set_cookie);
@@ -88,10 +84,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	cookie_handle_set_cookie(cookie_jar, "x", 80, "p/d/", set_cookie);
 	cookie_jar_delete(cookie_jar);
 
-	fclose(stderr);
-	stderr = bak;
+	RESTORE_STDERR
 
-        free(set_cookie);
+	free(set_cookie);
 
 	return 0;
 }

fuzz/wget_css_fuzzer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -67,16 +67,26 @@ FILE *fopen_wgetrc(const char *pathname, const char *mode)
 #endif
 }
 
+static int do_jump;
 static jmp_buf jmpbuf;
 #ifdef FUZZING
 void exit_wget(int status)
 {
 	longjmp(jmpbuf, 1);
 }
-#else
+#elif defined HAVE_DLFCN_H
+#include <dlfcn.h> // dlsym
+#ifndef RTLD_NEXT
+#define RTLD_NEXT RTLD_GLOBAL
+#endif
 void exit(int status)
 {
-	longjmp(jmpbuf, 1);
+	if (do_jump) {
+		longjmp(jmpbuf, 1);
+	} else {
+		void (*libc_exit)(int) = (void(*)(int)) dlsym (RTLD_NEXT, "exit");
+		libc_exit(status);
+	}
 }
 #endif
 
@@ -91,8 +101,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 		.document_file = NULL,
 	};
 
-	FILE *bak = stderr;
-	stderr = fopen("/dev/null", "w");
+	CLOSE_STDERR
+
+	do_jump = 1;
 
 	if (setjmp(jmpbuf))
 		goto done;
@@ -102,8 +113,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	free((void *) ctx.parent_base);
 
 done:
-	fclose(stderr);
-	stderr = bak;
+	do_jump = 0;
+
+	RESTORE_STDERR
 
 	return 0;
 }

fuzz/wget_ftpls_fuzzer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -59,30 +59,48 @@ FILE *fopen_wgetrc(const char *pathname, const char *mode)
 	return NULL;
 }
 
+static int do_jump;
+static jmp_buf jmpbuf;
 #ifdef FUZZING
 void exit_wget(int status)
 {
+	longjmp(jmpbuf, 1);
 }
-#else
+#elif defined HAVE_DLFCN_H
+#include <dlfcn.h> // dlsym
+#ifndef RTLD_NEXT
+#define RTLD_NEXT RTLD_GLOBAL
+#endif
 void exit(int status)
 {
+	if (do_jump) {
+		longjmp(jmpbuf, 1);
+	} else {
+		void (*libc_exit)(int) = (void(*)(int)) dlsym (RTLD_NEXT, "exit");
+		libc_exit(status);
+	}
 }
 #endif
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-	FILE *fp, *bak;
+#ifdef HAVE_FMEMOPEN
+	FILE *fp;
 	struct fileinfo *fi;
 
 	if (size > 4096) // same as max_len = ... in .options file
 		return 0;
 
-	bak = stderr;
-	stderr = fopen("/dev/null", "w");
-
 	fp = fmemopen((void *) data, size, "r");
 	if (!fp) return 0;
 
+	CLOSE_STDERR
+
+	do_jump = 1;
+
+	if (setjmp(jmpbuf))
+		goto done;
+
 	fi = ftp_parse_ls_fp(fp, ST_UNIX);
 	freefileinfo(fi);
 	rewind(fp);
@@ -96,12 +114,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	rewind(fp);
 
 	fi = ftp_parse_ls_fp(fp, ST_MACOS);
-	freefileinfo(fi);
 
+done:
+	freefileinfo(fi);
 	fclose(fp);
 
-	fclose(stderr);
-	stderr = bak;
+	do_jump = 0;
 
+	RESTORE_STDERR
+#endif
 	return 0;
 }

fuzz/wget_html_fuzzer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -72,23 +72,17 @@ FILE *fopen_wgetrc(const char *pathname, const char *mode)
 void exit_wget(int status)
 {
 }
-#else
-void exit(int status)
-{
-}
 #endif
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
    struct urlpos *urls;
    struct file_memory fm;
-   FILE *bak;
 
    if (size > 4096) // same as max_len = ... in .options file
 		return 0;
 
-	bak = stderr;
-	stderr = fopen("/dev/null", "w");
+	CLOSE_STDERR
 
    fm.content = (char *) data;
    fm.length = size;
@@ -97,8 +91,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
    urls = get_urls_html_fm("xxx", &fm, "https://x.y", NULL, NULL);
 	free_urlpos(urls);
 
-	fclose(stderr);
-	stderr = bak;
+	RESTORE_STDERR
 
 	return 0;
 }

fuzz/wget_netrc_fuzzer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -25,6 +25,8 @@
 #include <string.h>  // strncmp
 #include <stdlib.h>  // free
 #include <setjmp.h> // longjmp, setjmp
+#include <fcntl.h>  // open flags
+#include <unistd.h>  // close
 
 #include "wget.h"
 #undef fopen_wgetrc
@@ -56,31 +58,49 @@ FILE *fopen_wgetrc(const char *pathname, const char *mode)
 	return NULL;
 }
 
+static int do_jump;
+static jmp_buf jmpbuf;
 #ifdef FUZZING
 void exit_wget(int status)
 {
+	longjmp(jmpbuf, 1);
 }
-#else
+#elif defined HAVE_DLFCN_H
+#include <dlfcn.h> // dlsym
+#ifndef RTLD_NEXT
+#define RTLD_NEXT RTLD_GLOBAL
+#endif
 void exit(int status)
 {
+	if (do_jump) {
+		longjmp(jmpbuf, 1);
+	} else {
+		void (*libc_exit)(int) = (void(*)(int)) dlsym (RTLD_NEXT, "exit");
+		libc_exit(status);
+	}
 }
 #endif
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
-	FILE *fp, *bak;
+#ifdef HAVE_FMEMOPEN
+	FILE *fp;
 	struct fileinfo *fi;
-        const char *user = NULL, *pw = NULL;
+	const char *user = NULL, *pw = NULL;
 
 	if (size > 4096) // same as max_len = ... in .options file
 		return 0;
 
-	bak = stderr;
-	stderr = fopen("/dev/null", "w");
-
 	fp = fmemopen((void *) data, size, "r");
 	if (!fp) return 0;
 
+	CLOSE_STDERR
+
+	do_jump = 1;
+
+	if (setjmp(jmpbuf))
+		goto done;
+
 	opt.netrc = 1;
 
 	user = NULL; // get first entry
@@ -89,12 +109,16 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 
 	user = "u"; // get entry for user 'u'
 	search_netrc("x", &user, &pw, 1, fp);
+
+done:
 	netrc_cleanup();
 
 	fclose(fp);
 
-	fclose(stderr);
-	stderr = bak;
+	do_jump = 0;
 
+	RESTORE_STDERR
+
+#endif
 	return 0;
 }

fuzz/wget_options_fuzzer.c

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017-2018 Free Software Foundation, Inc.
+ * Copyright (c) 2017-2019 Free Software Foundation, Inc.
  *
  * This file is part of GNU Wget.
  *
@@ -60,16 +60,26 @@ FILE *fopen_wgetrc(const char *pathname, const char *mode)
 #endif
 }
 
+static int do_jump;
 static jmp_buf jmpbuf;
 #ifdef FUZZING
 void exit_wget(int status)
 {
 	longjmp(jmpbuf, 1);
 }
-#else
+#elif defined HAVE_DLFCN_H
+#include <dlfcn.h> // dlsym
+#ifndef RTLD_NEXT
+#define RTLD_NEXT RTLD_GLOBAL
+#endif
 void exit(int status)
 {
-	longjmp(jmpbuf, 1);
+	if (do_jump) {
+		longjmp(jmpbuf, 1);
+	} else {
+		void (*libc_exit)(int) = (void(*)(int)) dlsym (RTLD_NEXT, "exit");
+		libc_exit(status);
+	}
 }
 #endif
 
@@ -83,8 +93,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	g_data = data;
 	g_size = size;
 
-	FILE *bak = stderr;
-	stderr = fopen("/dev/null", "w");
+	CLOSE_STDERR
+
+	do_jump = 1;
 
 	if (setjmp(jmpbuf))
 		goto done;
@@ -94,8 +105,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 done:
 	cleanup();
 
-	fclose(stderr);
-	stderr = bak;
+	do_jump = 0;
+
+	RESTORE_STDERR
 
 	return 0;
 }

fuzz/wget_read_hunk_fuzzer.c

@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2019 Free Software Foundation, Inc.
+ *
+ * This file is part of GNU Wget.
+ *
+ * GNU Wget is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GNU Wget is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Wget.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#include <sys/types.h>
+#include <stdint.h> // uint8_t
+#include <stdio.h>  // fmemopen
+#include <string.h>  // strncmp
+#include <stdlib.h>  // free
+#include <unistd.h>  // close
+#include <fcntl.h>  // open flags
+#include <unistd.h>  // close
+#include <unistd.h>  // close
+
+#include "wget.h"
+#include "connect.h"
+#undef fopen_wgetrc
+
+#ifdef __cplusplus
+  extern "C" {
+#endif
+  #include "retr.h"
+
+  // declarations for wget internal functions
+  int main_wget(int argc, const char **argv);
+  void cleanup(void);
+//  FILE *fopen_wget(const char *pathname, const char *mode);
+//  FILE *fopen_wgetrc(const char *pathname, const char *mode);
+  void exit_wget(int status);
+#ifdef __cplusplus
+  }
+#endif
+
+#include "fuzzer.h"
+
+FILE *fopen_wget(const char *pathname, const char *mode)
+{
+	return fopen("/dev/null", mode);
+}
+
+FILE *fopen_wgetrc(const char *pathname, const char *mode)
+{
+	return NULL;
+}
+
+#ifdef FUZZING
+void exit_wget(int status)
+{
+}
+#endif
+
+static const uint8_t *g_data;
+static size_t g_size, g_read;
+
+struct my_context {
+	int peeklen;
+	char peekbuf[512];
+};
+
+static int my_peek (int fd _GL_UNUSED, char *buf, int bufsize, void *arg)
+{
+	if (g_read < g_size) {
+		struct my_context *ctx = (struct my_context *) arg;
+		int n = rand() % (g_size - g_read);
+		if (n > bufsize)
+			n = bufsize;
+		if (n > sizeof(ctx->peekbuf))
+			n = sizeof(ctx->peekbuf);
+		memcpy(buf, g_data + g_read, n);
+		memcpy(ctx->peekbuf, g_data + g_read, n);
+		g_read += n;
+		ctx->peeklen=n;
+		return n;
+	}
+	return 0;
+}
+static int my_read (int fd _GL_UNUSED, char *buf, int bufsize, void *arg)
+{
+	struct my_context *ctx = (struct my_context *) arg;
+
+	if (ctx->peeklen) {
+      /* If we have any peek data, simply return that. */
+      int copysize = MIN (bufsize, ctx->peeklen);
+      memcpy (buf, ctx->peekbuf, copysize);
+      ctx->peeklen -= copysize;
+      if (ctx->peeklen)
+        memmove (ctx->peekbuf, ctx->peekbuf + copysize, ctx->peeklen);
+
+      return copysize;
+	}
+
+	if (g_read < g_size) {
+		int n = rand() % (g_size - g_read);
+		if (n > bufsize)
+			n = bufsize;
+		memcpy(buf, g_data + g_read, n);
+		g_read += n;
+		return n;
+	}
+
+	return 0;
+}
+static int my_write (int fd _GL_UNUSED, char *buf _GL_UNUSED, int bufsize, void *arg _GL_UNUSED)
+{
+	return bufsize;
+}
+static int my_poll (int fd _GL_UNUSED, double timeout _GL_UNUSED, int wait_for _GL_UNUSED, void *arg)
+{
+	struct my_context *ctx = (struct my_context *) arg;
+
+   return ctx->peeklen || g_read < g_size;
+}
+static const char *my_errstr (int fd _GL_UNUSED, void *arg _GL_UNUSED)
+{
+	return "Success";
+}
+static void my_close (int fd _GL_UNUSED, void *arg _GL_UNUSED)
+{
+}
+
+static struct transport_implementation my_transport =
+{
+  my_read, my_write, my_poll,
+  my_peek, my_errstr, my_close
+};
+
+/* copied from wget's http.c */
+static const char *
+response_head_terminator (const char *start, const char *peeked, int peeklen)
+{
+  const char *p, *end;
+
+  /* If at first peek, verify whether HUNK starts with "HTTP".  If
+     not, this is a HTTP/0.9 request and we must bail out without
+     reading anything.  */
+  if (start == peeked && 0 != memcmp (start, "HTTP", MIN (peeklen, 4)))
+    return start;
+
+  /* Look for "\n[\r]\n", and return the following position if found.
+     Start two chars before the current to cover the possibility that
+     part of the terminator (e.g. "\n\r") arrived in the previous
+     batch.  */
+  p = peeked - start < 2 ? start : peeked - 2;
+  end = peeked + peeklen;
+
+  /* Check for \n\r\n or \n\n anywhere in [p, end-2). */
+  for (; p < end - 2; p++)
+    if (*p == '\n')
+      {
+        if (p[1] == '\r' && p[2] == '\n')
+          return p + 3;
+        else if (p[1] == '\n')
+          return p + 2;
+      }
+  /* p==end-2: check for \n\n directly preceding END. */
+  if (peeklen >= 2 && p[0] == '\n' && p[1] == '\n')
+    return p + 2;
+
+  return NULL;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+	char *hunk;
+
+	if (size > 4096) // same as max_len = ... in .options file
+		return 0;
+
+//	CLOSE_STDERR
+
+	g_data = data;
+	g_size = size;
+	g_read = 0;
+
+	struct my_context *ctx = calloc(1, sizeof(struct my_context));
+	fd_register_transport(99, &my_transport, ctx);
+
+	while ((hunk = fd_read_hunk(99, response_head_terminator, 512, 65536)))
+		free(hunk);
+
+   connect_cleanup();
+	free(ctx);
+
+//	RESTORE_STDERR
+
+	return 0;
+}

fuzz/wget_read_hunk_fuzzer.in/0223d9b49784338193078cc1ce4937d1ae0843ee

@@ -0,0 +1,3 @@
+HTTP
+
+'

fuzz/wget_read_hunk_fuzzer.in/03324eed2a844c2c712c3a736395dd996ff5da3a

@@ -0,0 +1,2 @@
+HTTP
+<EFBFBD>

fuzz/wget_read_hunk_fuzzer.in/04a822e65bb1e8bb2c19f09b187004bb3f8b1650

@@ -0,0 +1,5 @@
+HTTPHH~:þ±
+
+QHTTPHH~:þ±
+
+QTT)÷)÷ÿ

fuzz/wget_read_hunk_fuzzer.in/06f4cdeb821523a02351795dc305b330196dd95e

@@ -0,0 +1,3 @@
+H
+
+<EFBFBD>

fuzz/wget_read_hunk_fuzzer.in/09543c8214d579ab15c96c09267a61c8aaf8c9ef

@@ -0,0 +1 @@
+HTTP<EFBFBD><EFBFBD>)<29>M<EFBFBD><4D>

fuzz/wget_read_hunk_fuzzer.in/0e3db6c438a6d2126102efdfdd9d7485d2a985d1

@@ -0,0 +1 @@
+HTTP<EFBFBD><EFBFBD><EFBFBD><EFBFBD>)<29><><EFBFBD><EFBFBD><EFBFBD>HTTP<54><50>

fuzz/wget_read_hunk_fuzzer.in/15c59fa0420280649a7844b6298b961865efdd76

@@ -0,0 +1 @@
+HHT

fuzz/wget_read_hunk_fuzzer.in/200e3883469d4d99b28d29167bf9a48d33efcf5b

@@ -0,0 +1 @@
+H!T'<27>

fuzz/wget_read_hunk_fuzzer.in/223e49ca36b549c406edaee2b04a68bdb2baeb08

@@ -0,0 +1 @@
+HTT~~~

fuzz/wget_read_hunk_fuzzer.in/25db0570cd4eeeafb131bd3698f4c544f40017e1

@@ -0,0 +1,3 @@
+HTTPH
+<EFBFBD>
+~

fuzz/wget_read_hunk_fuzzer.in/2a4630b84731f6f7ae444a86c0f1026faca44d82

@@ -0,0 +1 @@
+HHH;

fuzz/wget_read_hunk_fuzzer.in/2bcb1b518040217690b627761d4f5c904bf437d8

@@ -0,0 +1,2 @@
+HTTH
+:

fuzz/wget_read_hunk_fuzzer.in/35cc83ddacfa942592c1f8ca25f2889772739b5c

@@ -0,0 +1 @@
+'!<21><>!

fuzz/wget_read_hunk_fuzzer.in/3da72e604871236487499a2669c0edd6e462d543

@@ -0,0 +1 @@
+HTTPHHH

fuzz/wget_read_hunk_fuzzer.in/40a50e5bfeaf182fb9def6c12891c11f4259f81b

@@ -0,0 +1 @@
+HIHHT

fuzz/wget_read_hunk_fuzzer.in/4c93a532486c2f5736dcba17309daf0f66315821

@@ -0,0 +1,4 @@
+HTP
+
+
+

fuzz/wget_read_hunk_fuzzer.in/4dbe2f639ae782ad317f578f150d8960d5d45798

@@ -0,0 +1,3 @@
+H
+
+

fuzz/wget_read_hunk_fuzzer.in/63451c9bbffc0dc2d1b6b2cd83b039ad73a4aad4

@@ -0,0 +1,3 @@
+HTTP<EFBFBD>
+
+<EFBFBD>

fuzz/wget_read_hunk_fuzzer.in/64c38cf29a15ea03a3844b4bbabeaa53f7a42d28

@@ -0,0 +1 @@
+HT

fuzz/wget_read_hunk_fuzzer.in/6837896608e80848e5b6b1df5da5e05d484a616a

@@ -0,0 +1 @@
+HTH

fuzz/wget_read_hunk_fuzzer.in/6d528a0c51d29738948554967691cdc796957122

@@ -0,0 +1 @@
+HHHT