Merge remote-tracking branch 'origin/master' into parallel-wget

Conflicts:
	src/http.c
	testenv/Makefile.am
	testenv/Test--https.py
	testenv/Test--spider-r.py
	testenv/Test-Content-disposition-2.py
	testenv/Test-Content-disposition.py
	testenv/Test-Head.py
	testenv/Test-O.py
	testenv/Test-Parallel-Proto.py
	testenv/Test-Post.py
	testenv/Test-Proto.py
	testenv/Test-auth-basic-fail.py
	testenv/Test-auth-basic.py
	testenv/Test-auth-both.py
	testenv/Test-auth-digest.py
	testenv/Test-auth-no-challenge-url.py
	testenv/Test-auth-no-challenge.py
	testenv/Test-auth-retcode.py
	testenv/Test-auth-with-content-disposition.py
	testenv/Test-c-full.py
	testenv/Test-cookie-401.py
	testenv/Test-cookie-domain-mismatch.py
	testenv/Test-cookie-expires.py
	testenv/Test-cookie.py
	testenv/conf/__init__.py
	testenv/conf/authentication.py
	testenv/conf/expect_header.py
	testenv/conf/expected_files.py
	testenv/conf/expected_ret_code.py
	testenv/conf/files_crawled.py
	testenv/conf/hook_sample.py
	testenv/conf/local_files.py
	testenv/conf/reject_header.py
	testenv/conf/response.py
	testenv/conf/send_header.py
	testenv/conf/server_files.py
	testenv/conf/urls.py
	testenv/conf/wget_commands.py
	testenv/misc/colour_terminal.py
	testenv/server/http/http_server.py
	testenv/test/base_test.py
	testenv/test/http_test.py

ChangeLog

@@ -73,6 +73,20 @@
 
 	* gnulib: add git submodule.
 
+2013-10-23  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Add --race-detection.
+	* configure.ac: Remove redundant --enable-threads option.
+        Add Metalink to configure summary
+
+2013-07-02  Darshit Shah  <darnir@gmail.com>
+
+	* README.checkout: Add dependency on libmetalink.
+
+2013-06-23  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* configure.ac: By default disable threads and metalink support.
+
 2013-09-13  Tim Ruehsen  <tim.ruehsen@gmx.de>
 
 	* configure.ac: added a summary of build options
@@ -80,6 +94,11 @@
 	  removed the unconditionally adding of libz with --with-ssl
 	  removed -lgcrypt and -lgpg-error for gnutls
 
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Add testenv/Makefile to	AC_CONFIG_FILES.
+	* Makefile.am: Add testenv to SUBDIRS
+
 2013-07-23  Tim Ruehsen  <tim.ruehsen@gmx.de>
 
 	* configure.ac: Remove AM_CONDITIONAL HAVE_NETTLE.
@@ -110,6 +129,62 @@
 
 	* po/POTFILES.in: Add more files.
 
+2012-11-28  Domenico Chierico <spaghetty@gmail.com>
+
+	* bootstrap: update from upstream.
+	* configure.ac: Change tests for iri/threads/metalink detection.
+
+2012-08-03  Ilim Ugur  <ilimugur@gmail.com>
+
+	* bootstrap.conf : Add crypto/sha256 to the list of Gnulib modules used
+	by this package.
+
+2012-08-03  Ilim Ugur  <ilimugur@gmail.com>
+
+	* bootstrap.conf : Add crypto/sha1 to the list of Gnulib modules used
+	by this package.
+
+2012-07-29  Ilim Ugur  <ilimugur@gmail.com>
+
+	* configure.ac : Check the availability of pthread library, before
+	enabling thread support.
+
+2012-07-27  Ilim Ugur  <ilimugur@gmail.com>
+
+	* configure.ac : Update libmetalink check so that metalink support is
+	not taken into consideration for compilation, if thread support is not
+	desired.
+
+2012-07-23  Ilim Ugur  <ilimugur@gmail.com>
+
+	* configure.ac : Make Metalink check in configure.ac automatic by
+	assigning auto to ENABLE_METALINK in AC_ARG_ENABLE macro. Update the
+	libmetalink check accordingly.
+
+2012-07-23  Micah Cowan  <micah@cowan.name>
+
+	* bootstrap : Replace the bootstrap with the latest bootstrap from
+	Gnulib.
+	* bootstrap.conf : Remove the line "$build_aux/missing".
+
+2012-07-22  Ilim Ugur  <ilimugur@gmail.com>
+
+	* configure.ac : Correct the faulty assignment by assigning to CFLAGS
+	instead of LIBS. Add AM_CONDITIONAL macro to assign a value to a
+	variable(i.e. METALINK_IS_ENABLED) to be used in src/Metalink.am.
+
+2012-07-21  Ilim Ugur  <ilimugur@gmail.com>
+
+	* configure.ac : Update the check for libmetalink. The new version tries
+	using pkg-config.
+
+2012-07-20  Ilim Ugur  <ilimugur@gmail.com>
+
+	* configure.ac : Add AC_ARG_ENABLE macros to be able to check for the
+	essential libraries for multi-threaded and metalink downloading support.
+	Add checks for libmetalink and make essential additions to LIBS and
+	CFLAGS if found, as well as prompting messages if it is not found.
+
 2012-07-08  Giuseppe Scrivano  <gscrivano@gnu.org>
 
 	* bootstrap: Update from gnulib.

README.checkout

@@ -4,8 +4,8 @@ Compiling From Repository Sources
    does not contain automatically-generated files, even when these are
    normally present in the distribution tarballs. Therefore, to build GNU
    Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, libidn, libiconv
-   and libpsl are not absolutely required):
+   more of the following (note that gettext, OpenSSL, libidn, libiconv,
+   libpsl and libmetalink are not absolutely required):
 
      * [20]autoconf (currently, GNU Wget requires version 2.61). This is
        needed to generate the configure script from configure.in. This is
@@ -68,6 +68,8 @@ Compiling From Repository Sources
 
      * [34]git is used to fetch gnulib files trough the bootstrap.sh script.
 
+     * [45]libmetalink is required for compiling with Metalink support.
+
      * [35]libpsl is required for using a public suffix list to check for valid
        cookie domains.
 
@@ -142,3 +144,4 @@ References
   42. http://moinmo.in/GPL
   43. http://validator.w3.org/check?uri=referer
   44. http://wget.addictivecode.org/WikiLicense
+  45. https://launchpad.net/libmetalink

bootstrap.conf

@@ -55,6 +55,7 @@ mkstemp
 mkostemp
 crypto/md5
 crypto/sha1
+crypto/sha256
 pipe
 quote
 quotearg

configure.ac

@@ -100,6 +100,18 @@ ENABLE_DEBUG=$enableval, ENABLE_DEBUG=yes)
 test x"${ENABLE_DEBUG}" = xyes && AC_DEFINE([ENABLE_DEBUG], 1,
    [Define if you want the debug output support compiled in.])
 
+AC_ARG_ENABLE(metalink,
+[  --disable-metalink      disable support for metalink file format],
+ENABLE_METALINK=$enableval, ENABLE_METALINK=no)
+test x"${ENABLE_METALINK}" = xyes && AC_DEFINE([ENABLE_METALINK], 1,
+   [Define if you want the metalink file format support compiled in.])
+
+AC_ARG_ENABLE(race-detection,
+[  --enable-race-detection enable support for detecting race conditions],
+ENABLE_RACES=$enableval, ENABLE_RACES=no)
+test x"${ENABLE_RACES}" = xyes && AC_DEFINE([ENABLE_RACES], 1,
+   [Define if you want runtime Race Detection enabled.])
+
 dnl
 dnl Find the compiler
 dnl
@@ -365,6 +377,74 @@ else
   fi
 fi
 
+AS_IF([test x"$ENABLE_THREADS" != xno],[
+  AC_CHECK_LIB(pthread, pthread_create, [
+dnl    AC_CHECK_LIB(semaphore, sem_init, [
+      threads=yes
+      LIBS="${LIBS} -lpthread"
+      AC_DEFINE([ENABLE_THREADS], 1,
+        [Define if you want the thread support compiled in.])
+      AC_MSG_NOTICE([compiling with support for threads])
+dnl    ],[
+dnl      threads=no
+dnl      if test x"$ENABLE_THREADS" == xyes
+dnl      then
+dnl        AC_MSG_ERROR([Could not find the sem_init function in semaphore.])
+dnl      fi
+dnl    ])
+  ],[
+      threads=no
+      if test x"$ENABLE_THREADS" == xyes
+      then
+        AC_MSG_ERROR([Could not find the pthread_create function in pthread.])
+      fi
+  ])
+])
+
+AS_IF([test x"$ENABLE_RACES" != xno],[
+   CFLAGS="$CFLAGS -fsanitize=thread -fPIE";
+   LDFLAGS="$LDFLAGS -fsanitize=thread -pie";
+])
+
+AS_IF([test x"$ENABLE_METALINK" != xno],[
+  dnl If metalink support is requested
+  PKG_CHECK_MODULES(LIBMETALINK, libmetalink, [
+    dnl If metalink module is found
+    LIBS="$LIBMETALINK_LIBS $LIBS"
+    CFLAGS="$LIBMETALINK_CFLAGS $CFLAGS"
+    AC_DEFINE([ENABLE_METALINK], 1,
+     [Define if you want the metalink support compiled in.])
+
+    AC_CHECK_LIB(metalink, metalink_parse_file, [
+      if test x"$DISABLE_THREADS" == xyes
+      then
+        AC_MSG_ERROR([Metalink support depends on threads, which are requested to be disabled.])
+      elif test "X$threads" != "Xyes"
+      then
+        AC_MSG_ERROR([Metalink support depends on threads, support for which could not be enabled.])
+      else
+        metalink=yes
+        AC_MSG_NOTICE([compiling in support for metalink])
+      fi
+    ], [
+      if test x"$ENABLE_METALINK" == xyes
+      then
+        AC_MSG_ERROR([Could not find the metalink_parse_file function in libmetalink.])
+      fi
+    ])
+  ], [
+    dnl If metalink module is not found
+    if test x"$ENABLE_METALINK" == xyes
+    then
+      AC_MSG_ERROR([libmetalink not found])
+    fi
+  ])
+],[
+  dnl If metalink support is not requested
+  metalink=no
+  AC_MSG_NOTICE([compiling without support for metalink])
+])
+
 
 dnl **********************************************************************
 dnl Checks for IPv6
@@ -571,8 +651,10 @@ AC_CHECK_HEADER(pcre.h,
 
 
 dnl Needed by src/Makefile.am
-AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
-
+AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" == "Xyes"])
+AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$metalink" == "Xyes"])
+AM_CONDITIONAL([THREADS_ARE_ENABLED], [test "X$threads" == "Xyes"])
+AM_CONDITIONAL([RACE_CHECKING_IS_ENABLED], [test "X$race-detection" == "Xyes"])
 
 dnl
 dnl Create output
@@ -598,5 +680,7 @@ AC_MSG_NOTICE([Summary of build options:
   Digest:            $ENABLE_DIGEST
   NTLM:              $ENABLE_NTLM
   OPIE:              $ENABLE_OPIE
+  Metalink:          $ENABLE_METALINK
   Debugging:         $ENABLE_DEBUG
+  Race Detection:    $ENABLE_RACES
 ])

doc/ChangeLog

@@ -65,6 +65,11 @@
 
 	* wget.texi: Document --regex-type and --preserve-permissions.
 
+2013-07-04  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Download Options): Use '@item' instead of '@itemx'.
+	Reported by: Darshit Shah  <darnir@gmail.com>.
+
 2013-06-17  Dave Reisner  <dreisner@archlinux.org> (tiny change)
 
 	* texi2pod.pl: Fix formatting error that causes build to fail with
@@ -112,6 +117,10 @@
 
         * doc/wget.texi: remove -nv from --report-speed
 
+2012-08-14  Ilim Ugur <ilimugur@gmail.com>
+
+	* wget.texi: Document --metalink-file and --retries.
+
 2012-08-04  mancha <mancha@mac.hush.com> (tiny change)
 
 	* wget.texi: Export ENVIRONMENT to the man page.

doc/wget.texi

@@ -533,6 +533,22 @@ would be resolved to @samp{http://foo/baz/b.html}.
 @cindex specify config 
 @item --config=@var{FILE}
 Specify the location of a startup file you wish to use.
+
+@cindex metalink 
+@item --metalink=@var{FILE}
+Read @sc{url}s from a local or external metalink @var{file}.  If @samp{-} is
+specified as @var{file}, @sc{url}s are read from the standard input.  
+(Use @samp{./-} to read from a file literally named @samp{-}.)
+
+If this function is used, no @sc{url}s need be present on the command
+line.
+
+If the @var{file} is an external one, the document will be automatically
+downloaded and will not be kept even after wget terminates.
+
+See option @samp{--jobs} to enable multi-threaded downloads from metalink
+files, option @samp{--retries} to set a number of retries for downloading a @var{file}
+in case of failure.
 @end table
 
 @node Download Options, Directory Options, Logging and Input File Options, Invoking
@@ -558,6 +574,11 @@ infinite retrying.  The default is to retry 20 times, with the exception
 of fatal errors like ``connection refused'' or ``not found'' (404),
 which are not retried.
 
+@item --retries=@var{number}
+Set number of retries for a file specified in a metalink file to @var{number}. 
+Specify 0 or @samp{inf} for no retrying in case of failed attempt.  The default 
+is to retry 1 time.
+
 @item -O @var{file}
 @itemx --output-document=@var{file}
 The documents will not be written to the appropriate files, but all
@@ -1145,6 +1166,14 @@ command line.
 Force Wget to unlink file instead of clobbering existing file. This
 option is useful for downloading to the directory with hardlinks.
 
+@cindex jobs
+@item --jobs=@var{number}
+
+Specify the @var{number} of threads to use. Unless specified, the number of 
+threads used is 1.
+
+Currently this option works only for recursive downloading and when specified with 
+option @samp{--metalink}.
 @end table
 
 @node Directory Options, HTTP Options, Download Options, Invoking

src/ChangeLog

@@ -1,3 +1,7 @@
+2014-07-21  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (wget_SOURCES): Remove space-tab indentation.
+
 2014-07-23  Darshit Shah  <darnir@gmail.com>
 
 	* http.c (gethttp): Fix a memory leak when retrying authorization
@@ -304,15 +308,6 @@
 	* ftp.c (getftp): Send filename of the local file so that it can be printed
 	with the progress bar.
 
-2013-11-02  Giuseppe Scrivano  <gscrivan@redhat.com>
-
-	* progress.c (struct progress_implementation): New method draw.
-	(bar_update): Split into...
-	(bar_draw): ...this new function.
-	(dot_update): Split into...
-	(dot_draw): New function.
-	(progress_update): Also invoke draw on `current_impl'.
-
 2014-04-22  Tim Ruehsen  <tim.ruehsen@gmx.de>
 
 	* http.c (gethttp): Fix 204 response handling
@@ -392,6 +387,80 @@
 
 	* gnutls.c (ssl_connect_wget): Fix connect timeout failure
 
+2013-11-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* options.h (struct options) [!ENABLE_THREADS]: Define jobs.
+
+	* progress.c (progress_create): Do not set `current_progress'.
+	(progress_finish): Do not reset `current_progress' when the progress bar
+	to remove is the first one in the list.
+
+2013-11-03  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* retr.h (fd_read_body): Accept an additional argument `url'.
+	* retr.c (fd_read_body): Likewise.
+	(fd_read_body): Pass the url to the progress bar.
+	* ftp.c (getftp): Adjust call to fd_read_body.
+	* http.c (read_response_body): Likewise.
+	* progress.h (progress_create): Add new argument `url'.
+	* progress.c (create_image): Show url near the progress bar.
+	(process_create): Add new argument `url'.
+
+2013-11-02  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* progress.c (struct progress_header): New struct
+	(progress_update): Dispatch the draw call to a progress struct.
+	include <pthread.h>.
+	(dot_draw): Add argument `force'.
+	(bar_draw): Likewise.
+	(dot_draw): Likewise.
+
+	* progress.c (struct progress_implementation): New method draw.
+	(bar_update): Split into...
+	(bar_draw): ...this new function.
+	(dot_update): Split into...
+	(dot_draw): New function.
+	(progress_update): Also invoke draw on `current_impl'.
+
+	* http.c (CLOSE_INVALIDATE): Set socket fd to be invalid.
+	(register_persistent): Add control and avoid to add the same fd
+	multiple times.
+
+2013-10-30  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* recur.c (retrieve_tree): Correctly initialize `free_threads' to
+	N_THREADS.
+
+2013-10-13  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* multi.h (struct s_thread_ctx) [!ENABLE_THREADS]: Define "retr_sem"
+	* recur.c (THREAD_JOIN): Define.
+	(retrieve_tree) [!ENABLE_THREADS]: Refactor to use similar code
+	paths to the threads enabled version.
+	(SEM_INIT): Likewise.
+	(SEM_WAIT): Likewise.
+
+	* retr.c (retrieve_from_file): Internationalize messages.
+
+	* retr.c (retrieve_from_file): Follow GNU Coding style.
+
+2013-10-06  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* http.c (gethttp): Unregister a connection while it is used.
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* wget.h (option_data[]): Use wgint instead of int.
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* main.c (option_data[]): Change option to metalink-file in accordance
+	with the information in the documentation
+
+2013-07-12  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* utils.c (run_with_timeout): abort when there are more threads.
+
 2013-11-02  Giuseppe Scrivano  <gscrivan@redhat.com>
 
 	* http.c (gethttp): Increase max header value length to 512.
@@ -564,6 +633,37 @@
         * html-url.c: Define TAG_TD, TAG_TH, TAG_VIDEO, TAG_AUDIO, TAG_SOURCE.
 	(tag_url_attributes, known_tags): Likewise.
 
+2013-06-23  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (PCONN_LOCK) [!ENABLE_THREADS]: Define.
+	(PCONN_UNLOCK) [!ENABLE_THREADS]: Likewise.
+
+	* http.c (gethttp): Lock access to `connect_to_host'.
+
+2013-06-03  Jochen Hein  <jochen@jochen.org> (tiny change)
+
+	* main.c (main): Remove double empty space from string.
+
+2013-05-22  Bykov Aleksey <gnfalex@rambler.ru>
+
+	* multi.c (merge_temp_files): Changed fopen mode from text to binary.
+	* metalink.c (verify_file_hash): Likewise.
+
+2013-05-21  Ray Satiro  <raysatiro@yahoo.com>
+
+	* http.c (create_authorization_line): Fix syntax error in calls to NTLM
+	functions.
+	* url.c (url_file_name): Use MAX_PATH in Windows.
+
+2013-05-19  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* Makefile.am (EXTRA_DIST): Add iri.c multi.c multi.h metalink.c metalink.h.
+
+2013-04-21  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (gethttp): Do not register the persistent connection too
+	early.
+
 2013-06-22  Ángel González <keisial@gmail.com>
 
 	* init.c (cmd_string_uppercase): Rewrite function.
@@ -574,10 +674,6 @@
 	avoid access to uninitialized values by inet_ntop().
 	* ftp.c (ftp_loop_internal): fix segfault caused by warc_tmp NULL pointer.
 
-2013-05-21  Ray Satiro  <raysatiro@yahoo.com>
-
-	* url.c (url_file_name): Use MAX_PATH in Windows.
-
 2013-06-13  Darshit Shah  <darnir@gmail.com>
 
 	* http.c (gethttp): Follow RFC 2616 and httpbis specifications when
@@ -830,6 +926,651 @@
 	* connect.c (connect_to_ip) [ENABLE_IPV6]: Attempt to use IPv6.
 	* http.c (gethttp): Likewise.
 
+2013-02-07  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c (parse_metalink): Rename `mlink' to 'm_link'.
+
+2012-12-09  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* recur.c (retrieve_tree): Do not attempt to re-download a file if
+	`next_url' is known.
+
+	* test.c: Include "wget.h".
+
+2012-12-04  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* metalink.c: Include "wget.h" as first file.
+
+2012-12-02  Domenico Chierico <spaghetty@gmail.com>
+
+	* Makefile.am (LDADD): Remove $(LIBS).
+
+2012-12-02  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* recur.c (retrieve_tree): New variable `next_i'.
+
+	* http.c (gethttp): Check for pconn before access it.
+
+2012-12-01  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* convert.c (lock_convert_mutex): New function.
+	(THREAD_SAFE): Use `lock_convert_mutex'.
+	(THREAD_SAFE_VOID): Likewise.
+
+2012-11-28  Domenico Chierico <spaghetty@gmail.com>
+
+	* Makefile.am (THREAD_LIB): Add -lpthread.
+	* convert.h (register_download): Add definition.
+	(convert_all_links): Likewise.
+	(register_delete_file): Likewise.
+	(register_redirection): Likewise.
+	(register_css): Likewise.
+	(register_html): Likewise.
+	* convert.c: Remove static from register_download, convert_all_links,
+	register_delete_file, register_redirection, register_css,
+	register_html.
+	* retr.c: Call register_html and register_css properly.
+
+2012-11-29  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* http.c (pconn_lock): New function.
+	(PCONN_LOCK): Use pconn_lock.
+	(persistent_available_p): Change signature.
+	(get_persistent): Change calls to persistent_available_p.
+
+2012-10-07  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.h (enum): Add THREADS_ERR and SEM_ERR.
+	* recur.c (retrieve_tree): Use `pthread_join' to free a terminated
+	thread resources.
+
+2012-09-18  Ilim Ugur  <ilimugur@gmail.com>
+	* metalink.c (elect_resources): Rewrite function definition, correcting
+	faulty logic in branching statements.
+	* metalink.c (elect_checksums): Rewrite function definition, correcting
+	faulty logic in branching statements.
+	* metalink.c (delete_mlink): Rename variables tempfile, tempres,
+	tempcsum, temphash as file_temp, res_temp, csum_temp and phash_temp.
+	* retr.c : Take functions elect_resources and elect_checksums out of
+	comment brackets.
+
+2012-09-17  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c : Add comments for functions delete_mlink, elect_checksums,
+	elect_resources and parse_metalink.
+	* metalink.h : Add prototypes of elect_resources and elect_checksums.
+	* multi.c : Remove static variable temp.
+	* options.h : Remove ifdefs that contain metalink_file member, due to
+	the opt.metalink_file check made in an if statement in main.c.
+	* retr.c (retrieve_from_file): Use functions elect_checksums and
+	elect_resources to filter the parsed information during metalink
+	downloads.
+
+2012-08-22  Ilim Ugur  <ilimugur@gmail.com>
+
+	* main.c : Correct malloc size for temp_option.
+
+2012-08-20  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c (verify_file_hash): Change type of function parameter
+	checksums. Add a variable checksum of type mlink_checksum. Use checksum
+	as the loop variable in for loop instead of i. Replace usage of
+	checksums[i] with checksum within the scope of the loop.
+	* metalink.c : Add functions parse_metalink, elect_resources,
+	elect_checksums and delete_mlink.
+	* metalink.h : Define new types mlink_piece_hash, mlink_checksum,
+	mlink_resource, mlink_chunk_checksum, mlink_file and mlink. Add
+	prototypes of parse_metalink and delete_mlink. Update prototype for
+	verify_file_hash.
+	* retr.c : Update metalink downloading portion of the code so that new
+	data types are used in retr.c instead of types provided by libmetalink.
+
+2012-08-18  Ilim Ugur  <ilimugur@gmail.com>
+
+	* ftp.c : Correct shallow copying of *local_file to con.target.
+	* retr.c (retrieve_from_file): Correct malloc size for file_path.
+
+2012-08-17  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file): Treat file attribute of a file in a
+	metalink file as a relative path, and create any non-existent
+	directories before attempting to merge temporary files.
+
+2012-08-17  Ilim Ugur  <ilimugur@gmail.com>
+
+	* main.c : Change the option metalink-file to metalink in option_data.
+	Extend and make, in terms of readability, more concise the interoption
+	availability checks.
+	* multi.c (merge_temp_files): Change type of parameter output from const
+	char to char.
+	* multi.c (clean_range_res_data): Use opt.jobs instead of
+	num_of_resources in the termination condition of the for loop. Remove
+	parameter num_of_resources.
+	* multi.h : Update prototypes of functions clean_range_res_data and
+	merge_temp_files.
+	* retr.c (retrieve_from file): Update calls to clean_range_res_data and
+	merge_temp_files.
+
+2012-08-16  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c : Add header comment in the existing source files of Wget.
+	Update comment above function verify_file_hash.
+	* multi.c (init_ranges): Exit after prompting user with a message when
+	allocation to ranges fails.
+	* multi.c (collect_thread): Delete the assignment statement of 0 to
+	thread_ctx[].terminated.
+	* multi.c : Add header comment in the existing source files of Wget. Add
+	comments above functions init_temp_files, name_temp_files,
+	merge_temp_files, delete_temp_files, clean_temp_files, init_ranges,
+	fill_ranges_data, clean_range_res_data, clean_ranges, spawn_thread,
+	collect_thread and segmented_retrieve_url which explain their procedure.
+	* multi.h : Update header comment in the existing source files of Wget.
+	* retr.c (retrieve_from_file): Add several comments to the parts of the
+	code that deals with metalink downloading.
+
+2012-08-16  Ilim Ugur  <ilimugur@gmail.com>
+
+	* muti.c (init_ranges): Control allocation of space for ranges using an
+	if statement and prompting user with a message in case of a failure in
+	allocation.
+	* muti.c (clean_ranges): Assign NULL to ranges after freeing.
+	* multi.c (spawn_thread): Assign files[index] to thread_ctx[].file
+	variable. The statement is moved to here from retr.c after making files
+	of retr.c a static variable of multi.c.
+	* multi.c (merge_temp_files): Remove parameter inputs as the mentioned
+	variable is now a static variable of multi.c.
+	* multi.c (delete_temp_files): Remove parameter files as the mentioned
+	variable is now a static variable of multi.c.
+	* multi.c : Add static variables temp and files (moved from retr.c).
+	Write functions init_temp_files, name_temp_files and clean_temp_files.
+	Move definitions of functions merge_temp_files and delete_temp_files.
+	* multi.h : Add prototypes of the new functions. Update prototypes of
+	merge_temp_files and delete_temp_files.
+	* retr.c (retrieve_from_file): Remove variables temp, files and index.
+	Replace statements using mentioned variables with the new functions in
+	multi.c. Update calls to merge_temp_files and delete_temp_files.
+
+2012-08-16  Ilim Ugur  <ilimugur@gmail.com>
+
+	* init.c (defaults): Add an assignment of 1 to opt.jobs within ifdef
+	brackets.
+	* multi.c (init_ranges): Remove parameter numthreads from function
+	definition. Replace numthreads with opt.jobs in the malloc call.
+	* multi.c (fill_ranges_data): Remove parameter numthreads from function
+	definition.
+	* multi.c (collect_thread): Remove parameter numthreads from function
+	definition. Replace numthreads with opt.jobs in termination condition of
+	the for loop.
+	* multi.c (merge_temp_files): Remove parameter numthreads from function
+	definition. Replace numthreads with opt.jobs in termination condition of
+	the for loop.
+	* multi.c (delete_temp_files): Remove parameter numthreads from function
+	definition. Replace numthreads with opt.jobs in termination condition of
+	the while loop.
+	* multi.h : Update prototypes of the functions modified in multi.c;
+	init_ranges, fill_ranges_data, collect_thread, merge_temp_files and
+	delete_temp_files.
+	* retr.c (retrieve_from_file): Update calls to the updated functions in
+	multi.c appropriately. Replace N_THREADS with opt.jobs throughout the
+	scope of the N_THREADS definition. Remove variable N_THREADS.
+
+2012-08-16  Ilim Ugur  <ilimugur@gmail.com>
+
+	* multi.c (spawn_thread): Assign ranges + index to thread_ctx[].range
+	variable. The statement is moved to here from retr.c after making ranges
+	of retr.c a static variable of multi.c.
+	* multi.c : Add new static variable ranges.(moved from retr.c).
+	Write functions init_ranges, fill_ranges_data, clean_range_res_data and
+	clean_range to initialize, fill, clean and free the data in ranges.
+	* multi.h : Add the prototypes of the functions which are added to
+	multi.c; init_ranges, fill_ranges_data, clean_range_res_data and
+	clean_range.
+	* retr.c (retrieve_from_file): Replace statements involving the variable
+	ranges with the new functions defined in multi.c. Delete two obsolete
+	variables.
+
+2012-08-14  Ilim Ugur  <ilimugur@gmail.com>
+
+	* multi.c (spawn_thread): Add new argument, numthreads, to prevent a
+	potential memory leak.
+	* multi.h : Update the prototype of collect_thread accordingly.
+	* retr.c (retrieve_from_file): Update the call made to collect_thread
+	accordingly.
+
+2012-08-14  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file): Move the statement in which a call to
+	metalink_context is being made, to prevent a potential segmentation
+	fault in case a non-existing metalink file location is provided.
+
+2012-08-14  Ilim Ugur  <ilimugur@gmail.com>
+
+	* init.c (defaults): Update the default value for n_retries to 1.
+	* main.c (main): Update the output prompted when "wget --help" is used.
+
+2012-08-12  Ilim Ugur  <ilimugur@gmail.com>
+
+	* multi.c (spawn_thread): Delete an argument, name.
+	* multi.c (delete_temp_files): Change type of variable, inputs from
+	const char ** to char **.
+	* multi.c (merge_temp_files): Change type of variable, files from
+	const char ** to char **.
+	* multi.h : Remove obsolete TEMP_PREFIX  macro. Update function
+	prototypes to reflect the changes mentioned above.
+	* retr.c : Use tmpnam to generate unique temporary file names, making
+	sure no file with the same name will exist, before the download.
+	delete_temp_files call before starting to download is, now, not needed.
+	Update the spawn_thread calls accordingly. Update message printing
+	statements (i.e. use LOG_NOTQUIET instead of LOG_VERBOSE).
+
+2012-08-12  Ilim Ugur  <ilimugur@gmail.com>
+
+	* multi.c (delete_temp_files): Instead of passing the name of the final
+	file, pass the array in which the temporary file names are stored.
+	Update the execution to quickly traverse the names and unlink them,
+	deleting an allocation and a now-obsolete loop.
+	* multi.h : Update the prototype of delete_temp_files.
+	* retr.c : Update the calls to delete_temp_files.
+
+2012-08-12  Ilim Ugur  <ilimugur@gmail.com>
+
+	* main.c : Initial URL check is updated so it considers metalink-file
+	option as well as input-file. The conditional statement for printing the
+	downloaded sum is updated so it considers metalink-file option as well
+	as input-file. A conditional statement is added to handle calling
+	retrieve_from_file when opt.metalink_file is specified.
+	* multi.c (merge_temp_files): Add a new parameter which will be used
+	internally to improve the execution of the function.
+	* multi.h : Update the prototype of merge_temp_files.
+	* retr.c : Update the call to merge_temp_files, passing the newly
+	required argument.
+
+2012-08-12  Ilim Ugur  <ilimugur@gmail.com>
+
+	* init.c : Add metalink-file option to commands.
+	* main.c : Add metalink-file option to option_data, as well as a few
+	interoption availability checks of the new option with the existing
+	options output_document, base_href and force_html, enclosed within an
+	ifdef directive.
+	* options.h : Add a new member,metalink_file, to struct options.
+	* retr.c (retrieve_from_file): Update the Metalink portion of the
+	function code so that it is activated when a metalink file is specified
+	using --metalink-file.
+
+2012-08-12  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file): Initialize status_least_severe member of
+	all the members of ranges array with RETROK.
+
+2012-08-11  Ilim Ugur  <ilimugur@gmail.com>
+
+	* init.c : Add retries option to commands.
+	* init.c (defaults): Add an initialization statement for opt.n_retries
+	within an ifdef directive.
+	* main.c : Add retries option to option_data.
+	* metalnk.c (lower_hex_case): Make function static.
+	* metalnk.c (verify_file_hash): Update the return values in case of an
+	error.
+	* metalink.h : Remove the prototype of find_file_hash, which has become
+	obsolete for some time now.
+	* options.h : Add a new member,n_retries, to struct options.
+	* retr.c (verify_file_hash): Call delete_temp_files to delete all the
+	files that have the same name with one of the temporary files, which can
+	cause errors. 
+	* retr.c (retrieve_from_file): Update the code so that, in case of a
+	failed attempt at downloading a file (or corrupt hash after a successful
+	download), it is retried to be downloaded opt.n_retries times in total.
+
+2012-08-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* init.c : Remove verify option from commands.
+	* main.c : Remove verify option from option_data.
+	* metalink.c : Readjust the elements in supported_hashes, digest_sizes
+	and hash_function in descending order of the strength of the encryption.
+	* metalnk.c (verify_file_hash): Readjust the execution of function so
+	that instead of comparing an explicitly stated type of hash, which used
+	to be stored in opt.hashtype, compare the strongest type available in
+	metalink file and is supported by Wget.
+	* options.h : Remove hashtype member of struct options.
+	* retr.c : Modify a message prompting statement to be more concise.
+
+2012-08-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c : Switch back to using <hash_type>_stream functions,
+	instead of taking a lower level approach. Remove functions init_ctx,
+	finish_hash_ctx, process_block, process_bytes and find_file_hash, as
+	they are no longer needed.
+	* metalink.c (verify_file_hash): Use <hash_type>_stream instead of
+	find_file_hash.
+
+2012-08-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c : Correct the value of MAX_DIGEST_LENGTH macro.
+	* metalink.c (verify_file_hash): Correct the size of two-dimensional
+	array hashes. Correct format error in conversion from byte-form to
+	hex-form of hash. Update the value of the argument fed to lower_hex_case
+	calls, due to the change in value of MAX_DIGEST_LENGTH.
+
+2012-08-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c (find_file_hash): Convert byte-form hash to hex-form.
+	* metalink.c : Add a function lower_hex_case to lower the characters
+	A-E in the hash given in the metalink file.
+	* retr.c : Add message prompting statements to be printed to inform
+	users of the result of the verification.
+
+2012-08-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c (find_file_hash): Update function calls made in this scope.
+	Move a buffer allocation statement.
+	* metalink.c (verify_file_hash): Correct the type of multi-dimensional
+	array, hashes.
+	* metalink.c : Rename init_ctx and finish_ctx to be init_hash_ctx,
+	finish_hash_ctx. Correct passed arguments while making calls to
+	functions *_init_ctx, *_finish_ctx, *_process_blocks, *_process_bytes.
+
+2012-08-08  Ilim Ugur  <ilimugur@gmail.com>
+
+	* metalink.c : Hash verification code is totally rewritten, using a
+	lower level (i.e. in terms of abstraction) approach. Functions named
+	init_ctx, finish_ctx, process_blocks and process_bytes are introduced,
+	to handle the operation which was conducted by functions md5_stream,
+	sha1_steam and sha256_stream. Add function find_file_hash to find the
+	hash of a file. verify_hash is renamed as verify_file_hash. Add error
+	message printing statements to inform user of the progress.
+	* metalink.h : Update the prototype of verify_hash and add the prototype
+	of find_file_hash.
+	* retr.c: Update call to verify_hash.
+
+2012-08-03  Ilim Ugur  <ilimugur@gmail.com>
+
+	* init.c : Correct a faulty ifdef directive and add new verify option
+	to commands array to correspond to the new command-line option.
+	* main.c : Add a command-line option verify to indicate the hash type
+	to be compared.
+	* metalink.c : Include essential modules, md5.h and sha1.h. Define
+	HASH_TYPES and MAX_DIGEST_VALUE macros. Define arrays supported_hashes,
+	digest_values and function pointer hash_function. Write function
+	verify_hash that verifies hash for a given file and its checksums
+	specified in the relevant metalink file.
+	* metalink.h : Add the prototype of verify_hash.
+	* multi.h : Add new member hashtype to struct options.
+	* retr.c : Call verify_hash after successful downloading of a file
+	specified in metalink file, and act according to its return value.
+
+2012-08-01  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c (create_authorization_line): Replace an incorrect ifdef
+	directive with an if defined directive.
+	* http.c : Move the inclusion of multi.h within the scope of an ifdef
+	directive.
+	* http.h : Include wget.h instead of multi.h, as that is where the
+	definition of the struct range currently is in.
+	* multi.c : Correct where wget.h is included. Include unistd.h. Two
+	functions named merge_temp_files and delete_temp_files are written to
+	deal with merging and deletion of temporary files, by using fread/fwrite
+	and unlink, respectively.
+	* multi.c (spawn_thread): Correct the type of a variable, thread. Delete
+	a now obsolete portion of code, which used to check if a file name of
+	which is same with a temporary file's name exists. If such a file was
+	found, it used to be deleted.
+	* multi.h : Add the prototypes of merge_temp_files and delete_temp_files
+	and remove the macro FILENAME_SIZE, which is not needed.
+	* recur.c : Move the inclusion of multi.h within the scope of an ifdef
+	directive.
+	* retr.c : Enclose the inclusion of thread and metalink related header
+	files within the scope of an ifdef directive. Remove a variable,
+	command. Use new functions merge_temp_files and delete_temp_files,
+	instead of calling system function and execute commands cat and rm.
+	* retr.h : Remove the inclusion of multi.h, as the definition of struct
+	range is no longer in multi.h and wget.h is known to be included aboe
+	retr.h.
+
+2012-07-31  Ilim Ugur  <ilimugur@gmail.com>
+
+	* ftp.c (ftp_loop_internal): Add a struct range parameter to function
+	signature. Assign first_byte and last_byte members of the new struct
+	range parameter to relevant variables, restval and len, respectively.
+	Assign the size of the data read, qtyread, to range->bytes_covered.
+	Update an if statament which checks the availability of local_file
+	before assigning a value to it.
+	* ftp.c (ftp_loop): Add a struct range parameter to function signature.
+	Pass struct range parameter to ftp_loop_internal.
+	* ftp.c : Update all ftp_loop_internal calls but the one in ftp_loop so
+	that NULL is passed as the value for the expected struct range argument.
+	* ftp.h : Update the prototype of ftp_loop to include its new struct
+	range parameter. Include wget.h, as it is where the definition of struct
+	range is.
+	* http.c : Use an if statement instead of an ifdef directive, which was,
+	in this case, a bad way of trying to simulate conditional execution of a
+	statement.
+	* retr.c (retrieve_url): Update ftp_loop_internal call and pass
+	segment_range as the value of the expected struct range argument.
+
+2012-07-31  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c (http_loop): Assign range->bytes_covered the size of retrieved
+	data, hstat.len.
+	* retr.c : Update initialization statements of struct range instances to
+	consider new struct range member, bytes_covered and not is_covered.
+	Update the first_byte member of a struct range instance in case of
+	partially retrieved segments, using new member variable, bytes_covered.
+	* wget.h : Replace the variable is_covered in the definition of struct
+	range with another variable, bytes_covered.
+
+2012-07-29  Ilim Ugur  <ilimugur@gmail.com>
+
+	* Makefile.am : Add a conditional statement for adding metalink.c to
+	wget_SOURCES.
+	* build_info.c.in : Add a line to indicate thread support.
+	* connect.h : Enclose portions of code written either by Giuseppe
+	Scrivano in his patch for multiple threaded recursive downloads or as a
+	part of Google Summer of Code project for adding Metalink support to GNU
+	Wget in ifdef and ifndef directives, based on the availability of thread
+	and/or metalink support.
+	* convert.c : Likewise.
+	* http.c : Likewise.
+	* init.c : Likewise.
+	* main.c : Likewise.
+	* multi.c : Likewise.
+	* options.h : Likewise.
+	* recur.c : Likewise.
+	* retr.c : Likewise.
+
+2012-07-27  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c : Take assertion of local_file having a NULL value and
+	assignment of value to local_file within scope of an ifndef directive.
+	* multi.c : Delete a part of a comment, which became redundant.
+	* multi.h : Correct where wget.h is included. Also, delete the
+	definition of struct range, which is moved to wget.h.
+	* retr.c : Take the inclusion of libmetalink header files and metalink.h
+	within scope of an ifdef directive, to include them only if Metalink
+	support is compiled.
+	* retr.c (retrieve_from_file): Take the Metalink downloading portions of
+	the code within ifdef directives, so that they are compiled only if
+	Metalink support is compiled.
+	* wget.h : Definition of struct range is moved to here.
+	*build_info.c.in : Add a line to indicate Metalink support.
+
+2012-07-26  Ilim Ugur  <ilimugur@gmail.com>
+
+	* multi.h : Delete variable resources_tried, which is obsolete, from
+	the definition of struct range.
+	* retr.c (retrieve_from_file): Delete redundant variable k. Add variable
+	file_name to be used in merging and deleting temporary files. Replace
+	strcat calls in merging and deleting temporary files with sprintf calls,
+	which are tidier than the sprintf calls used before (i.e. does not make
+	illegal writes, as Valgrind used to claim). Also, an else if branch is
+	added to the error checking after collecting a thread, to disable
+	fallbacking in case of an IOERROR.
+
+2012-07-25  Ilim Ugur  <ilimugur@gmail.com>
+
+	* multi.h : Define a macro, MIN_CHUNK_SIZE to stand for the minimum size
+	for a segment that can be downloaded by a thread.
+	* retr.c (retrieve_from_file): Check the size of the segments for the
+	specified number of threads. If it is smaller than MIN_CHUNK_SIZE,
+	assign MIN_CHUNK_SIZE to chunk_size and use
+	(file->size / chunk_size) + 1 threads, instead of opt.jobs. In addition,
+	replace sprintf calls with strcpy and strcat in merging-temporary-files
+	and deleting-temporary-files portions of the code.
+
+2012-07-24  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file): Move the code block which deals with
+	Metalink downloads out of the else if scope it used to be, deleting the
+	relevant else if branch. Instead, the code is inserted after the if
+	statement of which it was previously a part(i.e. the if statement which
+	deals with downloading the input file if its URL was given instead of
+	its physical location).
+
+2012-07-24  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c (http_loop): Add a missing check to 
+	* multi.c : Correct where wget.h is included.
+	* retr.c (retrieve_from_file): Correct the variable being incorrectly
+	used as index variable in a loop for an array. Add the forgotten
+	pre-incrementation statement of the loop variable of file-traversing
+	loop to the end of the loop scope of the code.
+
+2012-07-24  Ilim Ugur  <ilimugur@gmail.com>
+
+	* Makefile.am : Add multi.c and multi.h to wget_SOURCES.
+	* multi.c : Add functions spawn_thread, segmented_retrieve_url and
+	collect_thread to handle thread creation and collection in function
+	retrieve_from_file of retr.c.
+	* multi.h : Add the prototypes of the functions that are added to
+	multi.c. Add a member resources to struct range to point to a bool array
+	that shows which resources are exhausted. Moved the definition of macros
+	which are relevant to multi-thread downloading, FILENAME_SIZE and
+	TEMP_PREFIX, to here.
+	* retr.c (retrieve_from_file): Rewrite the multi-thread downloading.
+	Instead of adapting the code written by Giuseppe Scrivano for recursive
+	downloading, use an algorithm that separates thread creation from
+	collection. Move some of the assignments to newly written thread
+	spawning and collecting functions in multi.c.
+
+2012-07-23  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c : Define a TEMP_PREFIX macro to use instead of "temp_" within
+	retrieve_from_file.
+	* retr.c (retrieve_from_file): Replace "temp_" with TEMP_PREFIX. Move
+	the sem_destroy call.
+
+2012-07-22  Ilim Ugur  <ilimugur@gmail.com>
+
+	* Makefile.am : Add a conditional statement for adding metalink.c to
+	wget_SOURCES.
+	* retr.c : Include metalink/metalink_types.h, which is a part of
+	libmetalink.
+
+2012-07-21  Ilim Ugur  <ilimugur@gmail.com>
+
+	* Makefile.am : Add metalink.c to EXTRA_wget_SOURCES.
+
+2012-07-20  Ilim Ugur  <ilimugur@gmail.com>
+
+	* Makefile.am : Delete metalink.c from wget_SOURCES.
+
+2012-07-15  Ilim Ugur  <ilimugur@gmail.com>
+
+	* multi.h : Add 4 new variables to the definition of struct range.
+	* retr.c (retrieve_from_file) : Update the multi-thread downloading code
+	so that it does fallbacking(i.e. uses another one of the resources till
+	all the resources are exhausted) if a thread is unable to download a
+	segment from a resource successfully.
+
+2012-07-15  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file) : Add an assignment statement to *count, to
+	correct the "No URLs found in <file>" message that is printed even after
+	a successful download.
+
+2012-07-14  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file) : Totally separate fallbacking and multiple
+	threaded downloading portions of the code based on the value of
+	N_THREADS.
+
+2012-07-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file) : Remove obsolete retry tag.
+
+2012-07-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c (gethttp) : A case in an if statement is removed to prevent the
+	error it causes for multiple thread downloads.
+	* retr.c (retrieve_from_file) : Initializations of variables that are
+	used in multi-thread downloading are taken into an if(1) block for
+	clarification. Encapsulate the thread collecting code within a branching
+	statement that checks if the file range is covered and if there are any
+	free threads. Update the loop in which sem_wait is called so that
+	N_THREADS and free_threads are used in termination condition. Update the
+	allocation statement for the command, and move the merging-deletion code
+	into the outer loop(i.e. the one that traverses files).
+
+2012-07-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c (gethttp) : An if statement is updated so that a risky case is
+	considered only for the downloads that are not multi-threaded. Update
+	the assignments to hstat.restval and hstat.restval_last.
+	* retr.c (retrieve_from_file) : Move the memory allocation for temp_name
+	inside the inner loop(i.e. the one that traverses resources).
+
+2012-07-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file) : Add a variable file_extension to be able
+	to label the temporary files by numbers. The variable is used in naming
+	the files as well and corrects the mistake of threads writing to the
+	same file. Also, usage of the variable j by two loops that are nested is
+	corrected by using another variable, k, for the inner loop.
+
+2012-07-10  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c (gethttp) : Correct faulty branching block while calling
+	request_set_header for Range header.
+	* retr.c (retrieve_from_file) : Add a variable temp_name for generating
+	to use while name generation for temporary files. Ensure the continuity
+	of the inner loop even if number of resources is smaller than opt.jobs.
+	Further separation of fallbacking and segmented download blocks. Add a
+	block of code for merging temporary files to the resulting file and
+	deleting the temporary files after the merging. Add freeing statements
+	for struct range instances and temporary file names allocated.
+
+2012-07-09  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c : Add wgint restval_last to struct http_stat, to contain the
+	value for the end of the range. Include multi.h.
+	* http.c (gethttp) : Take into account the restval_last while calling
+	request_set_header for Range header. 
+	* http.c (http_loop) : Update the assignments to hstat.restval and
+	hstat.restval_last. Add a struct range argument to function signature.
+	* http.h : Update the prototype for http_loop so that it has the latest
+	argument added, a struct range.
+	* main.c (main) : While calling retrieve_url, add the value for its new
+	argument, a struct range.
+	* multi.h : Add the definition of the struct range. Add the definition
+	of struct s_thread_ctx, which was previously in recur.c.
+	* recur.c () : Remove the definition of the struct s_thread_ctx from here.
+	* recur.c (start_retrieve_url) : While calling retrieve_url, add the
+	value for its new argument, a struct range.
+	* recur.c (retrieve_tree) : Initialize thread_ctx[].range values to NULL.
+	* res.c (res_retrieve_file) : While calling retrieve_url, add the value
+	for its new argument, a struct range.
+	* retr.c (retrieve_url) : Add a struct range parameter to function
+	signature. While calling http_loop, add the value for its new argument,
+	a struct range.
+	* retr.c : Include semaphore.h and pthread.h. Add a function called
+	segmented_retrieve_url for pthread_create to call.
+	* retr.c (retrieve_from_file) : While calling retrieve_url, add the value
+	for its new argument, a struct range. Temporarily disable fallbacking
+	using a branchin statement if(1)-else and adapt multi-threaded download
+	code, written by Giuseppe Scrivano, into the branch that shall be
+	executed.
+	* retr.h : Update the prototype for retrieve_url so that it has the
+	latest argument added, a struct range.
+
 2012-07-03  Steven Schubiger  <stsc@member.fsf.org>
 
 	* init.c: Include warc.h for warc_close in cleanup function.
@@ -850,6 +1591,43 @@
 	* main.c (main): Likewise.
 	Reported by: Petr Pisar <petr.pisar@atlas.cz>
 
+2012-07-05  Micah Cowan  <micah@cowan.name>
+
+	* http.c (invalidate_persistent): Use xfree instead of xzero for pconn
+	freeing.
+	* http.c (register_persistent): Use xfree instead of xzero for pconn
+	freeing. Also, free after using the soon-to-be freed value, not before.
+
+2012-07-05  Ilim Ugur  <ilimugur@gmail.com>
+
+	* http.c (http_loop): Do not assert that no value for *local_file is
+	passed. If a value is passed, assign that to hstat.local_file.
+	* retr.c (retrieve_from_file): Do not use opt.output_document to pass
+	the name of the file to be downloaded. Instead, pass it to retrieve_url
+	through its 3rd argument (i.e. file parameter).
+	* retr.c (retrieve_url): Do not assign NULL value to *file if file is
+	not NULL. Do not assign NULL value to local_file. Instead assign *file.
+
+2012-07-05  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c (retrieve_from_file): Add nested loop structure to utilize the
+	information parsed from the metalink files; outer loop traverses the
+	files in metalink file while the inner loop traverses the resources for
+	that file.
+
+2012-07-03  Micah Cowan  <micah@cowan.name>
+
+	* http.c: Add a parenthesis to fix a bad persistence check.
+
+2012-07-03  Ilim Ugur  <ilimugur@gmail.com>
+
+	* retr.c: Add an else if branch to retrieve_from_file to check if input
+	file is a metalink file.
+	* metalink.c: Add a function metalink_context to parse metalink files
+	using libmetalink.
+	* metalink.h: Add the prototype of the new function in metalink.c.
+	* Makefile.am: Update wget_SOURCES to include metalink.c and metalink.h.
+
 2012-06-17  Giuseppe Scrivano  <gscrivano@gnu.org>
 
 	* wget.h: Define `CLOSEFAILED'.

src/Makefile.am

@@ -1,6 +1,6 @@
 # Makefile for `wget' utility
 # Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software
 # Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
@@ -34,29 +34,44 @@
 if IRI_IS_ENABLED
 IRI_OBJ = iri.c
 endif
+if METALINK_IS_ENABLED
+METALINK_OBJ = metalink.c
+METALINK_HEADER = metalink.h
+endif
+if THREADS_ARE_ENABLED
+THREAD_OBJ = multi.c
+MULTI_HEADER = multi.h
+endif
 
 # The following line is losing on some versions of make!
 DEFS     += -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
 LIBS     += $(LIBICONV) $(LIBINTL) $(LIB_CLOCK_GETTIME)
 
-EXTRA_DIST = css.l css.c css_.c build_info.c.in
+EXTRA_DIST = css.l css.c css_.c build_info.c.in iri.c multi.c multi.h metalink.c metalink.h
 
 bin_PROGRAMS = wget
-wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c	\
-		css_.c css-url.c	\
-		ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c	\
-		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
-		recur.c res.c retr.c spider.c url.c warc.c	\
-		utils.c exits.c build_info.c $(IRI_OBJ)	\
-		css-url.h css-tokens.h connect.h convert.h cookies.h	\
-		ftp.h hash.h host.h html-parse.h html-url.h	\
-		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
-		options.h progress.h ptimer.h recur.h res.h retr.h	\
-		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
-		exits.h gettext.h
+
+wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c			\
+	       css_.c css-url.c							\
+	       ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c	\
+	       http.c init.c log.c main.c netrc.c progress.c ptimer.c		\
+	       recur.c res.c retr.c spider.c url.c warc.c			\
+	       utils.c exits.c build_info.c $(IRI_OBJ)				\
+	       $(THREAD_OBJ) $(METALINK_OBJ)					\
+	       css-url.h css-tokens.h connect.h convert.h cookies.h		\
+	       ftp.h hash.h host.h html-parse.h html-url.h			\
+	       http.h http-ntlm.h init.h log.h mswindows.h netrc.h		\
+	       options.h progress.h ptimer.h recur.h res.h retr.h		\
+	       spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
+	       exits.h gettext.h $(THREAD_HEADER) $(METALINK_HEADER)
+
+am__wget_SOURCES_DIST = $(wget_SOURCES)
+am__libunittest_a_SOURCES_DIST = $(wget_SOURCES)
+
+
 nodist_wget_SOURCES = version.c
-EXTRA_wget_SOURCES = iri.c
 LDADD = $(LIBOBJS) ../lib/libgnu.a
+AM_LDFLAGS = $(THREAD_LIB)
 AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib
 
 

src/build_info.c.in

@@ -7,6 +7,8 @@ large-file      SIZEOF_OFF_T >= 8
 nls             defined ENABLE_NLS
 ntlm            defined ENABLE_NTLM
 opie            defined ENABLE_OPIE
+threads         defined ENABLE_THREADS
+metalink        defined ENABLE_METALINK
 psl             defined HAVE_LIBPSL
 
 ssl choice:

src/connect.h

@@ -60,7 +60,11 @@ enum {
   WAIT_FOR_READ = 1,
   WAIT_FOR_WRITE = 2
 };
+#ifdef ENABLE_THREADS
+int select_fds (int *, int *, int, double, int);
+#else
 int select_fd (int, double, int);
+#endif
 bool test_socket_open (int);
 
 struct transport_implementation {

src/convert.c

@@ -36,6 +36,9 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <errno.h>
 #include <assert.h>
+#ifdef ENABLE_THREADS
+#include <pthread.h>
+#endif
 #include "convert.h"
 #include "url.h"
 #include "recur.h"
@@ -55,6 +58,49 @@ struct hash_table *dl_url_file_map;
 struct hash_table *downloaded_html_set;
 struct hash_table *downloaded_css_set;
 
+#ifdef ENABLE_THREADS
+static pthread_mutex_t convert_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static void
+lock_convert_mutex()
+{
+  static int convert_mutex_init_p = 0;
+  if (! convert_mutex_init_p)
+    {
+      pthread_mutexattr_t mta;
+      pthread_mutexattr_init (&mta);
+      pthread_mutexattr_settype (&mta, PTHREAD_MUTEX_RECURSIVE);
+
+      pthread_mutex_init (&convert_mutex, &mta);
+      convert_mutex_init_p = 1;
+    }
+  pthread_mutex_lock (&convert_mutex);
+}
+
+
+#define THREAD_SAFE(ret, fn, args, argv)          \
+  ret fn args                                     \
+  {                                               \
+    ret r;                                        \
+    lock_convert_mutex();                         \
+    r = fn##_1 argv;                              \
+    pthread_mutex_unlock (&convert_mutex);        \
+    return r;                                     \
+  }
+
+#define THREAD_SAFE_VOID(fn, args, argv)            \
+  void fn args                                      \
+  {                                                 \
+    lock_convert_mutex();                           \
+    fn##_1 argv;                                    \
+    pthread_mutex_unlock (&convert_mutex);          \
+  }
+
+#define FNNAME_WTHREADS(fn) fn##_1
+#else
+#define FNNAME_WTHREADS(fn) fn
+#endif
+
 static void convert_links (const char *, struct urlpos *);
 
 
@@ -182,7 +228,7 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
    extracted from these two lists.  */
 
 void
-convert_all_links (void)
+FNNAME_WTHREADS(convert_all_links) (void)
 {
   double secs;
   int file_count = 0;
@@ -761,7 +807,7 @@ dissociate_urls_from_file (const char *file)
    URL has already been downloaded.  */
 
 void
-register_download (const char *url, const char *file)
+FNNAME_WTHREADS(register_download) (const char *url, const char *file)
 {
   char *old_file, *old_url;
 
@@ -841,7 +887,7 @@ register_download (const char *url, const char *file)
    register_download() above.  */
 
 void
-register_redirection (const char *from, const char *to)
+FNNAME_WTHREADS(register_redirection) (const char *from, const char *to)
 {
   char *file;
 
@@ -856,7 +902,7 @@ register_redirection (const char *from, const char *to)
 /* Register that the file has been deleted. */
 
 void
-register_delete_file (const char *file)
+FNNAME_WTHREADS(register_delete_file) (const char *file)
 {
   char *old_url, *old_file;
 
@@ -874,7 +920,7 @@ register_delete_file (const char *file)
 /* Register that FILE is an HTML file that has been downloaded. */
 
 void
-register_html (const char *file)
+FNNAME_WTHREADS(register_html) (const char *url, const char *file)
 {
   if (!downloaded_html_set)
     downloaded_html_set = make_string_hash_table (0);
@@ -884,7 +930,7 @@ register_html (const char *file)
 /* Register that FILE is a CSS file that has been downloaded. */
 
 void
-register_css (const char *file)
+FNNAME_WTHREADS(register_css) (const char *url, const char *file)
 {
   if (!downloaded_css_set)
     downloaded_css_set = make_string_hash_table (0);
@@ -895,8 +941,8 @@ static void downloaded_files_free (void);
 
 /* Cleanup the data structures associated with this file.  */
 
-void
-convert_cleanup (void)
+static void
+FNNAME_WTHREADS(convert_cleanup) (void)
 {
   if (dl_file_url_map)
     {
@@ -971,7 +1017,7 @@ downloaded_mode_to_ptr (downloaded_file_t mode)
    URLs.  */
 
 downloaded_file_t
-downloaded_file (downloaded_file_t mode, const char *file)
+FNNAME_WTHREADS(downloaded_file) (downloaded_file_t mode, const char *file)
 {
   downloaded_file_t *ptr;
 
@@ -1085,6 +1131,17 @@ html_quote_string (const char *s)
   return res;
 }
 
+#ifdef ENABLE_THREADS
+THREAD_SAFE (downloaded_file_t, downloaded_file, (downloaded_file_t a, const char *b), (a, b));
+THREAD_SAFE_VOID (register_download, (const char *a, const char *b), (a, b));
+THREAD_SAFE_VOID (register_redirection, (const char *a, const char *b), (a, b));
+THREAD_SAFE_VOID (register_html, (const char *a, const char *b), (a, b));
+THREAD_SAFE_VOID (register_css, (const char *a, const char *b), (a, b));
+THREAD_SAFE_VOID (register_delete_file, (const char *a), (a));
+THREAD_SAFE_VOID (convert_cleanup, (void), ());
+THREAD_SAFE_VOID (convert_all_links, (void), ());
+#endif
+
 /*
  * vim: et ts=2 sw=2
  */

src/convert.h

@@ -98,14 +98,12 @@ typedef enum
 } downloaded_file_t;
 
 downloaded_file_t downloaded_file (downloaded_file_t, const char *);
-
-void register_download (const char *, const char *);
-void register_redirection (const char *, const char *);
-void register_html (const char *);
-void register_css (const char *);
-void register_delete_file (const char *);
+void register_download (const char *a, const char *b);
 void convert_all_links (void);
-void convert_cleanup (void);
+void register_delete_file (const char *a);
+void register_redirection (const char *a, const char *b);
+void register_css (const char *a, const char *b);
+void register_html (const char *a, const char *b);
 
 char *html_quote_string (const char *);
 

src/ftp.c

@@ -1306,6 +1306,11 @@ Error in server response, closing control connection.\n"));
 
   /* Get the contents of the document.  */
   flags = 0;
+#ifdef ENABLE_METALINK
+  if (opt.metalink_file && expected_bytes)
+    /* If content-length is present, read that much; otherwise, read until EOF. */
+    flags |= rb_read_exactly;
+#endif
   if (restval && rest_failed)
     flags |= rb_skip_startpos;
   rd_size = 0;
@@ -1365,8 +1370,13 @@ Error in server response, closing control connection.\n"));
   /* If retrieval failed for any reason, return FTPRETRINT, but do not
      close socket, since the control connection is still alive.  If
      there is something wrong with the control connection, it will
-     become apparent later.  */
-  if (*respline != '2')
+     become apparent later.
+     In case of a metalink file download, proabably due to taking only a chunk
+     of the file, respline becomes '2', even though file download is successful.
+     TODO: Make sure exempting metalink downloads from this check does not cause
+     any troubles.
+     */
+  if (*respline != '2' && !opt.metalink_file)
     {
       xfree (respline);
       if (res != -1)
@@ -1536,7 +1546,7 @@ Error in server response, closing control connection.\n"));
    This loop either gets commands from con, or (if ON_YOUR_OWN is
    set), makes them up to retrieve the file given by the URL.  */
 static uerr_t
-ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
+ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file, struct range *range)
 {
   int count, orig_lp;
   wgint restval, len = 0, qtyread = 0;
@@ -1651,6 +1661,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
       else
         restval = 0;
 
+
       /* Get the current time string.  */
       tms = datetime_str (time (NULL));
       /* Print fetch message, if opt.verbose.  */
@@ -1674,11 +1685,24 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
       else
         len = 0;
 
+      /* If range is not NULL, then this is a segmented download. Get the relevant
+         segment information from the specified range parameter. */
+      if (range)
+        {
+          restval = range->first_byte;
+          /* It is not the length in the usual sense, but this is the correct
+             value for getftp to use. */
+          len = range->last_byte + 1;
+        }
+
       /* If we are working on a WARC record, getftp should also write
          to the warc_tmp file. */
       err = getftp (u, len, &qtyread, restval, con, count, &last_expected_bytes,
                     warc_tmp);
 
+      if (range)
+        range->bytes_covered = qtyread;
+
       if (con->csock == -1)
         con->st &= ~DONE_CWD;
       else
@@ -1813,7 +1837,7 @@ Removing file due to --delete-after in ftp_loop_internal():\n"));
       else
         con->cmd &= ~LEAVE_PENDING;
 
-      if (local_file)
+      if (local_file && !*local_file)
         *local_file = xstrdup (locf);
 
       return RETROK;
@@ -1851,7 +1875,7 @@ ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
 
   con->target = xstrdup (lf);
   xfree (lf);
-  err = ftp_loop_internal (u, NULL, con, NULL);
+  err = ftp_loop_internal (u, NULL, con, NULL, NULL);
   lf = xstrdup (con->target);
   xfree (con->target);
   con->target = old_target;
@@ -2046,7 +2070,7 @@ Already have correct symlink %s -> %s\n\n"),
           else                /* opt.retr_symlinks */
             {
               if (dlthis)
-                err = ftp_loop_internal (u, f, con, NULL);
+                err = ftp_loop_internal (u, f, con, NULL, NULL);
             } /* opt.retr_symlinks */
           break;
         case FT_DIRECTORY:
@@ -2057,7 +2081,7 @@ Already have correct symlink %s -> %s\n\n"),
         case FT_PLAINFILE:
           /* Call the retrieve loop.  */
           if (dlthis)
-            err = ftp_loop_internal (u, f, con, NULL);
+            err = ftp_loop_internal (u, f, con, NULL, NULL);
           break;
         case FT_UNKNOWN:
           logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
@@ -2340,7 +2364,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
         {
           /* Let's try retrieving it anyway.  */
           con->st |= ON_YOUR_OWN;
-          res = ftp_loop_internal (u, NULL, con, NULL);
+          res = ftp_loop_internal (u, NULL, con, NULL, NULL);
           return res;
         }
 
@@ -2361,7 +2385,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
    encoded into a URL.  */
 uerr_t
 ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
-          bool recursive, bool glob)
+          bool recursive, bool glob, struct range *range)
 {
   ccon con;                     /* FTP connection */
   uerr_t res;
@@ -2375,6 +2399,10 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
   con.rs = ST_UNIX;
   con.id = NULL;
   con.proxy = proxy;
+  /* To let ftp_loop_internal AND getftp know of the desired file name. Added
+     while implementing metalink support to wget. */
+  if(local_file && *local_file)
+    con.target = xstrdup (*local_file);
 
   /* If the file name is empty, the user probably wants a directory
      index.  We'll provide one, properly HTML-ized.  Unless
@@ -2440,7 +2468,7 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
                                    ispattern ? GLOB_GLOBALL : GLOB_GETONE);
         }
       else
-        res = ftp_loop_internal (u, NULL, &con, local_file);
+        res = ftp_loop_internal (u, NULL, &con, local_file, range);
     }
   if (res == FTPOK)
     res = RETROK;

src/ftp.h

@@ -32,6 +32,8 @@ as that of the covered work.  */
 #ifndef FTP_H
 #define FTP_H
 
+#include "wget.h"
+
 #include "host.h"
 
 /* System types. */
@@ -152,7 +154,7 @@ enum wget_ftp_fstatus
 };
 
 struct fileinfo *ftp_parse_ls (const char *, const enum stype);
-uerr_t ftp_loop (struct url *, char **, int *, struct url *, bool, bool);
+uerr_t ftp_loop (struct url *, char **, int *, struct url *, bool, bool, struct range *);
 
 uerr_t ftp_index (const char *, struct url *, struct fileinfo *);
 

src/http.c

@@ -39,7 +39,11 @@ as that of the covered work.  */
 #include <errno.h>
 #include <time.h>
 #include <locale.h>
+#ifdef ENABLE_THREADS
+#include <pthread.h>
 
+# include "multi.h"
+#endif
 #include "hash.h"
 #include "http.h"
 #include "utils.h"
@@ -1231,10 +1235,42 @@ parse_content_disposition (const char *hdr, char **filename)
    below.  Ideally, it should be possible to cache an arbitrary fixed
    number of these connections.  */
 
+#ifndef ENABLE_THREADS
 /* Whether a persistent connection is active. */
 static bool pconn_active;
 
+#define PCONN_LOCK()
+#define PCONN_UNLOCK()
+
 static struct {
+#else
+static pthread_mutex_t pconn_mutex;
+
+static void
+pconn_lock()
+  {
+    static int pconn_mutex_init_p = 0;
+    if (! pconn_mutex_init_p)
+      {
+        pthread_mutexattr_t mta;
+        pthread_mutexattr_init (&mta);
+        pthread_mutexattr_settype (&mta, PTHREAD_MUTEX_RECURSIVE);
+
+        pthread_mutex_init (&pconn_mutex, &mta);
+        pconn_mutex_init_p = 1;
+      }
+
+    pthread_mutex_lock (&pconn_mutex);
+  }
+
+#define PCONN_LOCK()  pconn_lock()
+#define PCONN_UNLOCK() pthread_mutex_unlock (&pconn_mutex)
+
+static struct s_pconn {
+
+  struct s_pconn *next;
+#endif
+
   /* The socket of the connection.  */
   int socket;
 
@@ -1255,12 +1291,19 @@ static struct {
   /* NTLM data of the current connection.  */
   struct ntlmdata ntlm;
 #endif
-} pconn;
 
+#ifndef ENABLE_THREADS
+} pconn;
+#else
+} *pconn = NULL;
+
+size_t pconn_length = 0;
+#endif
 /* Mark the persistent connection as invalid and free the resources it
    uses.  This is used by the CLOSE_* macros after they forcefully
    close a registered persistent connection.  */
 
+#ifndef ENABLE_THREADS
 static void
 invalidate_persistent (void)
 {
@@ -1270,7 +1313,40 @@ invalidate_persistent (void)
   xfree (pconn.host);
   xzero (pconn);
 }
+#else
+static void
+invalidate_persistent (int fd)
+{
+  struct s_pconn *prev, *it;
+  DEBUGP (("Disabling further reuse of socket %d.\n", fd));
 
+  PCONN_LOCK ();
+
+  prev = NULL;
+  it = pconn;
+  for (; it; it = it->next)
+    {
+      if (it->socket == fd)
+        break;
+      prev = it;
+    }
+
+  fd_close (fd);
+
+  if (it)
+    {
+      if (prev)
+        prev->next = it->next;
+      else
+        pconn = it->next;
+      pconn_length--;
+      xfree (it->host);
+      xfree (it);
+    }
+
+  PCONN_UNLOCK ();
+}
+#endif
 /* Register FD, which should be a TCP/IP connection to HOST:PORT, as
    persistent.  This will enable someone to use the same connection
    later.  In the context of HTTP, this must be called only AFTER the
@@ -1282,6 +1358,7 @@ invalidate_persistent (void)
 static void
 register_persistent (const char *host, int port, int fd, bool ssl)
 {
+#ifndef ENABLE_THREADS
   if (pconn_active)
     {
       if (pconn.socket == fd)
@@ -1308,32 +1385,103 @@ register_persistent (const char *host, int port, int fd, bool ssl)
   pconn.authorized = false;
 
   DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
+#else
+  PCONN_LOCK ();
+
+  struct s_pconn *it;
+  for (it = pconn; it; it = it->next)
+    if (it->socket == fd)
+      {
+        DEBUGP (("Socket %d already registered.\n", fd));
+        goto exit;
+      }
+
+  if (pconn_length >= opt.jobs)
+    {
+      struct s_pconn *prev = NULL;
+      if (pconn)
+        {
+        for (it = pconn; it->next; it = it->next)
+          prev = it;
+
+        if (prev)
+          {
+            prev->next = it->next;
+            xfree (it->host);
+            xfree (it);
+          }
+        else
+          {
+            xfree (pconn->host);
+            xfree (pconn);
+            pconn = NULL;
+          }
+        }
+    }
+
+  it = malloc (sizeof *it);
+  if (it == NULL)
+    goto exit;
+
+  memset (it, 0, sizeof *it);
+
+  it->socket = fd;
+  it->host = xstrdup (host);
+  it->port = port;
+  it->ssl = ssl;
+  it->authorized = false;
+  it->next = pconn;
+  pconn = it;
+  pconn_length++;
+
+  DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
+ exit:
+  PCONN_UNLOCK ();
+#endif
 }
 
+#ifndef ENABLE_THREADS
 /* Return true if a persistent connection is available for connecting
    to HOST:PORT.  */
-
 static bool
 persistent_available_p (const char *host, int port, bool ssl,
                         bool *host_lookup_failed)
+#else
+/* Return 1 if a persistent connection is available for connecting
+   to HOST:PORT.  0 if it is not available, -1 if the connection must
+   be destroyed by the caller.  */
+static int
+persistent_available_p (struct s_pconn *pconn, const char *host, int port,
+                        bool ssl, bool *host_lookup_failed)
+#endif
 {
+#ifndef ENABLE_THREADS
   /* First, check whether a persistent connection is active at all.  */
   if (!pconn_active)
-    return false;
+    return 0;
 
   /* If we want SSL and the last connection wasn't or vice versa,
      don't use it.  Checking for host and port is not enough because
      HTTP and HTTPS can apparently coexist on the same port.  */
   if (ssl != pconn.ssl)
-    return false;
+    return 0;
 
   /* If we're not connecting to the same port, we're not interested. */
   if (port != pconn.port)
-    return false;
+    return 0;
 
   /* If the host is the same, we're in business.  If not, there is
      still hope -- read below.  */
   if (0 != strcasecmp (host, pconn.host))
+#else
+  if (ssl != pconn->ssl)
+    return 0;
+
+  if (port != pconn->port)
+    return 0;
+
+  if (0 != strcasecmp (host, pconn->host))
+#endif
     {
       /* Check if pconn.socket is talking to HOST under another name.
          This happens often when both sites are virtual hosts
@@ -1351,31 +1499,37 @@ persistent_available_p (const char *host, int port, bool ssl,
         /* Don't try to talk to two different SSL sites over the same
            secure connection!  (Besides, it's not clear that
            name-based virtual hosting is even possible with SSL.)  */
-        return false;
+        return 0;
 
       /* If pconn.socket's peer is one of the IP addresses HOST
          resolves to, pconn.socket is for all intents and purposes
          already talking to HOST.  */
 
+#ifndef ENABLE_THREADS
       if (!socket_ip_address (pconn.socket, &ip, ENDPOINT_PEER))
         {
           /* Can't get the peer's address -- something must be very
              wrong with the connection.  */
-          invalidate_persistent ();
-          return false;
+          return 0;
         }
+#else
+      if (!socket_ip_address (pconn->socket, &ip, ENDPOINT_PEER))
+        {
+          return -1;
+        }
+#endif
       al = lookup_host (host, 0);
       if (!al)
         {
           *host_lookup_failed = true;
-          return false;
+          return 0;
         }
 
       found = address_list_contains (al, &ip);
       address_list_release (al);
 
       if (!found)
-        return false;
+        return 0;
 
       /* The persistent connection's peer address was found among the
          addresses HOST resolved to; therefore, pconn.sock is in fact
@@ -1395,17 +1549,69 @@ persistent_available_p (const char *host, int port, bool ssl,
      body in response to HEAD, or if it sends more than conent-length
      data, we won't reuse the corrupted connection.)  */
 
+#ifndef ENABLE_THREADS
   if (!test_socket_open (pconn.socket))
     {
       /* Oops, the socket is no longer open.  Now that we know that,
          let's invalidate the persistent connection before returning
          0.  */
-      invalidate_persistent ();
-      return false;
+      return -1;
+    }
+#else
+  if (!test_socket_open (pconn->socket))
+    {
+      return -1;
+    }
+#endif
+
+  return 1;
+}
+
+#ifdef ENABLE_THREADS
+static struct s_pconn *
+get_persistent (const char *host, int port, bool ssl, bool *host_lookup_failed)
+{
+  struct s_pconn *prev, *it;
+
+  PCONN_LOCK ();
+
+  prev = NULL;
+  it = pconn;
+  for (; it; it = it->next)
+    {
+      int ret;
+      ret = persistent_available_p (it, host, port, ssl, host_lookup_failed);
+
+      if (ret == 1)
+        break;
+
+      if (ret == 0)
+        prev = it;
+      else
+        {
+          struct s_pconn *next = it->next;
+          invalidate_persistent (it->socket);
+          it = next;
+          if (it == NULL)
+            break;
+        }
     }
 
-  return true;
+  if (it)
+    {
+      pconn_length--;
+      if (prev)
+        prev->next = it->next;
+      else
+        pconn = it->next;
+    }
+
+  PCONN_UNLOCK ();
+
+  return it;
 }
+#endif
+
 
 /* The idea behind these two CLOSE macros is to distinguish between
    two cases: one when the job we've been doing is finished, and we
@@ -1422,6 +1628,8 @@ persistent_available_p (const char *host, int port, bool ssl,
    `pc_active_p && (fd) == pc_last_fd' is "we're *now* using an
    active, registered connection".  */
 
+#ifndef ENABLE_THREADS
+
 #define CLOSE_FINISH(fd) do {                   \
   if (!keep_alive)                              \
     {                                           \
@@ -1442,12 +1650,30 @@ persistent_available_p (const char *host, int port, bool ssl,
     fd_close (fd);                              \
   fd = -1;                                      \
 } while (0)
+
+#else
+
+#define CLOSE_FINISH(fd) do {                   \
+  if (!keep_alive)                              \
+    {                                           \
+      invalidate_persistent (fd);               \
+    }                                           \
+} while (0)
+
+#define CLOSE_INVALIDATE(fd) do {               \
+    invalidate_persistent (fd);                 \
+    fd = -1;                                    \
+} while (0)
+
+#endif
 
 struct http_stat
 {
   wgint len;                    /* received length */
   wgint contlen;                /* expected length */
   wgint restval;                /* the restart value */
+  wgint restval_last;           /* last byte to download while downloading in
+                                   segments */
   int res;                      /* the result of last read */
   char *rderrmsg;               /* error message from read error */
   char *newloc;                 /* new location (redirection) */
@@ -1687,6 +1913,10 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
 
   int sock = -1;
 
+#ifdef ENABLE_THREADS
+  struct s_pconn *pconn = NULL;
+#endif
+
   /* Set to 1 when the authorization has already been sent and should
      not be tried again. */
   bool auth_finished = false;
@@ -1796,7 +2026,16 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
       /* ... but some HTTP/1.0 caches doesn't implement Cache-Control.  */
       request_set_header (req, "Pragma", "no-cache", rel_none);
     }
-  if (hs->restval)
+
+  if(hs->restval_last)
+    {
+      request_set_header (req, "Range",
+                          aprintf ("bytes=%s-%s",
+                                   number_to_static_string (hs->restval),
+                                   number_to_static_string (hs->restval_last)),
+                          rel_value);
+    }
+  else if (hs->restval)
     request_set_header (req, "Range",
                         aprintf ("bytes=%s-",
                                  number_to_static_string (hs->restval)),
@@ -1955,6 +2194,8 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
         relevant = u;
 #endif
 
+#ifndef ENABLE_THREADS
+
       if (persistent_available_p (relevant->host, relevant->port,
 #ifdef HAVE_SSL
                                   relevant->scheme == SCHEME_HTTPS,
@@ -1978,6 +2219,32 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
                         pconn.port);
           DEBUGP (("Reusing fd %d.\n", sock));
           if (pconn.authorized)
+#else
+
+      pconn = get_persistent (relevant->host, relevant->port,
+#ifdef HAVE_SSL
+                              relevant->scheme == SCHEME_HTTPS,
+#else
+                              0,
+#endif
+                              &host_lookup_failed);
+
+      if (pconn)
+        {
+          int family = socket_family (pconn->socket, ENDPOINT_PEER);
+          sock = pconn->socket;
+          using_ssl = pconn->ssl;
+          if (family == AF_INET6)
+             logprintf (LOG_VERBOSE, _("Reusing existing connection to [%s]:%d.\n"),
+                        quotearg_style (escape_quoting_style, pconn->host),
+                         pconn->port);
+          else
+             logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
+                        quotearg_style (escape_quoting_style, pconn->host),
+                        pconn->port);
+          DEBUGP (("Reusing fd %d.\n", sock));
+          if (pconn->authorized)
+#endif
             /* If the connection is already authorized, the "Basic"
                authorization added by code above is unnecessary and
                only hurts us.  */
@@ -1999,7 +2266,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
 
   if (sock < 0)
     {
+      PCONN_LOCK();
       sock = connect_to_host (conn->host, conn->port);
+      PCONN_UNLOCK();
       if (sock == E_HOST)
         {
           request_free (req);
@@ -2318,10 +2587,14 @@ read_header:
         }
     }
 
-  if (keep_alive)
-    /* The server has promised that it will not close the connection
-       when we're done.  This means that we can register it.  */
-    register_persistent (conn->host, conn->port, sock, using_ssl);
+  /* The server has promised that it will not close the connection
+     when we're done.  This means that we can register it.  */
+#define REGISTER_PERSISTENT_CONNECTION(i) do {   \
+    if (keep_alive)                                                     \
+      register_persistent (conn->host, conn->port, sock, using_ssl);    \
+    else                                                                \
+      CLOSE_FINISH(sock);                                               \
+  } while (0)
 
   if (statcode == HTTP_STATUS_UNAUTHORIZED)
     {
@@ -2350,19 +2623,25 @@ read_header:
               return _err;
             }
           else
-            CLOSE_FINISH (sock);
+            REGISTER_PERSISTENT_CONNECTION (1);
         }
       else
         {
           /* Since WARC is disabled, we are not interested in the response body.  */
           if (keep_alive && !head_only
-              && skip_short_body (sock, contlen, chunked_transfer_encoding))
-            CLOSE_FINISH (sock);
+              && skip_short_body (sock, contlen, chunked_transfer_encoding)) {
+            REGISTER_PERSISTENT_CONNECTION (2);
+          }
           else
             CLOSE_INVALIDATE (sock);
         }
 
+#ifndef ENABLE_THREADS
       pconn.authorized = false;
+#else
+      if (pconn)
+        pconn->authorized = false;
+#endif
       uerr_t auth_err = RETROK;
       if (!auth_finished && (user && passwd))
         {
@@ -2441,6 +2720,8 @@ read_header:
       xfree_null (message);
       resp_free (resp);
       xfree (head);
+      REGISTER_PERSISTENT_CONNECTION (3);
+
       if (auth_err == RETROK)
         return AUTHFAILED;
       else
@@ -2450,7 +2731,14 @@ read_header:
     {
       /* Kludge: if NTLM is used, mark the TCP connection as authorized. */
       if (ntlm_seen)
+      {
+#ifndef ENABLE_THREADS
         pconn.authorized = true;
+#else
+        if (pconn)
+          pconn->authorized = true;
+#endif
+      }
     }
 
   /* Determine the local filename if needed. Notice that if -O is used
@@ -2492,6 +2780,7 @@ read_header:
           resp_free (resp);
           xfree (head);
           xfree_null (message);
+          REGISTER_PERSISTENT_CONNECTION (4);
           return RETRUNNEEDED;
         }
       else if (!ALLOW_CLOBBER)
@@ -2674,20 +2963,23 @@ read_header:
                   return _err;
                 }
               else
-                CLOSE_FINISH (sock);
+                REGISTER_PERSISTENT_CONNECTION (5);
             }
           else
             {
               /* Since WARC is disabled, we are not interested in the response body.  */
               if (keep_alive && !head_only
                   && skip_short_body (sock, contlen, chunked_transfer_encoding))
-                CLOSE_FINISH (sock);
+                {
+                  REGISTER_PERSISTENT_CONNECTION (11);
+                }
               else
                 CLOSE_INVALIDATE (sock);
             }
 
           xfree_null (type);
           xfree (head);
+          REGISTER_PERSISTENT_CONNECTION (6);
           /* From RFC2616: The status codes 303 and 307 have
              been added for servers that wish to make unambiguously
              clear which kind of reaction is expected of the client.
@@ -2775,8 +3067,7 @@ read_header:
       xfree (head);
       return RETRUNNEEDED;
     }
-  if ((contrange != 0 && contrange != hs->restval)
-      || (H_PARTIAL (statcode) && !contrange))
+  if ((contrange != 0 && (!(hs->restval_last) && contrange != hs->restval)))
     {
       /* The Range request was somehow misunderstood by the server.
          Bail out.  */
@@ -2851,7 +3142,7 @@ read_header:
               return _err;
             }
           else
-            CLOSE_FINISH (sock);
+            REGISTER_PERSISTENT_CONNECTION(7);
         }
       else
         {
@@ -2865,12 +3156,15 @@ read_header:
             CLOSE_FINISH (sock);
           else if (keep_alive
                    && skip_short_body (sock, contlen, chunked_transfer_encoding))
-            /* Successfully skipped the body; also keep using the socket. */
-            CLOSE_FINISH (sock);
+            {
+              /* Successfully skipped the body; also keep using the socket. */
+              REGISTER_PERSISTENT_CONNECTION(8);
+            }
           else
             CLOSE_INVALIDATE (sock);
         }
 
+      REGISTER_PERSISTENT_CONNECTION (9);
       xfree (head);
       xfree_null (type);
       return RETRFINISHED;
@@ -2972,6 +3266,8 @@ read_header:
   xfree (head);
   xfree_null (type);
 
+  REGISTER_PERSISTENT_CONNECTION (10);
+
   if (hs->res >= 0)
     CLOSE_FINISH (sock);
   else
@@ -2988,7 +3284,7 @@ read_header:
 uerr_t
 http_loop (struct url *u, struct url *original_url, char **newloc,
            char **local_file, const char *referer, int *dt, struct url *proxy,
-           struct iri *iri)
+           struct iri *iri, struct range *range)
 {
   int count;
   bool got_head = false;         /* used for time-stamping and filename detection */
@@ -3010,12 +3306,14 @@ http_loop (struct url *u, struct url *original_url, char **newloc,
     force_full_retrieve = true;
 
 
+#ifndef ENABLE_METALINK
   /* Assert that no value for *LOCAL_FILE was passed. */
   assert (local_file == NULL || *local_file == NULL);
 
   /* Set LOCAL_FILE parameter. */
   if (local_file && opt.output_document)
     *local_file = HYPHENP (opt.output_document) ? NULL : xstrdup (opt.output_document);
+#endif
 
   /* Reset NEWLOC parameter. */
   *newloc = NULL;
@@ -3041,8 +3339,11 @@ http_loop (struct url *u, struct url *original_url, char **newloc,
     }
   else if (!opt.content_disposition)
     {
-      hstat.local_file =
-        url_file_name (opt.trustservernames ? u : original_url, NULL);
+      if(local_file && *local_file)
+        hstat.local_file = xstrdup (*local_file);
+      else
+        hstat.local_file =
+          url_file_name (opt.trustservernames ? u : original_url, NULL);
       got_name = true;
     }
 
@@ -3128,6 +3429,9 @@ Spider mode enabled. Check if remote file exists.\n"));
       else
         *dt &= ~HEAD_ONLY;
 
+
+      /* GSoC TODO: should correct assignments to restval_last below to
+         include cases like server failure in one of the segments. */
       /* Decide whether or not to restart.  */
       if (force_full_retrieve)
         hstat.restval = hstat.len;
@@ -3140,12 +3444,20 @@ Spider mode enabled. Check if remote file exists.\n"));
         /* When -c is used, continue from on-disk size.  (Can't use
            hstat.len even if count>1 because we don't want a failed
            first attempt to clobber existing data.)  */
-        hstat.restval = st.st_size;
+          hstat.restval = st.st_size;
       else if (count > 1)
         /* otherwise, continue where the previous try left off */
-        hstat.restval = hstat.len;
+          hstat.restval = hstat.len;
       else
-        hstat.restval = 0;
+          hstat.restval = 0;
+
+      if (range)
+        {
+          hstat.restval = range->first_byte;
+          hstat.restval_last = range -> last_byte;
+        }
+      else
+        hstat.restval_last = 0;
 
       /* Decide whether to send the no-cache directive.  We send it in
          two cases:
@@ -3165,6 +3477,9 @@ Spider mode enabled. Check if remote file exists.\n"));
       /* Time?  */
       tms = datetime_str (time (NULL));
 
+      if(range)
+        range->bytes_covered = hstat.len;
+
       /* Get the new location (with or without the redirection).  */
       if (hstat.newloc)
         *newloc = xstrdup (hstat.newloc);
@@ -3562,7 +3877,7 @@ Remote file exists.\n\n"));
   while (!opt.ntry || (count < opt.ntry));
 
 exit:
-  if (ret == RETROK && local_file)
+  if (ret == RETROK && local_file && !(*local_file))
     {
       xfree_null (*local_file);
       *local_file = xstrdup (hstat.local_file);
@@ -3988,7 +4303,16 @@ create_authorization_line (const char *au, const char *user,
       *finished = true;
       return digest_authentication_encode (au, user, passwd, method, path, auth_err);
 #endif
-#ifdef ENABLE_NTLM
+
+#if defined ENABLE_NTLM && defined ENABLE_THREADS
+    case 'N':                   /* NTLM */
+      if (!ntlm_input (&pconn->ntlm, au))
+        {
+          *finished = true;
+          return NULL;
+        }
+      return ntlm_output (&pconn->ntlm, user, passwd, finished);
+#elif defined ENABLE_NTLM
     case 'N':                   /* NTLM */
       if (!ntlm_input (&pconn.ntlm, au))
         {
@@ -4026,7 +4350,18 @@ save_cookies (void)
 void
 http_cleanup (void)
 {
+#ifndef ENABLE_THREADS
   xfree_null (pconn.host);
+#else
+  struct s_pconn *it = pconn;
+
+  for (; it; it = it->next)
+    {
+      xfree_null (it->host);
+      xzero (it);
+    }
+#endif
+
   if (wget_cookie_jar)
     cookie_jar_delete (wget_cookie_jar);
 }

src/http.h

@@ -31,10 +31,12 @@ as that of the covered work.  */
 #ifndef HTTP_H
 #define HTTP_H
 
+#include "wget.h"
+
 struct url;
 
 uerr_t http_loop (struct url *, struct url *, char **, char **, const char *,
-                  int *, struct url *, struct iri *);
+                  int *, struct url *, struct iri *, struct range *);
 void save_cookies (void);
 void http_cleanup (void);
 time_t http_atotm (const char *);

src/init.c

@@ -208,6 +208,9 @@ static const struct {
 #endif
   { "input",            &opt.input_filename,    cmd_file },
   { "iri",              &opt.enable_iri,        cmd_boolean },
+#ifdef ENABLE_THREADS
+  { "jobs",             &opt.jobs,              cmd_number },
+#endif
   { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean },
   { "limitrate",        &opt.limit_rate,        cmd_bytes },
   { "loadcookies",      &opt.cookies_input,     cmd_file },
@@ -215,6 +218,9 @@ static const struct {
   { "logfile",          &opt.lfilename,         cmd_file },
   { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
   { "maxredirect",      &opt.max_redirect,      cmd_number },
+#ifdef ENABLE_METALINK
+  { "metalink",         &opt.metalink_file,     cmd_file },
+#endif
   { "method",           &opt.method,            cmd_string_uppercase },
   { "mirror",           NULL,                   cmd_spec_mirror },
   { "netrc",            &opt.netrc,             cmd_boolean },
@@ -259,6 +265,9 @@ static const struct {
   { "removelisting",    &opt.remove_listing,    cmd_boolean },
   { "reportspeed",             &opt.report_bps, cmd_spec_report_speed},
   { "restrictfilenames", NULL,                  cmd_spec_restrict_file_names },
+#ifdef ENABLE_METALINK
+  { "retries",          &opt.n_retries,         cmd_number_inf },
+#endif
   { "retrsymlinks",     &opt.retr_symlinks,     cmd_boolean },
   { "retryconnrefused", &opt.retry_connrefused, cmd_boolean },
   { "robots",           &opt.use_robots,        cmd_boolean },
@@ -342,6 +351,12 @@ defaults (void)
   opt.cookies = true;
   opt.verbose = -1;
   opt.ntry = 20;
+#ifdef ENABLE_THREADS
+  opt.jobs = 1;
+#endif
+#ifdef ENABLE_METALINK
+  opt.n_retries = 1;
+#endif
   opt.reclevel = 5;
   opt.add_hostdir = true;
   opt.netrc = true;

src/main.c

@@ -220,12 +220,18 @@ static struct cmdline_option option_data[] =
 #endif
     { "input-file", 'i', OPT_VALUE, "input", -1 },
     { "iri", 0, OPT_BOOLEAN, "iri", -1 },
+#ifdef ENABLE_THREADS
+    { "jobs", 0, OPT_VALUE, "jobs", 1 },
+#endif
     { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 },
     { "level", 'l', OPT_VALUE, "reclevel", -1 },
     { "limit-rate", 0, OPT_VALUE, "limitrate", -1 },
     { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
     { "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
     { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
+#ifdef ENABLE_METALINK
+    { "metalink-file", 0, OPT_VALUE, "metalink", -1 },
+#endif
     { "method", 0, OPT_VALUE, "method", -1 },
     { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
     { "no", 'n', OPT__NO, NULL, required_argument },
@@ -268,6 +274,9 @@ static struct cmdline_option option_data[] =
     { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 },
     { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
     { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
+#ifdef ENABLE_METALINK
+    { "retries", 0, OPT_VALUE, "retries", -1 },
+#endif
     { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
     { "save-cookies", 0, OPT_VALUE, "savecookies", -1 },
     { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 },
@@ -472,8 +481,18 @@ Logging and input file:\n"),
        --no-config                 Do not read any config file.\n"),
     "\n",
 
+#ifdef ENABLE_METALINK
+    N_("\
+       --metalink-file       download URLs found in local or external metalink FILE.\n"),
+#endif
+
     N_("\
 Download:\n"),
+#ifdef ENABLE_METALINK
+    N_("\
+       --retries                 specify the number of retries for a file.\n\
+                                 (needs to be used with --metalink-file)\n"),
+#endif
     N_("\
   -t,  --tries=NUMBER              set number of retries to NUMBER (0 unlimits).\n"),
     N_("\
@@ -552,8 +571,11 @@ Download:\n"),
        --remote-encoding=ENC       use ENC as the default remote encoding.\n"),
     N_("\
        --unlink                    remove file before clobber.\n"),
+#ifdef ENABLE_THREADS
+    N_("\
+       --jobs                      specify how many threads use.\n"),
+#endif
     "\n",
-
     N_("\
 Directories:\n"),
     N_("\
@@ -1365,7 +1387,11 @@ for details.\n\n"));
       opt.always_rest = false;
     }
 
-  if (!nurl && !opt.input_filename)
+  if (!nurl && !opt.input_filename
+#ifdef ENABLE_METALINK
+       && !opt.metalink_file
+#endif
+     )
     {
       /* No URL specified.  */
       fprintf (stderr, _("%s: missing URL\n"), exec_name);
@@ -1588,6 +1614,53 @@ outputting to a regular file.\n"));
         }
     }
 
+#ifdef ENABLE_METALINK
+  if(opt.metalink_file)
+    {
+      /* --protocol-directories is the longest option among the ones checked below. */
+      char *temp_option = malloc(sizeof "--protocol-directories");
+
+      if(opt.user || opt.passwd || opt.http_user || opt.http_passwd ||
+         opt.ftp_user || opt.ftp_passwd || opt.ask_passwd)
+        {
+          fprintf (stderr, _("Username and password information not needed to \
+be specified when downloading from a metalink.\n"));
+          exit (WGET_EXIT_GENERIC_ERROR);
+        }
+      temp_option[0] = '\0';
+      if(opt.input_filename)
+        sprintf(temp_option, "-i");
+      else if(opt.output_document)
+        sprintf(temp_option, "-O");
+      else if(opt.base_href)
+        sprintf(temp_option, "--base");
+      else if(opt.force_html)
+        sprintf(temp_option, "--force-html");
+      else if(opt.always_rest)
+        sprintf(temp_option, "-c");
+      else if(opt.spider)
+        sprintf(temp_option, "-spider");
+      else if(opt.cut_dirs)
+        sprintf(temp_option, "--cut-dirs");
+      else if(opt.wait)
+        sprintf(temp_option, "-w");
+      else if(opt.waitretry != 10)
+        sprintf(temp_option, "--waitretry");
+      else if(opt.timestamping)
+        sprintf(temp_option, "--timestamping");
+      else if(opt.protocol_directories)
+          sprintf(temp_option, "--protocol-directories");
+      else if(opt.dirstruct)
+          sprintf(temp_option, "--force-directories");
+      if(temp_option[0])
+        {
+          fprintf (stderr, _("%s can't be used with --metalink.\n"), temp_option);
+          free(temp_option);
+          exit (WGET_EXIT_GENERIC_ERROR);
+        }
+    }
+#endif
+
 #ifdef __VMS
   /* Set global ODS5 flag according to the specified destination (if
      any), otherwise according to the current default device.
@@ -1661,7 +1734,7 @@ outputting to a regular file.\n"));
           else
           {
             retrieve_url (url_parsed, *t, &filename, &redirected_URL, NULL,
-                          &dt, opt.recursive, iri, true);
+                          &dt, opt.recursive, iri, true, NULL);
           }
 
           if (opt.delete_after && filename != NULL && file_exists_p (filename))
@@ -1683,6 +1756,7 @@ outputting to a regular file.\n"));
     {
       int count;
       int status;
+
       status = retrieve_from_file (opt.input_filename, opt.force_html, &count);
       inform_exit_status (status);
       if (!count)
@@ -1690,6 +1764,21 @@ outputting to a regular file.\n"));
                    opt.input_filename);
     }
 
+#ifdef ENABLE_METALINK
+  /* And then from the metalink file, if any.  */
+  if (opt.metalink_file)
+    {
+      int count;
+      int status;
+
+      status = retrieve_from_file (opt.metalink_file, opt.force_html, &count);
+      inform_exit_status (status);
+      if (!count)
+        logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"),
+                   opt.input_filename);
+    }
+#endif
+
   /* Print broken links. */
   if (opt.recursive && opt.spider)
     print_broken_links ();
@@ -1697,7 +1786,11 @@ outputting to a regular file.\n"));
   /* Print the downloaded sum.  */
   if ((opt.recursive || opt.page_requisites
        || nurl > 1
-       || (opt.input_filename && total_downloaded_bytes != 0))
+       || (opt.input_filename && total_downloaded_bytes != 0)
+#ifdef ENABLE_METALINK
+       || (opt.metalink_file && total_downloaded_bytes != 0)
+#endif
+       )
       &&
       total_downloaded_bytes != 0)
     {

src/metalink.c

@@ -0,0 +1,470 @@
+/* Declarations for HTTP.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software
+   Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#include "wget.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <metalink/metalink_parser.h>
+#include <metalink/metalink_types.h>
+
+#include "log.h"
+#include "md5.h"
+#include "sha1.h"
+#include "sha256.h"
+#include "metalink.h"
+#include "utils.h"
+
+
+#define HASH_TYPES 3
+/* Between MD5, SHA1 and SHA256, SHA256 has the greatest hash length, which is
+   32. In the line below, 64 is written to have a more readable code. */
+#define MAX_DIGEST_LENGTH 32
+
+static char supported_hashes[HASH_TYPES][7] = {"sha256", "sha1", "md5"};
+static int digest_sizes[HASH_TYPES] = {SHA256_DIGEST_SIZE, SHA1_DIGEST_SIZE, MD5_DIGEST_SIZE};
+static int (*hash_function[HASH_TYPES]) (FILE *, void *) = {sha256_stream, sha1_stream, md5_stream};
+
+/* First, parse the metalink using libmetalink functions and structures. Then
+   pass the information to an internal set of structures. */
+mlink *
+parse_metalink(char *input_file)
+{
+  int err;
+  metalink_t *metalink;
+  metalink_file_t **files;
+  metalink_resource_t **resources;
+  metalink_checksum_t **checksums;
+  metalink_chunk_checksum_t *chunk_checksum;
+  metalink_piece_hash_t **piece_hashes;
+  mlink *m_link;
+  err = metalink_parse_file (input_file, &metalink);
+  if(err != 0 || !metalink)
+    {
+      logprintf (LOG_VERBOSE, "Libmetalink could not parse the metalink file.\n");
+      return NULL;
+    }
+  else if(metalink->files == NULL) {
+    logprintf (LOG_VERBOSE, "PARSE METALINK: Metalink doesn't have any file data.\n");
+    metalink_delete(metalink);
+    return NULL;
+  }
+
+  m_link = malloc (sizeof *m_link);
+  m_link->identity = (metalink->identity ? xstrdup (metalink->identity) : NULL);
+  m_link->tags = (metalink->tags ? xstrdup (metalink->tags) : NULL);
+  m_link->files = NULL;
+  m_link->num_of_files = 0;
+
+  for (files = metalink->files; *files; ++files)
+    {
+      mlink_file *file;
+
+      if (!(*files)->name)
+        {
+          /* File name is missing */
+          logprintf (LOG_VERBOSE, "PARSE METALINK: Skipping file"
+                     " due to missing name/path.\n");
+          continue;
+        }
+      else if (!(*files)->resources)
+        {
+          /* URL is missing */
+          logprintf (LOG_VERBOSE, "PARSE METALINK: Skipping file(%s)"
+                     " due to missing resources.\n", (*files)->name);
+          continue;
+        }
+
+      file = malloc(sizeof(mlink_file));
+      ++(m_link->num_of_files);
+      file -> next = (m_link->files);
+      (m_link->files) = file;
+
+      file->name = xstrdup ((*files)->name);
+      file->size = (*files)->size;
+      file->maxconnections = (*files)->maxconnections;
+      file->version = ((*files)->version ? xstrdup ((*files)->version) : NULL);
+      file->language = ((*files)->language ? xstrdup ((*files)->language) : NULL);
+      file->os = ((*files)->os ? xstrdup ((*files)->os) : NULL);
+      file->resources = NULL;
+      file->checksums = NULL;
+      file->chunk_checksum = NULL;
+      file->num_of_res = file->num_of_checksums = 0;
+
+      for (resources = (*files)->resources; *resources; ++resources)
+        {
+          mlink_resource *resource;
+
+          if (!(*resources)->url)
+            {
+              logprintf (LOG_VERBOSE, "PARSE METALINK: Skipping resource"
+                         " due to missing URL.\n");
+              continue;
+            }
+
+          resource = malloc (sizeof(mlink_resource));
+          ++(file->num_of_res);
+
+          resource->url = xstrdup ((*resources)->url);
+          resource->type = ((*resources)->type ? xstrdup ((*resources)->type) : NULL);
+          resource->location = ((*resources)->location ? xstrdup ((*resources)->location) : NULL);
+          resource->preference = (*resources)->preference;
+          resource->maxconnections = (*resources)->maxconnections;
+
+          resource->next = (file->resources);
+          (file->resources) = resource;
+        }
+
+      for (checksums = (*files)->checksums; *checksums; ++checksums)
+        {
+          mlink_checksum *checksum = malloc (sizeof(mlink_checksum));
+
+          if (!(*checksums)->type)
+            {
+              logprintf (LOG_VERBOSE, "PARSE METALINK: Skipping checksum"
+                         " due to missing hash type.\n");
+              continue;
+            }
+          else if (!(*checksums)->hash)
+            {
+              logprintf (LOG_VERBOSE, "PARSE METALINK: Skipping resource"
+                         " due to missing hash value.\n");
+              continue;
+            }
+
+          checksum->type = ((*checksums)->type ? xstrdup ((*checksums)->type) : NULL);
+          checksum->hash = ((*checksums)->hash ? xstrdup ((*checksums)->hash) : NULL);
+
+          checksum->next = (file->checksums);
+          (file->checksums) = checksum;
+        }
+
+      if((chunk_checksum = (*files)->chunk_checksum))
+        {
+          mlink_chunk_checksum *chunk_sum;
+
+          if(!chunk_checksum->type)
+            logprintf (LOG_VERBOSE, "PARSE METALINK: Skipping chunk checksum"
+                       " due to missing type information.\n");
+          else
+            {
+              chunk_sum = malloc (sizeof(mlink_chunk_checksum));
+              chunk_sum->length = chunk_checksum->length;
+              chunk_sum->type = (chunk_checksum->type ? xstrdup (chunk_checksum->type) : NULL);
+              for (piece_hashes = chunk_checksum->piece_hashes; *piece_hashes; ++piece_hashes)
+                {
+                  mlink_piece_hash piece_hash;
+                  if(!chunk_checksum->type)
+                    {
+                      logprintf (LOG_VERBOSE, "PARSE METALINK: Skipping chunk checksum"
+                                 " due to missing hash value for piece(%d).\n",
+                                 (*piece_hashes)->piece);
+                      free (chunk_sum);
+                      break;
+                    }
+                }
+            }
+        }
+    }
+
+  metalink_delete(metalink);
+  return m_link;
+}
+
+/* Elect resouces so that only the URLs with type HTTP and FTP (i.e. the
+   protocols supported by Metalink&GNU Wget) remain on the list of resources. */
+void
+elect_resources (mlink *mlink)
+{
+  mlink_file *file;
+  mlink_resource *res, *res_next;
+
+  for (file = mlink->files; file; file = file->next)
+    {
+      res = file->resources;
+      if (!res)
+        continue;
+
+      while (res_next = res->next)
+        {
+          if (strcmp(res_next->type, "ftp") && strcmp(res_next->type, "http"))
+            {
+              res->next = res_next->next;
+              free(res_next);
+            }
+          else
+            res = res_next;
+        }
+      res = file->resources;
+      if (strcmp(res->type, "ftp") && strcmp(res->type, "http"))
+        {
+          file->resources = res->next;
+          free(res);
+        }
+    }
+}
+
+/* Elect checksums so that only the hashes with types MD5, SHA-1 or SHA-256
+   (i.e. the hashes supported by Metalink) remain on the list of checksums. */
+void
+elect_checksums (mlink *mlink)
+{
+  int i;
+  mlink_file *file = mlink -> files;
+  mlink_checksum *csum, *csum_next;
+
+  for (; file; file = file->next)
+    {
+      csum = file->checksums;
+      if (!csum)
+        continue;
+
+      while (csum_next = csum->next)
+        {
+          /* Traverse supported hash types & break if csum->type is the same. */
+          for (i = 0; i < HASH_TYPES; ++i)
+            if (!strcmp(csum_next->type, supported_hashes[i]))
+              break;
+
+          if(i == HASH_TYPES)
+            {
+              csum->next = csum_next->next;
+              free(csum_next);
+            }
+          else
+            csum=csum_next;
+        }
+      csum = file->checksums;
+      for (i = 0; i < HASH_TYPES; ++i)
+        if (!strcmp(csum->type, supported_hashes[i]))
+          break;
+      if(i == HASH_TYPES)
+        {
+          file->checksums = csum->next;
+          free(csum);
+        }
+    }
+}
+
+/* Free the space allocated for the whole mlink structure. */
+void
+delete_mlink(mlink *metalink)
+{
+  mlink_file *file, *file_temp;
+  mlink_resource *res, *res_temp;
+  mlink_checksum *csum, *csum_temp;
+  mlink_piece_hash *phash, *phash_temp;
+
+  if(!metalink)
+    return;
+
+  xfree_null (metalink->tags);
+  xfree_null (metalink->identity);
+
+  file = metalink->files;
+  while (file)
+    {
+      xfree_null(file->os);
+      xfree_null(file->language);
+      xfree_null(file->version);
+      xfree_null(file->name);
+
+      res = file->resources;
+      while (res)
+        {
+          xfree_null (res->url);
+          xfree_null (res->type);
+          xfree_null (res->location);
+
+          res_temp = res;
+          res = res->next;
+          free (res_temp);
+        }
+
+      csum = file->checksums;
+      while (csum)
+        {
+          xfree_null (csum->type);
+          xfree_null (csum->hash);
+
+          csum_temp = csum;
+          csum = csum->next;
+          free (csum_temp);
+        }
+
+      if(file->chunk_checksum)
+        {
+          free (file->chunk_checksum->type);
+          phash = file->chunk_checksum->piece_hashes;
+          while (phash)
+            {
+              xfree_null (phash->hash);
+
+              phash_temp = phash;
+              phash = phash->next;
+              free (phash_temp);
+            }
+        }
+
+      file_temp = file;
+      file = file->next;
+      free (file_temp);
+    }
+  free (metalink);
+}
+
+/* Parses metalink into type metalink_t and returns a pointer to it.
+   Returns NULL if the parsing is failed. */
+metalink_t*
+metalink_context (const char *url)
+{
+  metalink_error_t err;
+  metalink_t* metalink;
+
+  err = metalink_parse_file (url, &metalink);
+
+  if(err != 0)
+      metalink = NULL;
+  return metalink;
+}
+
+/* It should be taken into account that file hashes in metalink files may
+   include uppercase letter. This function turns the case of the first length
+   letters in the space pointed by hash into lowercase. */
+static void
+lower_hex_case (unsigned char *hash, int length)
+{
+  int i;
+
+  /* 32 is the difference between the ascii codes of 'a' and 'A'. */
+  for(i = 0; i < length; ++i)
+    if('A' <= hash[i] && hash[i] <= 'Z')
+      hash[i] += 32;
+}
+
+/* Verifies file hash by comparing the file hashes found by gnulib functions
+   and hashes provided by metalink file. Works by comparing strongest supported
+   hash type available in the metalink file.
+
+   Returns;
+   -1      if hashes that were compared turned out to be different.
+    0      if all pairs of hashes compared turned out to be the same.
+    1      if due to some error, comparisons could not be made.  */
+int
+verify_file_hash (const char *filename, mlink_checksum *checksums)
+{
+  int i, j, req_type, res = 0;
+
+  unsigned char hash_raw[MAX_DIGEST_LENGTH];
+  /* Points to a hash of supported type from the metalink file. The index dedicated
+     to a type is inversely proportional to its strength. (check supported_types
+     to see the supported hash types listed in decreasing order of strength)*/
+  unsigned char *metalink_hashes[HASH_TYPES];
+  unsigned char file_hash[2 * MAX_DIGEST_LENGTH + 1];
+  FILE *file;
+  mlink_checksum *checksum;
+
+  if (!checksums)
+    {
+      /* Metalink file has no hashes for this file. */
+      logprintf (LOG_VERBOSE, "Validating(%s) failed: digest missing in metalink file.\n",
+                 filename);
+      return 1;
+    }
+
+  for (i = 0; i < HASH_TYPES; ++i)
+    metalink_hashes[i] = NULL;
+
+  /* Fill metalink_hashes to contain an instance of supported types of hashes. */
+  for (checksum = checksums; checksum; checksum = checksum->next)
+    for (j = 0; j < HASH_TYPES; ++j)
+      if (!strcmp(checksum->type, supported_hashes[j]))
+        {
+          if(metalink_hashes[j])
+            {
+              /* As of libmetalin-0.03, it is not checked during parsing the
+                 information in the metalink file whether there are multiple
+                 hashes of same type for one file. That case should be checked,
+                 as none of those hashes can be trusted above the other. */
+              logprintf (LOG_VERBOSE, "Validating(%s) failed: metalink file contains different hashes of same type.\n",
+                         filename);
+              return 1;
+            }
+          else
+            metalink_hashes[j] = checksum->hash;
+        }
+
+  for (i = 0; !metalink_hashes[i]; ++i);
+
+  if (i == HASH_TYPES)
+    {
+      /* no hash of supported types could be found. */
+      logprintf (LOG_VERBOSE, "Validating(%s) failed: No hash of supported types could be found in metalink file.\n",
+                 filename);
+      return 1;
+    }
+  req_type = i;
+
+  if (!(file = fopen(filename, "rb")))
+    {
+      /* File could not be opened. */
+      logprintf (LOG_VERBOSE, "Validating(%s) failed: file could not be opened.\n",
+                 filename);
+      return 1;
+    }
+
+  res = (*hash_function[req_type]) (file, hash_raw);
+  fclose(file);
+
+  /* Find file hash accordingly. */
+  if (res)
+    {
+      logprintf (LOG_VERBOSE, "Validating(%s) failed: File hash could not be found.\n",
+                 filename);
+      return 1;
+    }
+
+  /* Turn byte-form hash to hex form. */
+  for(j = 0 ; j < digest_sizes[req_type]; ++j)
+    sprintf(file_hash + 2 * j, "%02x", hash_raw[j]);
+
+  lower_hex_case(metalink_hashes[req_type], 2 * digest_sizes[req_type]);
+  if (strcmp(metalink_hashes[req_type], file_hash))
+    {
+      logprintf (LOG_VERBOSE, "Verifying(%s) failed: %s hashes are different.\n",
+                 filename, supported_hashes[i]);
+      return -1;
+    }
+
+  logprintf (LOG_VERBOSE, "Verifying(%s): %s hashes are the same.\n",
+             filename, supported_hashes[i]);
+  return 0;
+}

src/metalink.h

@@ -0,0 +1,106 @@
+/* Declarations for metalink.c.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#ifndef MLINK_H
+#define MLINK_H
+
+typedef struct metalink_piece_hash
+{
+  struct metalink_piece_hash *next;
+
+  int piece;
+  char *hash;
+} mlink_piece_hash;
+
+typedef struct metalink_checksum
+{
+  struct metalink_checksum *next;
+
+  char *type;
+  char *hash;
+} mlink_checksum;
+
+typedef struct metalink_resource
+{
+  struct metalink_resource *next;
+
+  char *url;
+  char *type;
+  char *location;
+  int preference;
+  int maxconnections;
+} mlink_resource;
+
+typedef struct
+{
+  char *type;
+  int length;
+  mlink_piece_hash *piece_hashes;
+} mlink_chunk_checksum;
+
+typedef struct metalink_file
+{
+  struct metalink_file *next;
+
+  char *name;
+  long long int size;
+  char *version;
+  char *language;
+  char *os;
+  int maxconnections;
+  mlink_resource *resources;
+  int num_of_res;
+  mlink_checksum *checksums;
+  int num_of_checksums;
+  mlink_chunk_checksum *chunk_checksum;
+} mlink_file;
+
+typedef struct
+{
+  char *identity;
+  char *tags;
+  mlink_file *files;
+  int num_of_files;
+} mlink;
+
+mlink *parse_metalink (char *);
+
+void elect_resources (mlink *);
+
+void elect_checksums (mlink *);
+
+void delete_mlink (mlink *);
+
+metalink_t *metalink_context (const char *);
+
+int verify_file_hash (const char *, mlink_checksum *);
+
+#endif /* MLINK_H */

src/multi.c

@@ -0,0 +1,239 @@
+/* Declarations for HTTP.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software
+   Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#include "wget.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <unistd.h>
+
+#include "multi.h"
+#include "url.h"
+#include "exits.h"
+
+static struct range *ranges;
+char **files;
+
+/*  Allocate space for temporary file names. */
+void
+init_temp_files()
+{
+  int i;
+
+  if(!(files = malloc (opt.jobs * (sizeof *files))))
+    {
+      logprintf (LOG_VERBOSE, "Space for temporary file data could not be allocated.\n");
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+  for (i = 0; i < opt.jobs; ++i)
+    if(!(files[i] = malloc (L_tmpnam * sizeof(char))))
+      {
+        logprintf (LOG_VERBOSE, "Space for temporary file names could not be allocated.\n");
+        exit (WGET_EXIT_GENERIC_ERROR);
+      }
+}
+
+/*  Assign names to temporary files to be used. */
+void
+name_temp_files()
+{
+  int i;
+
+  for (i = 0; i < opt.jobs; ++i)
+    if(!tmpnam(files[i]))
+      {
+        logprintf (LOG_VERBOSE, "Temporary file name could not be assigned.\n");
+        exit (WGET_EXIT_GENERIC_ERROR);
+      }
+}
+
+/*  Merge the temporary files in which the chunks are stored to form the
+    resulting file(output). */
+void
+merge_temp_files(char *output)
+{
+  FILE *out, *in;
+  int j, ret;
+  void *buf = malloc (MIN_CHUNK_SIZE);
+
+  out = fopen (output, "wb");
+  for(j = 0; j < opt.jobs; ++j)
+    {
+      in = fopen(files[j],"rb");
+      ret = MIN_CHUNK_SIZE;
+      while(ret == MIN_CHUNK_SIZE)
+        {
+          ret = fread(buf, 1, MIN_CHUNK_SIZE, in);
+          fwrite(buf, 1, ret, out);
+        }
+      fclose(in);
+    }
+  fclose(out);
+  free(buf);
+}
+
+/* Delete the temporary files used. */
+void
+delete_temp_files()
+{
+  int j = 0;
+
+  while(j < opt.jobs)
+    unlink(files[j++]);
+}
+
+/* Clean the space allocated for temporary files data. */
+void
+clean_temp_files()
+{
+  int i;
+
+  for (i = 0; i < opt.jobs; ++i)
+    free (files[i]);
+  free(files);
+}
+
+/* Allocate ranges array to store the ranges data. */
+void
+init_ranges()
+{
+  if(!(ranges = malloc (opt.jobs * (sizeof *ranges))))
+    {
+      logprintf (LOG_VERBOSE, "Space for ranges data could not be allocated.\n");
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+}
+
+/* Assign values to the ranges.
+   Also allocates the resources array each struct range must have.
+
+   Returns the number of ranges to which values are assigned. */
+int
+fill_ranges_data(int num_of_resources, long long int file_size,
+                 long int chunk_size)
+{
+  int i, r;
+  i = 0;
+  do
+  {
+      ranges[i].first_byte = i * chunk_size;
+      ranges[i].last_byte = (i+1) * chunk_size - 1;
+      ranges[i].bytes_covered = ranges[i].is_assigned = 0;
+      ranges[i].resources = malloc(num_of_resources * sizeof(bool));
+      ranges[i].status_least_severe = RETROK;
+      for (r = 0; r < num_of_resources; ++r)
+        ranges[i].resources[r] = false;
+      ++i;
+    } while (ranges[i-1].last_byte < (file_size - 1));
+  ranges[i-1].last_byte = file_size -1;
+
+  return i;
+}
+
+/* Free the resources array of each range allocated by fill_ranges_data(). */
+void
+clean_range_res_data()
+{
+  int i;
+  for (i = 0; i < opt.jobs; ++i)
+    free (ranges[i].resources);
+}
+
+/* Free the ranges array that is used for storing ranges' data. */
+void
+clean_ranges()
+{
+  free (ranges);
+  ranges = NULL;
+}
+
+/* Assign 'last minute' data to struct s_thread_ctx instances regarding their
+   usage and range information. Then create a thread using that instance. */
+int
+spawn_thread (struct s_thread_ctx *thread_ctx, int index, int resource)
+{
+  static pthread_t thread;
+
+  thread_ctx[index].url_parsed = url_parse (thread_ctx[index].url,
+                       &(thread_ctx[index].url_err), thread_ctx[index].i, true);
+  if(!thread_ctx[index].url_parsed)
+    return 1;
+
+  thread_ctx[index].file = files[index];
+  thread_ctx[index].range = ranges + index;
+  (thread_ctx[index].range)->is_assigned = 1;
+  (thread_ctx[index].range)->resources[resource] = true;
+
+  thread_ctx[index].used = 1;
+  thread_ctx[index].terminated = 0;
+
+  return pthread_create (&thread, NULL, segmented_retrieve_url, &thread_ctx[index]);
+}
+
+/* Collects the first thread to terminate and updates struct s_thread_ctx
+   instance's data regarding its 'business' (i.e. being used by a thread).
+
+   Returns the index of the struct s_thread_ctx instance that was used in the
+   terminating thread. */
+int
+collect_thread (sem_t *retr_sem, struct s_thread_ctx *thread_ctx)
+{
+  int k, ret;
+  do
+    ret = sem_wait (retr_sem);
+  while (ret < 0 && errno == EINTR);
+
+  for (k = 0; k < opt.jobs; k++)
+    if (thread_ctx[k].used && thread_ctx[k].terminated)
+      {
+        url_free (thread_ctx[k].url_parsed);
+        thread_ctx[k].used = 0;
+        (thread_ctx[k].range)->is_assigned = 0;
+        return k;
+      }
+}
+
+/* The function which is being called by pthread_create in spawn_thread(). It
+   is used to call retrieve_url(), which requires many arguments. */
+static void *
+segmented_retrieve_url (void *arg)
+{
+  struct s_thread_ctx *ctx = (struct s_thread_ctx *) arg;
+
+  ctx->status = retrieve_url (ctx->url_parsed, ctx->url,
+                              &ctx->file, &ctx->redirected,
+                              ctx->referer, &ctx->dt,
+                              false, ctx->i, true, ctx->range);
+  ctx->terminated = 1;
+  sem_post (ctx->retr_sem);
+}

src/multi.h

@@ -0,0 +1,89 @@
+/* Declarations for HTTP.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
+   Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#ifndef MULTI_H
+#define MULTI_H
+
+#include <semaphore.h>
+
+#include "wget.h"
+
+#include "iri.h"
+#include "url.h"
+
+#define MIN_CHUNK_SIZE 2048
+
+struct s_thread_ctx
+{
+  pthread_t thread;
+  int used;
+  int terminated;
+  int dt, url_err;
+  char *redirected;
+  char *referer;
+  struct url *url_parsed;
+  struct iri *i;
+  struct range *range;
+  char *file;
+  char *url;
+#ifdef ENABLE_THREADS
+  sem_t *retr_sem;
+#else
+  /* Not used.  */
+  void *retr_sem;
+#endif
+  uerr_t status;
+};
+
+void init_temp_files();
+
+void name_temp_files();
+
+void merge_temp_files(char *);
+
+void delete_temp_files();
+
+void clean_temp_files();
+
+void init_ranges();
+
+int fill_ranges_data(int, long long int, long int);
+
+void clean_range_res_data();
+
+void clean_ranges();
+
+int spawn_thread (struct s_thread_ctx*, int, int);
+
+int collect_thread (sem_t *, struct s_thread_ctx *);
+
+static void * segmented_retrieve_url (void *);
+
+#endif /* MULTI_H */

src/options.h

@@ -35,6 +35,11 @@ struct options
                                    hence not boolean.) */
   bool quiet;                   /* Are we quiet? */
   int ntry;                     /* Number of tries per URL */
+#ifdef ENABLE_METALINK
+  int n_retries;                /* Number of tries per file */
+#endif
+  char *metalink_file;          /* Metalink filename */
+
   bool retry_connrefused;       /* Treat CONNREFUSED as non-fatal. */
   bool background;              /* Whether we should work in background. */
   bool ignore_length;           /* Do we heed content-length at all?  */
@@ -285,6 +290,8 @@ struct options
   bool show_all_dns_entries;    /* Show all the DNS entries when resolving a
                                    name. */
   bool report_bps;              /*Output bandwidth in bits format*/
+
+  int jobs;                 /* How many threads use at the same time.  */
 };
 
 extern struct options opt;

src/progress.c

@@ -36,18 +36,24 @@ as that of the covered work.  */
 #include <assert.h>
 #include <unistd.h>
 #include <signal.h>
+#include <time.h>
 #include <wchar.h>
 
+#ifdef ENABLE_THREADS
+#include <pthread.h>
+#endif
+
 #include "progress.h"
 #include "utils.h"
 #include "retr.h"
+#include "ptimer.h"
 
 struct progress_implementation {
   const char *name;
   bool interactive;
   void *(*create) (const char *, wgint, wgint);
   void (*update) (void *, wgint, double);
-  void (*draw) (void *);
+  void (*draw) (void *, bool);
   void (*finish) (void *, double);
   void (*set_params) (char *);
 };
@@ -57,15 +63,33 @@ struct progress_implementation {
 static void *dot_create (const char *, wgint, wgint);
 static void dot_update (void *, wgint, double);
 static void dot_finish (void *, double);
-static void dot_draw (void *);
+static void dot_draw (void *, bool);
 static void dot_set_params (char *);
 
 static void *bar_create (const char *, wgint, wgint);
 static void bar_update (void *, wgint, double);
-static void bar_draw (void *);
+static void bar_draw (void *, bool);
 static void bar_finish (void *, double);
 static void bar_set_params (char *);
 
+struct progress_header
+{
+  struct progress_header *next;
+};
+
+static struct progress_header *progress_list = NULL;
+static struct progress_header *current_progress = NULL;
+
+#ifdef ENABLE_THREADS
+static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
+# define LOCK_PROGRESS() pthread_mutex_lock (&progress_mutex);
+# define UNLOCK_PROGRESS() pthread_mutex_unlock (&progress_mutex);
+#else
+# define LOCK_PROGRESS()
+# define UNLOCK_PROGRESS()
+#endif
+
+
 static struct progress_implementation implementations[] = {
   { "dot", 0, dot_create, dot_update, dot_draw, dot_finish, dot_set_params },
   { "bar", 1, bar_create, bar_update, bar_draw, bar_finish, bar_set_params }
@@ -86,6 +110,37 @@ static int current_impl_locked;
 
 #define FALLBACK_PROGRESS_IMPLEMENTATION "dot"
 
+
+/* Time between screen refreshes will not be shorter than this, so
+   that Wget doesn't swamp the TTY with output.  */
+#define REFRESH_INTERVAL 0.2
+
+/* Time between screen refreshes the file to show when there are multiple
+   downloads at the same time.  */
+#define REFRESH_SWITCH_FILE 1.5
+
+/* Assumed screen width if we can't find the real value.  */
+#define DEFAULT_SCREEN_WIDTH 80
+
+/* Minimum screen width we'll try to work with.  If this is too small,
+   create_image will overflow the buffer.  */
+#define MINIMUM_SCREEN_WIDTH 45
+
+/* The last known screen width.  This can be updated by the code that
+   detects that SIGWINCH was received (but it's never updated from the
+   signal handler).  */
+static int screen_width;
+
+/* A flag that, when set, means SIGWINCH was received.  */
+static volatile sig_atomic_t received_sigwinch;
+
+static int bp_width;            /* screen width we're using at the
+                                   time the progress gauge was
+                                   created.  this is different from
+                                   the screen_width global variable in
+                                   that the latter can be changed by a
+                                   signal. */
+
 /* Return true if NAME names a valid progress bar implementation.  The
    characters after the first : will be ignored.  */
 
@@ -155,6 +210,7 @@ void *
 progress_create (const char *f_download, wgint initial, wgint total)
 {
   /* Check if the log status has changed under our feet. */
+  struct progress_header *ret;
   if (output_redirected)
     {
       if (!current_impl_locked)
@@ -162,7 +218,17 @@ progress_create (const char *f_download, wgint initial, wgint total)
       output_redirected = 0;
     }
 
-  return current_impl->create (f_download, initial, total);
+  ret = current_impl->create (f_download, initial, total);
+  if (ret)
+    {
+      LOCK_PROGRESS ();
+
+      ret->next = progress_list;
+      progress_list = ret;
+
+      UNLOCK_PROGRESS ();
+    }
+  return ret;
 }
 
 /* Return true if the progress gauge is "interactive", i.e. if it can
@@ -182,8 +248,50 @@ progress_interactive_p (void *progress _GL_UNUSED)
 void
 progress_update (void *progress, wgint howmuch, double dltime)
 {
+  static struct ptimer *last_draw = NULL;
+  static struct ptimer *last_switch = NULL;
+  bool force_screen_update = false;
+
   current_impl->update (progress, howmuch, dltime);
-  current_impl->draw (progress);
+  LOCK_PROGRESS ();
+
+  if (last_draw == NULL)
+    last_draw = ptimer_new ();
+  if (last_switch == NULL)
+    last_switch = ptimer_new ();
+
+  /* If SIGWINCH (the window size change signal) been received,
+     determine the new screen size and update the screen.  */
+  if (received_sigwinch)
+    {
+      int old_width = screen_width;
+      screen_width = determine_screen_width ();
+      if (!screen_width)
+        screen_width = DEFAULT_SCREEN_WIDTH;
+      else if (screen_width < MINIMUM_SCREEN_WIDTH)
+        screen_width = MINIMUM_SCREEN_WIDTH;
+      if (screen_width != old_width)
+        force_screen_update = true;
+      received_sigwinch = 0;
+    }
+
+  if (ptimer_measure (last_draw) >= REFRESH_INTERVAL || force_screen_update)
+    {
+      if (current_progress == NULL)
+        current_progress = progress_list;
+
+      bp_width = screen_width - 1;
+      current_impl->draw (current_progress, force_screen_update);
+      ptimer_reset (last_draw);
+
+      if (ptimer_measure (last_switch) >= REFRESH_SWITCH_FILE)
+        {
+          ptimer_reset (last_switch);
+          current_progress = current_progress->next;
+        }
+    }
+
+  UNLOCK_PROGRESS ();
 }
 
 /* Tell the progress gauge to clean up.  Calling this will free the
@@ -192,12 +300,38 @@ progress_update (void *progress, wgint howmuch, double dltime)
 void
 progress_finish (void *progress, double dltime)
 {
+  struct progress_header *header = progress;
+    {
+      struct progress_header *it, *prev = NULL;
+
+      LOCK_PROGRESS ();
+
+      for (it = progress_list; it; it = it->next)
+        {
+          if (it == progress)
+            {
+              if (current_progress == progress)
+                current_progress = it->next;
+
+              if (it == progress_list)
+                progress_list = progress_list->next;
+              else
+                prev->next = it->next;
+              break;
+            }
+          prev = it;
+        }
+
+      UNLOCK_PROGRESS ();
+    }
   current_impl->finish (progress, dltime);
 }
 
 /* Dot-printing. */
 
 struct dot_progress {
+  struct progress_header header;
+
   wgint initial_length;         /* how many bytes have been downloaded
                                    previously. */
   wgint total_length;           /* expected total byte count when the
@@ -352,12 +486,16 @@ dot_update (void *progress, wgint howmuch, double dltime)
 }
 
 static void
-dot_draw (void *progress)
+dot_draw (void *progress, bool force)
 {
   struct dot_progress *dp = progress;
   int dot_bytes = opt.dot_bytes;
   wgint ROW_BYTES = opt.dot_bytes * opt.dots_in_line;
 
+  /* FIXME: support threads.  */
+  if (opt.jobs > 1)
+    return;
+
   log_set_flush (false);
 
   for (; dp->accumulated >= dot_bytes; dp->accumulated -= dot_bytes)
@@ -467,21 +605,6 @@ dot_set_params (char *params)
 
 /* "Thermometer" (bar) progress. */
 
-/* Assumed screen width if we can't find the real value.  */
-#define DEFAULT_SCREEN_WIDTH 80
-
-/* Minimum screen width we'll try to work with.  If this is too small,
-   create_image will overflow the buffer.  */
-#define MINIMUM_SCREEN_WIDTH 45
-
-/* The last known screen width.  This can be updated by the code that
-   detects that SIGWINCH was received (but it's never updated from the
-   signal handler).  */
-static int screen_width;
-
-/* A flag that, when set, means SIGWINCH was received.  */
-static volatile sig_atomic_t received_sigwinch;
-
 /* Size of the download speed history ring. */
 #define DLSPEED_HISTORY_SIZE 20
 
@@ -496,15 +619,12 @@ static volatile sig_atomic_t received_sigwinch;
    download speeds are scratched.  */
 #define STALL_START_TIME 5
 
-/* Time between screen refreshes will not be shorter than this, so
-   that Wget doesn't swamp the TTY with output.  */
-#define REFRESH_INTERVAL 0.2
-
 /* Don't refresh the ETA too often to avoid jerkiness in predictions.
    This allows ETA to change approximately once per second.  */
 #define ETA_REFRESH_INTERVAL 0.99
 
 struct bar_progress {
+  struct progress_header header;
   const char *f_download;       /* Filename of the downloaded file */
   wgint initial_length;         /* how many bytes have been downloaded
                                    previously. */
@@ -512,19 +632,8 @@ struct bar_progress {
                                    download finishes */
   wgint count;                  /* bytes downloaded so far */
 
-  double last_screen_update;    /* time of the last screen update,
-                                   measured since the beginning of
-                                   download. */
-
   double dltime;                /* download time so far */
-  int width;                    /* screen width we're using at the
-                                   time the progress gauge was
-                                   created.  this is different from
-                                   the screen_width global variable in
-                                   that the latter can be changed by a
-                                   signal. */
-  char *buffer;                 /* buffer where the bar "image" is
-                                   stored. */
+
   int tick;                     /* counter used for drawing the
                                    progress bar where the total size
                                    is not known. */
@@ -559,7 +668,7 @@ struct bar_progress {
   int last_eta_value;
 };
 
-static void create_image (struct bar_progress *, double, bool);
+static void create_image (struct bar_progress *, char *buffer, double, bool);
 static void display_image (char *);
 
 static void *
@@ -589,15 +698,11 @@ bar_create (const char *f_download, wgint initial, wgint total)
     }
 
   /* - 1 because we don't want to use the last screen column. */
-  bp->width = screen_width - 1;
-  /* + enough space for the terminating zero, and hopefully enough room
-   * for multibyte characters. */
-  bp->buffer = xmalloc (bp->width + 100);
+  bp_width = screen_width - 1;
 
   logputs (LOG_VERBOSE, "\n");
 
-  create_image (bp, 0, false);
-  display_image (bp->buffer);
+  bar_draw (bp, false);
 
   return bp;
 }
@@ -624,37 +729,20 @@ bar_update (void *progress, wgint howmuch, double dltime)
 }
 
 static void
-bar_draw (void *progress)
+bar_draw (void *progress, bool force)
 {
-  bool force_screen_update = false;
   struct bar_progress *bp = progress;
+  static char *buffer = NULL;
 
-  /* If SIGWINCH (the window size change signal) been received,
-     determine the new screen size and update the screen.  */
-  if (received_sigwinch)
+  if (buffer == NULL)
+    buffer = xmalloc (bp_width + 100);
+
+  if (force)
     {
-      int old_width = screen_width;
-      screen_width = determine_screen_width ();
-      if (!screen_width)
-        screen_width = DEFAULT_SCREEN_WIDTH;
-      else if (screen_width < MINIMUM_SCREEN_WIDTH)
-        screen_width = MINIMUM_SCREEN_WIDTH;
-      if (screen_width != old_width)
-        {
-          bp->width = screen_width - 1;
-          bp->buffer = xrealloc (bp->buffer, bp->width + 100);
-          force_screen_update = true;
-        }
-      received_sigwinch = 0;
+      buffer = xrealloc (buffer, bp_width + 100);
     }
-
-  if (bp->dltime - bp->last_screen_update < REFRESH_INTERVAL && !force_screen_update)
-    /* Don't update more often than five times per second. */
-    return;
-
-  create_image (bp, bp->dltime, false);
-  display_image (bp->buffer);
-  bp->last_screen_update = bp->dltime;
+  create_image (bp, buffer, bp->dltime, false);
+  display_image (buffer);
 }
 
 static void
@@ -667,13 +755,11 @@ bar_finish (void *progress, double dltime)
     /* See bar_update() for explanation. */
     bp->total_length = bp->initial_length + bp->count;
 
-  create_image (bp, dltime, true);
-  display_image (bp->buffer);
+  bar_draw (bp, false);
 
   logputs (LOG_VERBOSE, "\n");
   logputs (LOG_PROGRESS, "\n");
 
-  xfree (bp->buffer);
   xfree (bp);
 }
 
@@ -871,10 +957,10 @@ get_eta (int *bcd)
 #endif
 
 static void
-create_image (struct bar_progress *bp, double dl_total_time, bool done)
+create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool done)
 {
-  const int MAX_FILENAME_LEN = bp->width / 4;
-  char *p = bp->buffer;
+  const int MAX_FILENAME_LEN = bp_width / 4;
+  char *p = buffer;
   wgint size = bp->initial_length + bp->count;
 
   const char *size_grouped = with_thousand_seps (size);
@@ -898,6 +984,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
 
      "file "           - Downloaded filename      - MAX_FILENAME_LEN chars + 1
      "xx% " or "100%"  - percentage               - 4 chars
+     " url "           - string                   - MAX MAX_URL_LEN chars + 2
      "[]"              - progress bar decorations - 2 chars
      " nnn.nnK"        - downloaded bytes         - 7 chars + 1
      " 12.5KB/s"       - download rate            - 8 chars + 1
@@ -913,7 +1000,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
 #define PROGRESS_DWNLOAD_RATE  8 + 1
 #define PROGRESS_ETA_LEN       14
 
-  int progress_size = bp->width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
+  int progress_size = bp_width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
                                    PROGRESS_DECORAT_LEN + PROGRESS_FILESIZE_LEN +
                                    PROGRESS_DWNLOAD_RATE + PROGRESS_ETA_LEN);
 
@@ -1018,8 +1105,8 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
         }
       *p++ = ']';
 
+      ++bp->tick;
     }
- ++bp->tick;
 
   /* " 234.56M" */
   const char * down_size = human_readable (size, 1000, 2);
@@ -1122,7 +1209,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       move_to_end (p);
     }
 
-  while (p - bp->buffer - bytes_cols_diff - size_grouped_diff < bp->width)
+  while (p - buffer - bytes_cols_diff - size_grouped_diff < bp_width)
     *p++ = ' ';
   *p = '\0';
 }

src/recur.c

@@ -37,6 +37,12 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <errno.h>
 #include <assert.h>
+#ifdef ENABLE_THREADS
+#include <pthread.h>
+#include <semaphore.h>
+#endif
+
+#include "multi.h"
 
 #include "url.h"
 #include "recur.h"
@@ -165,6 +171,27 @@ static bool download_child_p (const struct urlpos *, struct url *, int,
 static bool descend_redirect_p (const char *, struct url *, int,
                                 struct url *, struct hash_table *, struct iri *);
 
+#if !ENABLE_THREADS
+# define THREAD_JOIN(...)  (0)
+# define SEM_INIT(...)     (0)
+# define SEM_WAIT(...)     (0)
+#else
+# define THREAD_JOIN pthread_join
+# define SEM_INIT sem_init
+# define SEM_WAIT sem_wait
+static void *
+start_retrieve_url (void *arg)
+{
+  struct s_thread_ctx *ctx = (struct s_thread_ctx *) arg;
+
+  ctx->status = retrieve_url (ctx->url_parsed, ctx->url,
+                              &ctx->file, &ctx->redirected,
+                              ctx->referer, &ctx->dt,
+                              false, ctx->i, true, NULL);
+  ctx->terminated = 1;
+  sem_post (ctx->retr_sem);
+}
+#endif
 
 /* Retrieve a part of the web beginning with START_URL.  This used to
    be called "recursive retrieval", because the old function was
@@ -191,6 +218,21 @@ uerr_t
 retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 {
   uerr_t status = RETROK;
+  struct s_thread_ctx *thread_ctx;
+  char *next_url = NULL, *next_referer;
+  int next_depth;
+  bool next_html_allowed, next_css_allowed;
+  struct iri *next_i = NULL;
+
+#ifdef ENABLE_THREADS
+  const int N_THREADS = opt.jobs > 0 ? opt.jobs : 1;
+  sem_t retr_sem;
+#else
+  const int N_THREADS = 1;
+  void *retr_sem;
+#endif
+
+  int free_threads = N_THREADS;
 
   /* The queue of URLs we need to load. */
   struct url_queue *queue;
@@ -213,6 +255,11 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
     set_uri_encoding (i, opt.locale, true);
 #undef COPYSTR
 
+  thread_ctx = calloc (N_THREADS, sizeof *thread_ctx);
+
+  /* FIXME: CHECK FOR ERRORS.  */
+  SEM_INIT (&retr_sem, 0, 0);
+
   queue = url_queue_new ();
   blacklist = make_string_hash_table (0);
 
@@ -225,11 +272,14 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
   while (1)
     {
       bool descend = false;
-      char *url, *referer, *file = NULL;
-      int depth;
-      bool html_allowed, css_allowed;
+      char *file = NULL;
       bool is_css = false;
       bool dash_p_leaf_HTML = false;
+      char *url = NULL, *referer;
+      int depth;
+      bool html_allowed, css_allowed;
+      bool dequed = false;
+      int index = 0;
 
       if (opt.quota && total_downloaded_bytes > opt.quota)
         break;
@@ -238,10 +288,20 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 
       /* Get the next URL from the queue... */
 
-      if (!url_dequeue (queue, (struct iri **) &i,
-                        (const char **)&url, (const char **)&referer,
-                        &depth, &html_allowed, &css_allowed))
-        break;
+      if (next_url == NULL)
+        {
+          if (url_dequeue (queue, (struct iri **) &next_i,
+                           (const char **)&next_url, (const char **)&next_referer,
+                           &next_depth, &next_html_allowed, &next_css_allowed))
+            dequed = true;
+        }
+
+      i = next_i;
+      url = next_url;
+      referer = next_referer;
+      depth = next_depth;
+      html_allowed = next_html_allowed;
+      css_allowed = next_css_allowed;
 
       /* ...and download it.  Note that this download is in most cases
          unconditional, as download_child_p already makes sure a file
@@ -251,7 +311,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
          and again under URL2, but at a different (possibly smaller)
          depth, we want the URL's children to be taken into account
          the second time.  */
-      if (dl_url_file_map && hash_table_contains (dl_url_file_map, url))
+      if (dequed && url && dl_url_file_map && hash_table_contains (dl_url_file_map, url))
         {
           bool is_css_bool;
 
@@ -273,15 +333,110 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
         }
       else
         {
-          int dt = 0, url_err;
-          char *redirected = NULL;
-          struct url *url_parsed = url_parse (url, &url_err, i, true);
+          int j;
+retry:
+          if (! url)
+            {
+              int used = 0;
+              for (j = 0; j < N_THREADS; j++)
+                {
+                  if (thread_ctx[j].used)
+                    {
+                      used = 1;
+                      break;
+                    }
+                }
 
-          status = retrieve_url (url_parsed, url, &file, &redirected, referer,
-                                 &dt, false, i, true);
+              if (! used)
+                break;
+            }
+
+          if (url && free_threads)
+            {
+              int err;
+              for (j = 0; j < N_THREADS; j++)
+                if (! thread_ctx[j].used)
+                  {
+                    index = j;
+                    free_threads--;
+                    thread_ctx[j].used = 1;
+                    thread_ctx[j].terminated = 0;
+                    break;
+                  }
+              assert (j < N_THREADS);
+
+              thread_ctx[index].file = file;
+              thread_ctx[index].referer = referer;
+              thread_ctx[index].dt = 0;
+              thread_ctx[index].i = i;
+              thread_ctx[index].redirected = NULL;
+              thread_ctx[index].range = NULL;
+              thread_ctx[index].url = url;
+              thread_ctx[index].retr_sem = &retr_sem;
+              thread_ctx[index].url_parsed = url_parse (thread_ctx[index].url,
+                                                        &thread_ctx[index].url_err,
+                                                        i, true);
+
+#ifdef ENABLE_THREADS
+              err = pthread_create (&thread_ctx[index].thread, NULL,
+                                    start_retrieve_url, &thread_ctx[index]);
+#else
+              thread_ctx[index].status = retrieve_url (thread_ctx[index].url_parsed,
+                                                       thread_ctx[index].url,
+                                                       &thread_ctx[index].file,
+                                                       &thread_ctx[index].redirected,
+                                                       thread_ctx[index].referer,
+                                                       &thread_ctx[index].dt,
+                                                       false, i, true,
+                                                       NULL);
+              thread_ctx[index].used = 1;
+              thread_ctx[index].terminated = 1;
+              err = 0;
+#endif
+
+              if (err == 0)
+                next_url = NULL;
+              else
+                {
+                  logprintf (LOG_NOTQUIET, "pthread_create: %s\n", strerror (errno));
+                  url_free (thread_ctx[index].url_parsed);
+                  thread_ctx[index].used = 0;
+                  free_threads++;
+                  return THREADS_ERR;
+                }
+              continue;
+            }
+
+          index = -1;
+          for (j = 0; j < N_THREADS; j++)
+            if (thread_ctx[j].used && thread_ctx[j].terminated)
+              {
+                index = j;
+                thread_ctx[j].used = 0;
+                THREAD_JOIN (thread_ctx[j].thread, NULL);
+                free_threads++;
+                break;
+              }
+
+          if (index < 0)
+            {
+              int ret;
+              do
+                ret = SEM_WAIT (&retr_sem);
+              while (ret < 0 && errno == EINTR);
+              if (ret < 0)
+                return SEM_ERR;
+
+              goto retry;
+            }
+
+          file = thread_ctx[index].file;
+          referer = thread_ctx[index].referer;
+          i = thread_ctx[index].i;
+          url = thread_ctx[index].url;
 
           if (html_allowed && file && status == RETROK
-              && (dt & RETROKF) && (dt & TEXTHTML))
+              && (thread_ctx[index].dt & RETROKF) && (thread_ctx[index].dt & TEXTHTML))
             {
               descend = true;
               is_css = false;
@@ -291,21 +446,23 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
              lots of web servers serve css with an incorrect content type
           */
           if (file && status == RETROK
-              && (dt & RETROKF) &&
-              ((dt & TEXTCSS) || css_allowed))
+              && (thread_ctx[index].dt & RETROKF) &&
+              ((thread_ctx[index].dt & TEXTCSS) || css_allowed)
+             )
             {
               descend = true;
               is_css = true;
             }
 
-          if (redirected)
+          if (thread_ctx[index].redirected)
             {
               /* We have been redirected, possibly to another host, or
                  different path, or wherever.  Check whether we really
                  want to follow it.  */
               if (descend)
                 {
-                  if (!descend_redirect_p (redirected, url_parsed, depth,
+                  if (!descend_redirect_p (thread_ctx[index].redirected,
+                                           thread_ctx[index].url_parsed, depth,
                                            start_url_parsed, blacklist, i))
                     descend = false;
                   else
@@ -314,15 +471,15 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                     string_set_add (blacklist, url);
                 }
 
-              xfree (url);
-              url = redirected;
+              xfree (thread_ctx[index].url);
+              url = thread_ctx[index].redirected;
             }
           else
             {
-              xfree (url);
-              url = xstrdup (url_parsed->url);
+              xfree (thread_ctx[index].url);
+              url = xstrdup (thread_ctx[index].url_parsed->url);
             }
-          url_free(url_parsed);
+          url_free(thread_ctx[index].url_parsed);
         }
 
       if (opt.spider)
@@ -438,11 +595,12 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
           logputs (LOG_VERBOSE, "\n");
           register_delete_file (file);
         }
-
+#ifndef ENABLE_THREADS
       xfree (url);
       xfree_null (referer);
       xfree_null (file);
       iri_free (i);
+#endif
     }
 
   /* If anything is left of the queue due to a premature exit, free it

src/res.c

@@ -563,7 +563,7 @@ res_retrieve_file (const char *url, char **file, struct iri *iri)
   else
     {
       err = retrieve_url (url_parsed, robots_url, file, NULL, NULL, NULL,
-                          false, i, false);
+                          false, i, false, NULL);
       url_free(url_parsed);
     }
 

src/retr.c

@@ -37,10 +37,22 @@ as that of the covered work.  */
 #include <errno.h>
 #include <string.h>
 #include <assert.h>
+#ifdef ENABLE_THREADS
+#include <pthread.h>
+#include <semaphore.h>
+#endif
+#ifdef ENABLE_METALINK
+#include <metalink/metalink_parser.h>
+#include <metalink/metalink_types.h>
 #ifdef VMS
 # include <unixio.h>            /* For delete(). */
 #endif
 
+#include "metalink.h"
+#endif
+#ifdef ENABLE_THREADS
+#include "multi.h"
+#endif
 #include "exits.h"
 #include "utils.h"
 #include "retr.h"
@@ -57,6 +69,17 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "iri.h"
 
+#ifdef ENABLE_METALINK
+static pthread_mutex_t pconn_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+#define IS_IO_ERROR(status) status == FOPENERR || status == WRITEFAILED || \
+          status == UNLINKERR || status == FWRITEERR || status == FOPEN_EXCL_ERR
+
+#define PCONN_LOCK() pthread_mutex_lock (&pconn_mutex)
+
+#define PCONN_UNLOCK() pthread_mutex_unlock (&pconn_mutex)
+#endif
+
 /* Total size of downloaded files.  Used to enforce quota.  */
 SUM_SIZE_INT total_downloaded_bytes;
 
@@ -227,7 +250,6 @@ write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
 
 int
 fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread, wgint startpos,
-
               wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
               FILE *out2)
 {
@@ -712,7 +734,7 @@ static char *getproxy (struct url *);
 uerr_t
 retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
               char **newloc, const char *refurl, int *dt, bool recursive,
-              struct iri *iri, bool register_status)
+              struct iri *iri, bool register_status, struct range *segment_range)
 {
   uerr_t result;
   char *url;
@@ -739,8 +761,13 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
   url = xstrdup (origurl);
   if (newloc)
     *newloc = NULL;
+
+#ifndef ENABLE_METALINK
+  /* Note that, each and every call to retrieve_url(), except the ones made by
+     functions relevant to metalink support, the value of *file is NULL. */
   if (file)
     *file = NULL;
+#endif
 
   if (!refurl)
     refurl = opt.referer;
@@ -750,7 +777,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
   result = NOCONERROR;
   mynewloc = NULL;
-  local_file = NULL;
+  local_file = *file;
   proxy_url = NULL;
 
   proxy = getproxy (u);
@@ -792,7 +819,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       || (proxy_url && proxy_url->scheme == SCHEME_HTTP))
     {
       result = http_loop (u, orig_parsed, &mynewloc, &local_file, refurl, dt,
-                          proxy_url, iri);
+                          proxy_url, iri, segment_range);
     }
   else if (u->scheme == SCHEME_FTP)
     {
@@ -803,7 +830,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       if (redirection_count)
         oldrec = glob = false;
 
-      result = ftp_loop (u, &local_file, dt, proxy_url, recursive, glob);
+      result = ftp_loop (u, &local_file, dt, proxy_url, recursive, glob, segment_range);
       recursive = oldrec;
 
       /* There is a possibility of having HTTP being redirected to
@@ -942,10 +969,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
         register_redirection (origurl, u->url);
 
       if (*dt & TEXTHTML)
-        register_html (local_file);
+        register_html ("",local_file);
 
       if (*dt & TEXTCSS)
-        register_css (local_file);
+        register_css ("",local_file);
     }
 
   if (file)
@@ -1019,7 +1046,7 @@ retrieve_from_file (const char *file, bool html, int *count)
         opt.base_href = xstrdup (url);
 
       status = retrieve_url (url_parsed, url, &url_file, NULL, NULL, &dt,
-                             false, iri, true);
+                             false, iri, true, NULL);
       url_free (url_parsed);
 
       if (!url_file || (status != RETROK))
@@ -1043,71 +1070,302 @@ retrieve_from_file (const char *file, bool html, int *count)
   else
     input_file = (char *) file;
 
-  url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
-              : get_urls_file (input_file));
+#ifdef ENABLE_METALINK
+  mlink *mlink = parse_metalink(input_file);
 
-  xfree_null (url_file);
-
-  for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
+  if(opt.metalink_file && mlink)
     {
-      char *filename = NULL, *new_file = NULL;
-      int dt;
-      struct iri *tmpiri = iri_dup (iri);
-      struct url *parsed_url = NULL;
+      int i, j, r, ranges_covered, chunk_size, url_err, retries, ret, dt=0;
+      pthread_t thread;
+      sem_t retr_sem;
+      uerr_t status;
+      mlink_file* file;
+      mlink_resource *resource;
+      struct s_thread_ctx *thread_ctx;
 
-      if (cur_url->ignore_when_downloading)
-        continue;
+      /* Wget supports HTTP&FTP, and Metalink supports MD5, SHA1 & SHA-256. */
+      elect_resources (mlink);
+      elect_checksums (mlink);
 
-      if (opt.quota && total_downloaded_bytes > opt.quota)
+      init_temp_files();
+      init_ranges ();
+      thread_ctx = malloc (opt.jobs * (sizeof *thread_ctx));
+
+      retries = 0;
+      file = mlink->files;
+      while (file)
         {
-          status = QUOTEXC;
-          break;
+          memset (thread_ctx, '\0', opt.jobs * (sizeof *thread_ctx));
+
+          /* If chunk_size is too small, set it equal to MIN_CHUNK_SIZE. */
+          chunk_size = (file->size) / opt.jobs;
+          if(chunk_size < MIN_CHUNK_SIZE)
+            chunk_size = MIN_CHUNK_SIZE;
+
+          j = fill_ranges_data (file->num_of_res, file->size, chunk_size);
+
+          /* If chunk_size was set to MIN_CHUNK_SIZE, opt.jobs should be corrected. */
+          if (j < opt.jobs)
+            opt.jobs = j;
+
+          name_temp_files ();
+
+          sem_init (&retr_sem, 0, 0);
+          j = ranges_covered = 0;
+          resource = file->resources;
+
+          /* Assign values to thread_ctx[] elements and spawn threads that will
+             conduct the download. */
+          for (r = 0; r < opt.jobs; ++r)
+            {
+              if (!resource)
+                {
+                  j = 0;
+                  resource = file->resources;
+                }
+
+              thread_ctx[r].referer = NULL;
+              thread_ctx[r].redirected = NULL;
+              thread_ctx[r].dt = dt;
+              thread_ctx[r].i = iri;
+              thread_ctx[r].url = resource->url;
+              thread_ctx[r].retr_sem = &retr_sem;
+
+              ret = spawn_thread (thread_ctx, r, j);
+              if (ret)
+                {
+                  /* If thread creation is unsuccessful */
+                  char *error = url_error (thread_ctx[r].url, thread_ctx[r].url_err);
+                  logprintf (LOG_NOTQUIET, "%s: %s.\n", thread_ctx[r].url, error);
+                  xfree (error);
+                  free (thread_ctx);
+                  clean_range_res_data ();
+                  clean_ranges ();
+                  clean_temp_files ();
+                  return URLERROR;
+                }
+              ++j;
+              resource = resource->next;
+            }
+
+          /* Until all the ranges are covered, collect threads. */
+          while (ranges_covered < opt.jobs)
+            {
+              r = collect_thread (&retr_sem, thread_ctx);
+              ++ranges_covered;
+
+              status = thread_ctx[r].status;
+
+              /* Check return status of thread for errors. */
+              if (IS_IO_ERROR (status))
+                {
+                  /* The error is of type WGET_EXIT_IO_FAIL given in exits.c.
+                     No fallbacking is needed for this type of error. */
+                  inform_exit_status (status);
+                  break;
+                }
+              else if(status != RETROK)
+                {
+                  int error_severity;
+                  PCONN_LOCK ();
+
+                  /* Pick the least severe error.*/
+                  error_severity = get_exit_status();
+                  inform_exit_status ((thread_ctx[r].range)->status_least_severe);
+                  if(get_exit_status() != error_severity)
+                    (thread_ctx[r].range)->status_least_severe = status;
+
+                  PCONN_UNLOCK ();
+
+                  /* Look for resource from which downloading this range is not
+                     tried. */
+                  j = 0;
+                  resource = file->resources;
+                  while (j < file->num_of_res)
+                    {
+                      if (!((thread_ctx[r].range)->resources)[j])
+                        break;
+                      ++j;
+                      resource = resource -> next;
+                    }
+                  /* If there is such a resource, then update the range values
+                     to try that not-tried resource and spawn thread.
+                     If all the resources are exhausted, stop collecting the
+                     threads, as the download failed. */
+                  if (j < file->num_of_res)
+                    {
+                      if ((thread_ctx[r].range)->bytes_covered)
+                        {
+                          thread_ctx[r].url = resource->url;
+                          (thread_ctx[r].range)->first_byte =
+                            (thread_ctx[r].range)->bytes_covered;
+                          (thread_ctx[r].range)->bytes_covered = 0;
+                        }
+                      --ranges_covered;
+                      ret = spawn_thread (thread_ctx, r, j);
+                      if (ret)
+                        {
+                          /* If thread creation is unsuccessful */
+                          char *error = url_error (thread_ctx[r].url, thread_ctx[r].url_err);
+                          logprintf (LOG_NOTQUIET, "%s: %s.\n", thread_ctx[r].url, error);
+                          xfree (error);
+                          free (thread_ctx);
+                          clean_range_res_data ();
+                          clean_ranges ();
+                          clean_temp_files ();
+                          return URLERROR;
+                        }
+                    }
+                  else
+                    break;
+                }
+            }
+
+          sem_destroy(&retr_sem);
+
+          /* Check the download status. If conditions are suitable, retry. */
+          if (status != RETROK)
+            {
+              logprintf (LOG_VERBOSE, _("Downloading %s failed. Chunk %d could "
+                                        "not be downloaded from any of the "
+                                        "URLs listed in metalink file.\n"),
+                         file->name, r);
+
+              /* Failed downloads should only be retried if the error causing
+                 the failure is not an IO error. */
+              if (!(IS_IO_ERROR((thread_ctx[r].range)->status_least_severe)))
+                {
+                  if(retries  < opt.n_retries)
+                    {
+                      logprintf (LOG_VERBOSE,
+                                 _("Retrying to download(%s). (TRY #%d)\n"),
+                                 file->name, ++retries + 1);
+                      continue;
+                    }
+                }
+            }
+          else
+            {
+              char *file_path;
+              int res;
+              /* Form the actual file to be downloaded and verify hash. */
+              file_path = malloc((opt.dir_prefix ? strlen(opt.dir_prefix) : 0)
+                               + strlen(file->name) + (sizeof "/"));
+              if(opt.dir_prefix)
+                sprintf(file_path, "%s/%s", opt.dir_prefix, file->name);
+              else
+                sprintf(file_path, "%s", file->name);
+              mkalldirs(file_path);
+              merge_temp_files(file_path);
+              res = verify_file_hash(file_path, file->checksums);
+              free(file_path);
+              if(!res)
+                {
+                  ++*count;
+                  logprintf (LOG_VERBOSE, _("Verifying(%s) succeeded.\n"),
+                               file->name);
+                }
+              else if(res < 0)
+                {
+                  logprintf (LOG_VERBOSE, _("Verifying(%s) failed.\n"),
+                             file->name);
+                  if(retries  < opt.n_retries)
+                    {
+                      logprintf (LOG_VERBOSE,
+                                 _("Retrying to download(%s). (TRY #%d)\n"),
+                                 file->name, ++retries + 1);
+                      continue;
+                    }
+                }
+            }
+
+          delete_temp_files();
+
+          clean_range_res_data();
+          if (opt.quota && total_downloaded_bytes > opt.quota)
+            {
+              status = QUOTEXC;
+              break;
+            }
+          file = file->next;
         }
 
-      parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
+      free(thread_ctx);
+      clean_ranges ();
+      clean_temp_files ();
+      delete_mlink(mlink);
+    }
+  else
+    {
+#endif
+      url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
+                  : get_urls_file (input_file));
 
-      char *proxy = getproxy (cur_url->url);
-      if ((opt.recursive || opt.page_requisites)
-          && (cur_url->url->scheme != SCHEME_FTP || proxy))
+      xfree_null (url_file);
+
+      for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
         {
-          int old_follow_ftp = opt.follow_ftp;
+          char *filename = NULL, *new_file = NULL;
+          int dt;
+          struct iri *tmpiri = iri_dup (iri);
+          struct url *parsed_url = NULL;
 
-          /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
-          if (cur_url->url->scheme == SCHEME_FTP)
-            opt.follow_ftp = 1;
+          if (cur_url->ignore_when_downloading)
+            continue;
 
-          status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
+          if (opt.quota && total_downloaded_bytes > opt.quota)
+            {
+              status = QUOTEXC;
+              break;
+            }
+
+          parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
+
+          char *proxy = getproxy (cur_url->url);
+          if ((opt.recursive || opt.page_requisites)
+              && (cur_url->url->scheme != SCHEME_FTP || proxy))
+            {
+              int old_follow_ftp = opt.follow_ftp;
+
+              /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
+              if (cur_url->url->scheme == SCHEME_FTP)
+                opt.follow_ftp = 1;
+
+              status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
                                   tmpiri);
 
-          opt.follow_ftp = old_follow_ftp;
-        }
-      else
-        status = retrieve_url (parsed_url ? parsed_url : cur_url->url,
-                               cur_url->url->url, &filename,
-                               &new_file, NULL, &dt, opt.recursive, tmpiri,
-                               true);
-      free(proxy);
+              opt.follow_ftp = old_follow_ftp;
+            }
+          else
+            status = retrieve_url (parsed_url ? parsed_url : cur_url->url,
+                                   cur_url->url->url, &filename,
+                                   &new_file, NULL, &dt, opt.recursive, tmpiri,
+                                   true, NULL);
+          free(proxy);
 
-      if (parsed_url)
-          url_free (parsed_url);
+          if (parsed_url)
+              url_free (parsed_url);
 
-      if (filename && opt.delete_after && file_exists_p (filename))
-        {
-          DEBUGP (("\
+          if (filename && opt.delete_after && file_exists_p (filename))
+            {
+              DEBUGP (("\
 Removing file due to --delete-after in retrieve_from_file():\n"));
-          logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
-          if (unlink (filename))
-            logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
-          dt &= ~RETROKF;
+              logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
+              if (unlink (filename))
+                logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+              dt &= ~RETROKF;
+            }
+
+          xfree_null (new_file);
+          xfree_null (filename);
+          iri_free (tmpiri);
         }
 
-      xfree_null (new_file);
-      xfree_null (filename);
-      iri_free (tmpiri);
+      /* Free the linked list of URL-s.  */
+      free_urlpos (url_list);
+#ifdef ENABLE_METALINK
     }
-
-  /* Free the linked list of URL-s.  */
-  free_urlpos (url_list);
+#endif
 
   iri_free (iri);
 

src/retr.h

@@ -58,7 +58,7 @@ char *fd_read_hunk (int, hunk_terminator_t, long, long);
 char *fd_read_line (int);
 
 uerr_t retrieve_url (struct url *, const char *, char **, char **,
-                     const char *, int *, bool, struct iri *, bool);
+                     const char *, int *, bool, struct iri *, bool, struct range *);
 uerr_t retrieve_from_file (const char *, bool, int *);
 
 const char *retr_rate (wgint, double);

src/url.c

@@ -618,12 +618,12 @@ lowercase_str (char *str)
 }
 
 static const char *
-init_seps (enum url_scheme scheme)
+init_seps (enum url_scheme scheme, char *seps)
 {
-  static char seps[8] = ":/";
   char *p = seps + 2;
   int flags = supported_schemes[scheme].flags;
 
+  strcpy (seps, ":/");
   if (flags & scm_has_params)
     *p++ = ';';
   if (flags & scm_has_query)
@@ -676,6 +676,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
   const char *params_b,    *params_e;
   const char *query_b,     *query_e;
   const char *fragment_b,  *fragment_e;
+  char seps_b[8];
 
   int port;
   char *user = NULL, *passwd = NULL;
@@ -739,7 +740,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
   /* Initialize separators for optional parts of URL, depending on the
      scheme.  For example, FTP has params, and HTTP and HTTPS have
      query string and fragment. */
-  seps = init_seps (scheme);
+  seps = init_seps (scheme, seps_b);
 
   host_b = p;
 
@@ -1791,10 +1792,11 @@ path_end (const char *url)
 {
   enum url_scheme scheme = url_scheme (url);
   const char *seps;
+  char seps_b[8];
   if (scheme == SCHEME_INVALID)
     scheme = SCHEME_HTTP;       /* use http semantics for rel links */
   /* +2 to ignore the first two separators ':' and '/' */
-  seps = init_seps (scheme) + 2;
+  seps = init_seps (scheme, seps_b) + 2;
   return strpbrk_or_eos (url, seps);
 }
 

src/utils.c

@@ -2011,6 +2011,16 @@ run_with_timeout (double timeout, void (*fun) (void *), void *arg)
       return false;
     }
 
+#ifdef ENABLE_THREADS
+  if (opt.jobs > 1)
+    {
+      /* XXX: FIXME.  Support this functionality when there are
+         multiple threads.  Do not translate the string.  */
+      logputs (LOG_NOTQUIET, "\ntimeouts not (yet) supported with threads.\n");
+      abort ();
+    }
+#endif
+
   signal (SIGALRM, abort_run_with_timeout);
   if (SETJMP (run_with_timeout_env) != 0)
     {

src/wget.h

@@ -344,9 +344,18 @@ typedef enum
   RETRBADPATTERN, PROXERR,
   AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR,
   UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, ATTRMISSING, UNKNOWNATTR,
-  WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR
+  WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR, THREADS_ERR, SEM_ERR
 } uerr_t;
 
+struct range {
+  wgint first_byte;
+  wgint last_byte;
+  wgint bytes_covered;
+  wgint is_assigned;
+  bool *resources;
+  uerr_t status_least_severe;
+};
+
 /* 2005-02-19 SMS.
    Select an appropriate "orig" suffix and a separator character for
    adding a unique suffix to a file name.

testenv/ChangeLog

@@ -1,3 +1,459 @@
+2014-07-23  Darshit Shah  <darnir@gmail.com>
+
+	* test/base_test.py (BaseTest.gen_cmd_line): Add support for running all
+	tests through valgrind if the relevant environment variable is set
+	* conf/expected_ret_code (ExpectedRetCode.__call__): Valgrind returns error
+	code 45 when it detects a memory leak.
+	* Readme: Update with details about valgrind tests
+
+2014-07-22  Darshit Shah  <darnir@gmail.com>
+
+	* (README): Remove old TODO and document SERVER_WAIT variable
+
+2014-06-22  Darshit Shah  <darnir@gmail.com>
+
+	* (conf.files_crawled): diff is a set object and needs explicit str
+	conversion.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* base_test.py:
+	(CommonMethods): Rename to BaseTest.
+	(BaseTest): Implement __init__ method where the class-wide variables are
+	initialized. Also variable names like `xxx_list` is renamed to its plural
+	form, e.g. `server_list` => `servers`.
+	(BaseTest.init_test_env): Remove name argument due to its unnecessarity.
+	(BaseTest.get_test_dir): Because the path of the test directory is needed
+	in multiple methods, this method is implemented.
+	(BaseTest.get_domain_addr): Rewrite the return statement utilizing str
+	formatting (which is more Pythonic).
+	(BaseTest.get_cmd_line): Rename to gen_cmd_line. Change the variables with
+	capitcal characters to lower ones. Also, the nested for loop is rewritten
+	to a plain loop using the zip function.
+	(BaseTest.__gen_local_filesys): Rename to gen_local_fs_snapshot. Move to
+	ExpectedFiles in conf/expected_files.py and is marked as a static
+	method. Refactor to a less verbose implementation.
+	(BaseTest._check_downloaded_files): Rename to __call__ to agree with the
+	invocation in test case classes. Move to ExpectedFiles in
+	conf/expected_files.py.
+	(BaseTest.get_server_rules): Refactor to a more Pythonic form utilizing
+	dict.items() and is marked static.
+	(BaseTest.stop_server): (new method) an abstract method which should stop
+	the currently using servers.
+	(BaseTest.instantiate_server_by): (new method) an abstract method which
+	should instantiate a server instance according to the given argument.
+	(BaseTest.__enter__): (new method) method which initialize the context
+	manager
+	(BaseTest.__exit__): (new method) method that finilize the context manager
+	and deal with the exceptions during the execution of the with statement,
+	subclasses can override this method for extensibility
+	* http_test.py:
+	(HTTPTest.__init__): Add call to super.__init__. Default values of
+	pre_hook, test_params, post_hook are set to None to avoid a subtle bug of
+	Python. Argument servers is renamed to protocols.
+	(HTTPTest.Server_setup): Move to BaseTest and rename to server_setup.
+	Calls to pre_hook_call, call_test, post_hook_call are removed.
+	(HTTPTest.hook_call, pre_hook_call, call_test, post_hook_call): Move to
+	BaseTest for that both HTTP test cases and FTP test cases may use these
+	methods.
+	(HTTPTest.init_HTTP_Server, init_HTTPS_Server): Merge and rename to
+	instantiate_server_by to implement the abstract method in BaseTest.
+	(HTTPTest.stop_HTTP_Server): Rename to stop_server to implement the
+	abstract method in BaseTest. Also, pull out the part where remaining
+	requests are gathered into a new method request_remaining.
+	(BaseTest.act_retcode): Rename to ret_code because ExpectedRetCode is
+	moved out from BaseTest, so the name act_retcode is actually a bit
+	verbose.
+	* conf/expected_ret_code.py:
+	(ExpectedRetCode.__call__): Rewrite the str into a more readable form.
+	* conf/files_crawled.py:
+	(FilesCrawled.__call__): Refactor this method into a more Pythonic form
+	utilizing the zip function.
+	* conf/local_files.py:
+	(LocalFiles__call__): Rewrite this method with the recommended with
+	statement.
+	* conf/server_conf.py:
+	(ServerConf.__call__): Rewrite this method due to BaseTest.server_list is
+	renamed to BaseTest.servers.
+	* conf/server_files.py:
+	(ServerFiles.__call__): Refactor the nested for loop into a plain one
+	utilizing the zip function.
+	* conf/urls.py:
+	(URLs): Rename url_list to urls.
+	* conf/wget_commands.py:
+	(WgetCommands): Rename command_list to commands, rename test_obj.options
+	to test_obj.wget_options.
+	* Test--https.py, Test-Proto.py, Test-Parallel-Proto.py: Argument servers
+	is changed to protocols due to change in the signature of
+	HTTPTest.__init__.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* test: (new package) package for test case classes
+	* WgetTest.py: Split into test/base_test.py and test/http_test.py.
+	* Test-*.py: Optimize the imports according to changes of WgetTest.py
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* server: (new package) package for the server classes
+	* server.http: (new package) package for HTTP server
+	* server.ftp: (new package) package for FTP server
+	* HTTPServer.py: Move to server/http/http_server.py. Also change the
+	CERTFILE to '../certs/wget-cert.pem'.
+	* FTPServer.py: Move to server/ftp/ftp_server.py.
+	* WgetTest.py: Optimize import respect to the server classes.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* conf: (new package) package for rule classes and hook methods
+	* WgetTest.py:
+	(CommonMethods.Authentication): Move to conf/authentication.py.
+	(CommonMethods.ExpectHeader): Move to conf/expect_header.py.
+	(CommonMethods.RejectHeader): Move to conf/reject_header.py.
+	(CommonMethods.Response): Move to conf/response.py.
+	(CommonMethods.SendHeader): Move to conf/send_header.py.
+	(CommonMethods.ServerFiles): Move to conf/server_files.py.
+	(CommonMethods.LocalFiles): Move to conf/local_files.py.
+	(CommonMethods.ServerConf): Move to conf/server_conf.py.
+	(CommonMethods.WgetCommands): Move to conf/wget_commands.py.
+	(CommonMethods.Urls): Move to conf/urls.py.
+	(CommonMethods.ExpectedRetcode): Move to conf/expected_retcode.py.
+	(CommonMethods.ExpectedFiles): Move to conf/expected_files.py.
+	(CommonMethods.FilesCrawled): Move to conf/files_crawled.py.
+	(CommonMethods.__check_downloaded_files): Rename to
+	_check_downloaded_files, so that the method is callable from outside the
+	class.
+	(CommomMethods.get_server_rules): Modify so that it utilizes the conf
+	package.
+	(HTTPTest): Add a method hook_call(configs, name) to reduce duplications
+	in pre_hook_call, call_test and post_hook_call utilizing the conf package.
+	* conf/hook_sample.py: (new file) sample for hooks
+	* conf/rule_sample.py: (new file) sample for rules
+	* REAMDE: Update sections about customizing rules and hooks.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* exc: (new package) package for miscellaneous exceptions
+	* WgetTest.py: Move TestFailed to exc/test_failed.py.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* Test-Proto.py: Fix a typo (line 71: server to servers).
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* WgetTest.py: Move WgetFile to package misc.
+	* README: Modify documentation respect to WgetFile.
+	* Test-*.py: Optimize imports about WgetFile.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* misc: (new package) package for miscellaneous modules
+	* ColourTerm.py: Move to package misc and rename to colour_terminal.py,
+	add print_color functions to reduce the use of string literals like
+	"BLUE", "RED" etc.
+	* WgetTest.py:
+	(CommonMethods.Server_setup): Change invocation to printer to print_blue.
+	(CommonMethods.FilesCrawled): Change invocation to printer to print_red.
+	(HTTPTest.__init__): Change invocations to printer to print_red and
+	print_green respectively.
+
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+	* Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
+	Also replace all tabs with spaces in file for conformity.
+	* Test--https.py: New test to check if Wget works correctly with HTTPS
+	servers
+	* HTTPServer.py: Import new modules for use in HTTPS Servers
+	(HTTPSServer): New class that generates a SSL-wrapped socket for use in a
+	HTTPS Server.
+	(HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
+	* WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
+	types
+	(CommonMethods.exec_wget): Add the protocol information to the URL before
+	passing it to wget
+	(HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
+	servers defined by their type. E.g. HTTP, HTTPS.
+	(HTTPTest.Server_setup): Reflect change in type of variable servers.
+	However, we maintin the value of self.servers to allow most of the code to
+	remain unchanged.
+	(HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
+	* Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
+	* Test-Proto.py: Same
+
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling
+	the Wget executable.
+
+2013-12-27  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py: Add modeline
+	(CommonMethods.ServerConf): New pre-test hook that sets
+	BaseHTTPRequestHandler class variables in all available servers
+	* HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server
+	to set the class variables
+	(StoppableHTTPServer.server_sett): Set the handler class variables
+
+2013-12-26  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server.
+
+2013-12-25  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the
+	Wget executable is not found at src/wget
+	(HTTPTest.call_test): In case of error during execution, remove all existing
+	servers before quitting
+
+2013-12-15  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be
+	easily reused for other non-HTTP servers.
+	(HTTPTest.__init__): Call Server_setup instead of HTTP_setup
+	(HTTPTest.Server_setup): Split into three more functions, that handle
+	pre-hooks, test execution and post-hooks respectively.
+	(HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split
+	from HTTPTest.Server_setup
+	(HTTPTest.call_test): Execute wget and log exit code. Code split from
+	HTTPTest.Server_setup
+	(HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split
+	from HTTPTest.Server_setup
+
+2013-12-04  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am [RACE_CHECKING_IS_ENABLED]: Define `RACE_FAIL' and
+	RACE_TESTS'.
+
+2013-10-14  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py.
+
+2013-10-06  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (EXTRA_DIST): Distribute test files.
+
+2013-09-16  Darshit Shah  <darnir@gmail.com>
+
+	* README: Update documentation
+
+2013-09-14  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (StoppableHTTPServer): Define object variable
+	request_headers which stores a list of requests received by the server
+	(StoppableHTTPServer.get_req_headers): Return the list of Request
+	Headers stored by the server
+	(_Handler.do_HEAD): Send the Request MEthod string for identification
+	(_Handler.do_GET): Same
+	(_Handler.__log_request): Log the request in Request_Headers list
+	(_Handler.send_head): Make a call to __log_request
+	* Test--spider-r.py: Add new list, Request_List, which contains all
+	the requests that Wget is expected to send. This will allow for
+	fine-grained tests on recursive downloading.
+	* WgetTest.py (CommonMethods.FilesCrawled): New Post-Test Hook, that
+	ensures that all the expected Files on the server were accessed as
+	expected.
+	(HTTPTest.stop_HTTP_server): On stopping server, asks it to respond
+	with list of all requests it received.
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* Test--spider-r.py: Test retrieval in recursive spider mode.
+	* Makefile.am: add new file
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (_Handler.do_HEAD): If requested path is /, respond
+	with /index.html
+	(_Handler.do_HEAD): Smartly guess value of Content-Type Header from
+	file extension
+	(_Handler.guess_type): Use a preset list of extensions and
+	Content-Type strings. If the extension matches one in the list, use
+	that string, else default to "text/plain"
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods._replace_substring): New method that will
+	replace a substring delimited by {{ }} characters by the value of
+	self.<substring> variable
+	(CommonMethods.WgetCommands): Use the _replace_substring () call to
+	replace the substrings in the the command line.
+	(CommonMethods.ServerFiles): Run the _replace_substring () method on
+	the File contents too.
+
+2013-09-11  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods.exec_wget): Expect domain_list instead of
+	domain.
+	(CommonMethods.get_cmd_line): Same. Generate command line by
+	prepending to each file it's respective domain string
+	(CommonMethods.ServerFiles): Generate file_list and server_rules for
+	each Server and set the config details
+	(HTTPTest): New named parameter, servers which signifies number of
+	servers to spawn
+	(HTTPTest.HTTP_setup): This method now takes servers as a new
+	parameter. Instead of storing server and domain, we now store
+	server_list and domain_list. Each server must be initialized through a
+	loop.
+	(HTTPTest.stop_HTTP_server): Stop all servers in a loop.
+	* Test-Parallel-Proto.py: Prototype test file for multiple servers.
+
+2013-09-10  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.stop_HTTP_server): With the threaded servers,
+	we can simply use the socketserver.shutdown() method to close the
+	server instead of sending a QUIT command
+	* HTTPServer.py (StoppabelHTTPServer.serve_forever): Delete method. No
+	need to override this method anymore.
+	(WgetHTTPRequestHandler.do_QUIT): No longer required
+	(HTTPd): Rename self.server to self.server_inst to reduce ambiguity
+	when referenced from WgetTest
+
+2013-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* README (File Structure): Add explanation about various variables
+	used consistently across all tests.
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py: Remove bunch of old code artefacts
+	* WgetTest.py: Same
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (StoppableHTTPServer.server_conf): Change global
+	variable fileSys to an object variable. This is good programming
+	practice and required for parallel-wget support.
+	(StoppableHTTPServer.server_forever): Edit overridden method to remove
+	the global queue variable. No longer required under the new working
+	(WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue
+	(_Handler): Rename class __Handler to _Handler to match Python's
+	encapsulation rules
+	(_Handler.do_POST): fileSys is now an object variable of the server
+	(_Handler.do_PUT): Same
+	(_Handler.send_put): Same
+	(_Handler.send_head): Same
+	(HTTPd): New class that wraps around the server for Threading
+	(create_server): Make new object of HTTPd.
+	(spawn_server): Start the thread created through create_server
+	(ret_fileSys): Removed method. No longer required.
+	* WgetTest.py (HTTPTest.__init__): Don't explicitly set
+	self.act_retcode. Instead toggle tests_passed boolean to set the
+	correct return code.
+	(HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to
+	start a new instance of the server.
+	(HTTPTest.init_HTTP_server): We no longer call the old
+	create_server(), spawn_server() methods. Instead use the new HTTPd
+	class interface to create new instances of the server
+	(HTTPTest.stop_HTTP_server): Don't ask server to return fileSys.
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* Test-Post.py: Test basic functionality for sending HTTP POST
+	requests using the --method command
+	* Makefile.am: Add new test
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods.__check_downloaded_files): Print a
+	unified diff in case there is a mismatch in the file contents
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (WgetHTTPRequestHandler.test_cookies): Comment out the
+	old test_cookies code. This is no longer used and was causing problems
+	with expected cookies. The code will soon be removed anyways
+	* Test-cookie.py: Add new test for basic cookie functionality
+	* Test-cookie-401.py: Ensure cookies are saved during a 401 response
+	* Test-cookie-expires.py: Ensure that the Expires field is correctly
+	handled
+	* Test-cookies-domain-mismatch.py: Ensure that mismatched domains are
+	handled by Wget
+	* Makefile.am: Add the new tests
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* README: New section on pending work. Will keep updating this to keep
+	track of work that remains to be done on this implementation
+
+2013-09-05  Darshit Shah  <darnir@gmail.com>
+
+	* Test-auth-with-content-disposition.py: Add test that ensures Content
+	Disposition works alongwith authentication
+	* Makefile.am: Add new test
+
+2013-09-04  Darshit Shah  <darnir@gmail.com>
+
+	* Test-c-full.py: Test Continue options
+	* Makefile.am: Add Test-c-full.py and Test-O
+
+2013-09-02  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am:	Add new Test
+	* Test-Head.py: New Test to ensure HEAD requests are handled correctly
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* README: Explain that TEST_NAME needs to be unique
+	* Test-auth-no-challenge.py: Edit non-unique TEST_NAME
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPTest.py (ServerError): Define new Exception for handling
+	internal control flow.
+	(StoppableHTTPServer.SendHeader): Simply pass. Do nothing. Adding
+	functionality here seems to crash for no apparent reason.
+	(stoppableHTTPServer.send_cust_headers): Minor optimization. No need
+	for extra variable.
+	(__Handler.Response): Handle explicit Response Code Rules
+	(__Handler.Authentication): Handle Authentication rules
+	(__Handler.handle_auth): Actual worker method for authentication
+	(__Handler.ExpectHeader): Ensure Expected Headers are received
+	(__Handler.RejectHeader): Ensure Blacklisted Headers are NOT received
+	(__Handler.send_HEAD): Dynamically call server rule functions based on
+	the self.rules list. This feature will later be added to POST/PUT, etc
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py: Remove import module defaultdict.
+	(CommonMethods.get_server_rules): server_rules should be a dict, not a
+	defaultdict (list).
+	* HTTPServer.py (WgetHTTPRequestHandler.get_rule_list): If rule does
+	not exist, return None.	Not an emppty list.
+	(WgetHTTPRequestHandler.test_cookies): Rule variable is not a list
+	(__Handler.send_cust_headers): Same
+	(__Handler.custom_response): Same
+	(__Handler.is_authorized): Same
+	(__Handler.expect_headers): Same
+	(__Handler.reject_headers): Same
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* README: (newfile) Simple help / instructions about using the Test
+	Environment.
+	* Makefile.am: (newfile) Makefile for the Test Environment. Uses the
+	Automake Parallel Test Harness
+	* WgetTest.py: (newfile) Base module that executes the Test.
+	* HTTPServer.py: (newfile) Contains the custom HTTP Server for the
+	Test Environment. Creates an instance of http.server in Python3.
+	* FTPServer.py: (newfile) Overrides methods from pyftpdlib for use in
+	the Test Environment. ** Work under progress **.
+	* ColourTerm.py: (newfile) A custom module to output coloured text to
+	the terminal. Known to work on POSIX shells.
+	* Test-Proto.py: (newfile) A prototype Test File. This should be
+	copied when writing a new Test Case.
+	* Test-Content-disposition-2.py: Test Content Disposition clobbering
+	* Test-Content-disposition.py: Test Content Disposition Headers
+	* Test-O.py: Test Output filename command
+	* Test-auth-basic-fail.py: Test returncode on auth failure
+	* Test-auth-basic.py: Test Basic Auth negotiation
+	* Test-auth-both.py: Test handling of Multiple auth providers. This
+	test currently fails.
+	* Test-auth-digest.py: Test Digest Auth Negotiation
+	* Test-auth-no-challenge-url.py: Ensure --auth-no-challenge is handled
+	when auth details are in-URL.
+	* Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
+	* Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
+	response.
 2014-08-08  Darshit Shah  <darnir@gmail.com>
 
 	* conf/__init__.py: Add extra newline according to PEP8

testenv/Makefile.am

@@ -28,6 +28,12 @@
 
 AUTOMAKE_OPTIONS = parallel-tests
 AM_TESTS_ENVIRONMENT = MAKE_CHECK=True; export MAKE_CHECK;
+
+if RACE_CHECKING_IS_ENABLED
+RACE_FAIL = Test--spider-r.py
+RACE_TESTS =
+endif
+
 TESTS = Test-auth-basic-fail.py             \
     Test-auth-basic.py                      \
     Test-auth-both.py                       \
@@ -43,13 +49,15 @@ TESTS = Test-auth-basic-fail.py             \
     Test-cookie-domain-mismatch.py          \
     Test-cookie-expires.py                  \
     Test-cookie.py                          \
+    $(RACE_TESTS)                           \
     Test-Head.py                            \
     Test--https.py							\
     Test-O.py                               \
     Test-Post.py                            \
     Test--spider-r.py
 
-XFAIL_TESTS = Test-auth-both.py
+XFAIL_TESTS = Test-auth-both.py             \
+    $(RACE_FAIL)
 
 LOG_COMPILER = python3
 

testenv/conf/server_conf.py

@@ -0,0 +1,11 @@
+from conf import hook
+
+
+@hook()
+class ServerConf:
+    def __init__(self, server_settings):
+        self.server_settings = server_settings
+
+    def __call__(self, test_obj):
+        for server in test_obj.servers:
+            server.server_sett(self.server_settings)

testenv/exc/__init__.py

@@ -0,0 +1 @@
+

testenv/misc/__init__.py

@@ -0,0 +1 @@
+

testenv/server/__init__.py

@@ -0,0 +1 @@
+

testenv/server/ftp/__init__.py

@@ -0,0 +1 @@
+

testenv/server/http/__init__.py

@@ -0,0 +1 @@
+

testenv/test/__init__.py

@@ -0,0 +1 @@
+

tests/ChangeLog

@@ -183,6 +183,10 @@
 	* HTTPServer.pm: added check for must-not-match request-header
 	* Test-cookies.px: check cookie deletion and cookie domain matching
 
+2012-12-02  Domenico Chierico <spaghetty@gmail.com>
+
+	* Makefile.am (LDADD): Remove $(LIBS).
+
 2012-06-16  Giuseppe Scrivano  <gscrivano@gnu.org>
 
 	* Makefile.am (EXTRA_DIST): Add Test-stdouterr.px.

tests/Makefile.am

@@ -147,6 +147,6 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
 
 check_PROGRAMS = unit-tests
 unit_tests_SOURCES =
-LDADD = ../src/libunittest.a ../lib/libgnu.a $(LIBS)
+LDADD = ../src/libunittest.a ../lib/libgnu.a
 
 CLEANFILES = *~ *.bak core core.[0-9]*