github-repos/wget
1
0
Fork 0
mirror of https://github.com/mirror/wget.git synced 2026-08-19 09:23:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert " Hi, Thank you again Darshit for your response. The RejectHeaderField rule rejects ANY header"

Browse Source 
I accidentally commited this patch.

This reverts commit 4ce2f93600.
This commit is contained in:
Darshit Shah
2019-05-30 11:19:46 +02:00
parent 02c059a857
commit 60dc09b507
15 changed files with 10 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
doc/wget.texi
View File

@@ -1542,21 +1542,6 @@ wget --header="Host: foo.bar" http://localhost/
In versions of Wget prior to 1.10 such use of @samp{--header} caused
sending of duplicate headers.
@cindex disable header, choose
@item --disable-header=@var{list}
Specify comma-separated header fields to remove from the @sc{http}
request.
@example
@group
wget --disable-header='Accept,User-Agent,Authorization' \
https://example.com/
@end group
@end example
Specifying a header field with @samp{--header} after disabling it
will override it and include it in the @sc{http} request headers.
@cindex Content-Encoding, choose
@item --compression=@var{type}
Choose the type of compression to be used. Legal values are

17
fuzz/wget_options_fuzzer.dict
View File

@@ -30,22 +30,6 @@
"human"
"csv"
"json"
"Authorization"
"User-Agent"
"Referer"
"Cache-Control"
"Pragma"
"If-Modified-Since"
"Range"
"Accept"
"Accept-Encoding"
"Host"
"Connection"
"Proxy-Connection"
"Content-Type"
"Content-Length"
"Proxy-Authorization"
"Cookie"
"accept="
"accept-regex="
"adjust-extension="
@@ -82,7 +66,6 @@
"delete-after="
"directories="
"directory-prefix="
"disable-header="
"dns-caching="
"dns-timeout="
"domains="

28
src/http.c
View File

@@ -88,7 +88,6 @@ static char *basic_authentication_encode (const char *, const char *);
static bool known_authentication_scheme_p (const char *, const char *);
static void ensure_extension (struct http_stat *, const char *, int *);
static void load_cookies (void);
static bool disabled_header (const char*);
static bool cookies_loaded_p;
static struct cookie_jar *wget_cookie_jar;
@@ -237,7 +236,7 @@ request_set_header (struct request *req, const char *name, const char *value,
struct request_header *hdr;
int i;
if (disabled_header (name) || !value)
if (!value)
{
/* A NULL value is a no-op; if freeing the name is requested,
free it now to avoid leaks. */
@@ -1843,27 +1842,6 @@ time_to_rfc1123 (time_t time, char *buf, size_t bufsize)
return RETROK;
}
static bool
disabled_header (const char* header_name)
{
char** p = opt.disabled_headers;
char *s;
size_t n;
if (!p)
return 0;
for (; *p; ++p)
{
s = strchrnul (header_name, ':');
n = (size_t) (s - header_name);
if (n == strlen (*p) && 0 == strncmp (header_name, *p, n))
return 1;
}
return 0;
}
static struct request *
initialize_request (const struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
bool inhibit_keep_alive, bool *basic_auth_finished,
@@ -3285,10 +3263,6 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
if (opt.user_headers)
{
int i;
/* Empty the disabled headers as they are no longer used
and this will let headers to be overriden by the user */
free_vec (opt.disabled_headers);
opt.disabled_headers = NULL;
for (i = 0; opt.user_headers[i]; i++)
request_set_user_header (req, opt.user_headers[i]);
}

48
src/init.c
View File

@@ -101,7 +101,6 @@ CMD_DECLARE (cmd_spec_compression);
#endif
CMD_DECLARE (cmd_spec_dirstruct);
CMD_DECLARE (cmd_spec_header);
CMD_DECLARE (cmd_dis_header);
CMD_DECLARE (cmd_spec_warc_header);
CMD_DECLARE (cmd_spec_htmlify);
CMD_DECLARE (cmd_spec_mirror);
@@ -184,7 +183,6 @@ static const struct {
{ "deleteafter", &opt.delete_after, cmd_boolean },
{ "dirprefix", &opt.dir_prefix, cmd_directory },
{ "dirstruct", NULL, cmd_spec_dirstruct },
{ "disableheader", NULL, cmd_dis_header },
{ "dnscache", &opt.dns_cache, cmd_boolean },
#ifdef HAVE_LIBCARES
{ "dnsservers", &opt.dns_servers, cmd_string },
@@ -400,7 +398,6 @@ defaults (void)
opt.metalink_index = -1;
#endif
opt.disabled_headers = NULL;
opt.cookies = true;
opt.verbose = -1;
opt.ntry = 20;
@@ -1461,7 +1458,7 @@ cmd_cert_type (const char *com, const char *val, void *place)
/* Specialized helper functions, used by `commands' to handle some
options specially. */
static bool check_user_specified_header (const char *, bool);
static bool check_user_specified_header (const char *);
#ifdef HAVE_LIBZ
static bool
@@ -1496,33 +1493,6 @@ cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored _GL_UN
return true;
}
static bool
cmd_dis_header (const char *com, const char *val, void *place_ignored _GL_UNUSED)
{
/* Empty value means reset the list of headers. */
if (*val == '\0')
{
free_vec (opt.disabled_headers);
opt.disabled_headers = NULL;
return true;
}
cmd_vector (com, val, &opt.disabled_headers);
char **p = opt.disabled_headers;
for (; *p; ++p)
{
if (!check_user_specified_header (*p, false))
{
fprintf (stderr, _("%s: %s: Invalid header field %s.\n"),
exec_name, com, quote (*p));
free_vec (opt.disabled_headers);
opt.disabled_headers = NULL;
return false;
}
opt.user_headers = vec_remove_header (opt.user_headers, *p);
}
return true;
}
static bool
cmd_spec_header (const char *com, const char *val, void *place_ignored _GL_UNUSED)
{
@@ -1534,7 +1504,7 @@ cmd_spec_header (const char *com, const char *val, void *place_ignored _GL_UNUSE
return true;
}
if (!check_user_specified_header (val, true))
if (!check_user_specified_header (val))
{
fprintf (stderr, _("%s: %s: Invalid header %s.\n"),
exec_name, com, quote (val));
@@ -1555,7 +1525,7 @@ cmd_spec_warc_header (const char *com, const char *val, void *place_ignored _GL_
return true;
}
if (!check_user_specified_header (val, true))
if (!check_user_specified_header (val))
{
fprintf (stderr, _("%s: %s: Invalid WARC header %s.\n"),
exec_name, com, quote (val));
@@ -1881,17 +1851,18 @@ simple_atof (const char *beg, const char *end, double *dest)
contain newlines. */
static bool
check_user_specified_header (const char *s, bool full_header)
check_user_specified_header (const char *s)
{
const char *p;
for (p = s; *p && *p != ':' && !c_isspace (*p); p++)
;
if (*p != ':' && full_header)
return false; /* The header MUST contain `:' preceded by at least one
non-whitespace character. */
/* The header MUST contain `:' preceded by at least one
non-whitespace character. */
if (*p != ':' || p == s)
return false;
/* The header MUST NOT contain newlines. */
if (p == s || strchr (s, '\n'))
if (strchr (s, '\n'))
return false;
return true;
}
@@ -2006,7 +1977,6 @@ cleanup (void)
xfree (opt.http_passwd);
xfree (opt.dot_style);
free_vec (opt.user_headers);
free_vec (opt.disabled_headers);
free_vec (opt.warc_user_headers);
# ifdef HAVE_SSL
xfree (opt.cert_file);

3
src/main.c
View File

@@ -304,7 +304,6 @@ static struct cmdline_option option_data[] =
{ "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 },
{ "directories", 0, OPT_BOOLEAN, "dirstruct", -1 },
{ "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 },
{ "disable-header", 0, OPT_VALUE, "disableheader", -1 },
{ "dns-cache", 0, OPT_BOOLEAN, "dnscache", -1 },
#ifdef HAVE_LIBCARES
{ "dns-servers", 0, OPT_VALUE, "dnsservers", -1 },
@@ -793,8 +792,6 @@ HTTP options:\n"),
--ignore-length ignore 'Content-Length' header field\n"),
N_("\
--header=STRING insert STRING among the headers\n"),
N_("\
--disable-header=LIST comma-separated list of disabled header fields\n"),
#ifdef HAVE_LIBZ
N_("\
--compression=TYPE choose compression, one of auto, gzip and none. (default: none)\n"),

1
src/options.h
View File

@@ -147,7 +147,6 @@ struct options
char *http_user; /* HTTP username. */
char *http_passwd; /* HTTP password. */
char **user_headers; /* User-defined header(s). */
char **disabled_headers; /* User-disabled header(s) */
bool http_keep_alive; /* whether we use keep-alive */
bool use_proxy; /* Do we use proxy? */

43
src/utils.c
View File

@@ -1405,49 +1405,6 @@ wget_read_file_free (struct file_memory *fm)
xfree (fm);
}
/* Removes a header from a request headers vector */
char **
vec_remove_header (char **vec, const char *str)
{
char* s;
int i, cnt; /* count of vector elements */
size_t n;
if (vec != NULL)
{
for (cnt = 0; vec[cnt]; cnt++)
;
/* remove all duplicates */
i = 0;
while (vec[i])
{
s = strchrnul (vec[i], ':');
n = (size_t) (s - vec[i]);
if (n == strlen (str) && 0 == strncmp (vec[i], str, n))
{
if (cnt == 1)
{
xfree (vec[0]);
vec = xrealloc (vec, sizeof (char *));
vec[0] = NULL;
return vec;
}
else
{
vec[i] = xstrdup (vec[cnt-1]);
xfree (vec[cnt-1]);
vec = xrealloc (vec, cnt * sizeof (char *));
vec[cnt - 1] = NULL;
--cnt;
}
}
else
++i;
}
}
return vec;
}
/* Free the pointers in a NULL-terminated vector of pointers, then
free the pointer itself. */
void

1
src/utils.h
View File

@@ -110,7 +110,6 @@ bool has_html_suffix_p (const char *);
struct file_memory *wget_read_file (const char *);
void wget_read_file_free (struct file_memory *);
char ** vec_remove_header (char **, const char *);
void free_vec (char **);
char **merge_vecs (char **, char **);
char **vec_append (char **, const char *);

3
testenv/Makefile.am
View File

@@ -95,9 +95,6 @@ if HAVE_PYTHON3
Test-cookie-domain-mismatch.py \
Test-cookie-expires.py \
Test-cookie.py \
Test-disable-default-headers.py \
Test-disable-headers-after.py \
Test-disable-headers-before.py \
Test-Head.py \
Test-hsts.py \
Test--https.py \

4
testenv/README
View File

@@ -182,10 +182,6 @@ This section lists the currently supported File Rules and their structure.
* RejectHeader : This list of Headers must NEVER occur in a request. It
uses the same value format as ExpectHeader.
* RejectHeaderField : This list of Headers Fields must NOT appear in a request.
The value for this key is a list of strings where each header field is represented as:
|-->Header Field: <Header Field Name>
* SendHeader : This list of Headers will be sent in EVERY response to a
request for the respective file. It follows the same value format as
ExpectHeader. Additionally you can specify a list of strings as <Header Data>

74
testenv/Test-disable-default-headers.py
View File

@@ -1,74 +0,0 @@
#!/usr/bin/env python3
from sys import exit
from test.http_test import HTTPTest
from test.base_test import HTTP, HTTPS
from misc.wget_file import WgetFile
"""
This is test ensures that the --disable-header option removes default request
headers. There aren't any user defined header.
"""
############# File Definitions ###############################################
file_content = """Les paroles de la bouche d'un homme sont des eaux profondes;
La source de la sagesse est un torrent qui jaillit."""
Headers = {
'Authorization',
'User-Agent',
'Referer',
'Cache-Control',
'Pragma',
'If-Modified-Since',
'Range',
'Accept',
'Accept-Encoding',
'Host',
'Connection',
'Proxy-Connection',
'Content-Type',
'Content-Length',
'Proxy-Authorization',
'Cookie',
'MyHeader',
}
WGET_OPTIONS = '--disable-header="'
WGET_URLS = [[]]
Files = [[]]
headers_len = len(Headers)
for index, header in enumerate(Headers, start=1):
File_rules = {
"RejectHeaderField" : {
header
}
}
file_name = "File" + str(index)
Files[0].append (WgetFile(file_name, file_content, rules=File_rules))
WGET_OPTIONS += header + (',' if index < headers_len else '"')
WGET_URLS[0].append (file_name)
Servers = [HTTP]
ExpectedReturnCode = 0
################ Pre and Post Test Hooks #####################################
pre_test = {
"ServerFiles" : Files
}
test_options = {
"WgetCommands" : WGET_OPTIONS,
"Urls" : WGET_URLS
}
post_test = {
"ExpectedRetcode" : ExpectedReturnCode
}
err = HTTPTest (
pre_hook=pre_test,
test_params=test_options,
post_hook=post_test,
protocols=Servers
).begin ()
exit (err)

80
testenv/Test-disable-headers-after.py
View File

@@ -1,80 +0,0 @@
#!/usr/bin/env python3
from sys import exit
from test.http_test import HTTPTest
from test.base_test import HTTP, HTTPS
from misc.wget_file import WgetFile
"""
This is test ensures that the --disable-header option removes user headers
from the HTTP request when it's placed after --header="header: value".
"""
############# File Definitions ###############################################
file_content = """Les paroles de la bouche d'un homme sont des eaux profondes;
La source de la sagesse est un torrent qui jaillit."""
Headers = {
'Authorization',
'User-Agent',
'Referer',
'Cache-Control',
'Pragma',
'If-Modified-Since',
'Range',
'Accept',
'Accept-Encoding',
'Host',
'Connection',
'Proxy-Connection',
'Content-Type',
'Content-Length',
'Proxy-Authorization',
'Cookie',
'MyHeader',
}
WGET_OPTIONS = ''
WGET_URLS = [[]]
Files = [[]]
# Define user defined headers
for header in Headers:
WGET_OPTIONS += ' --header="' + header + ': any"'
WGET_OPTIONS += ' --disable-header="'
headers_len = len(Headers)
for index, header in enumerate(Headers, start=1):
File_rules = {
"RejectHeader" : {
header : 'any'
}
}
file_name = "File" + str(index)
Files[0].append(WgetFile(file_name, file_content, rules=File_rules))
WGET_OPTIONS += header + (',' if index < headers_len else '"')
WGET_URLS[0].append(file_name)
Servers = [HTTP]
ExpectedReturnCode = 0
################ Pre and Post Test Hooks #####################################
pre_test = {
"ServerFiles" : Files
}
test_options = {
"WgetCommands" : WGET_OPTIONS,
"Urls" : WGET_URLS
}
post_test = {
"ExpectedRetcode" : ExpectedReturnCode
}
err = HTTPTest (
pre_hook=pre_test,
test_params=test_options,
post_hook=post_test,
protocols=Servers
).begin ()
exit (err)

78
testenv/Test-disable-headers-before.py
View File

@@ -1,78 +0,0 @@
#!/usr/bin/env python3
from sys import exit
from test.http_test import HTTPTest
from test.base_test import HTTP, HTTPS
from misc.wget_file import WgetFile
"""
This is test ensures that the --disable-header option doesn't remove user headers
from the HTTP request when it's placed before --header="header: value".
"""
############# File Definitions ###############################################
file_content = """Les paroles de la bouche d'un homme sont des eaux profondes;
La source de la sagesse est un torrent qui jaillit."""
Headers = {
'Authorization',
'User-Agent',
'Referer',
'Cache-Control',
'Pragma',
'If-Modified-Since',
'Range',
'Accept',
'Accept-Encoding',
'Host',
'Connection',
'Proxy-Connection',
'Content-Type',
'Content-Length',
'Proxy-Authorization',
'Cookie',
'MyHeader',
}
WGET_OPTIONS = '--disable-header="'
WGET_URLS = [[]]
Files = [[]]
headers_len = len(Headers)
for index, header in enumerate(Headers, start=1):
File_rules = {
"ExpectHeader" : {
header : 'any'
}
}
file_name = "File" + str(index)
Files[0].append (WgetFile(file_name, file_content, rules=File_rules))
WGET_OPTIONS += header + (',' if index < headers_len else '"')
WGET_URLS[0].append (file_name)
# Define user defined headers
for header in Headers:
WGET_OPTIONS += ' --header="' + header + ': any"'
Servers = [HTTP]
ExpectedReturnCode = 0
################ Pre and Post Test Hooks #####################################
pre_test = {
"ServerFiles" : Files
}
test_options = {
"WgetCommands" : WGET_OPTIONS,
"Urls" : WGET_URLS
}
post_test = {
"ExpectedRetcode" : ExpectedReturnCode
}
err = HTTPTest (
pre_hook=pre_test,
test_params=test_options,
post_hook=post_test,
protocols=Servers
).begin ()
exit (err)

12
testenv/conf/reject_header_field.py
View File

@@ -1,12 +0,0 @@
from conf import rule
""" Rule: RejectHeaderField
This is a server side rule which expects a string list of Header Fields
which should be blacklisted by the server for a particular file's requests.
"""
@rule()
class RejectHeaderField:
def __init__(self, header_fields):
self.header_fields = header_fields

8
testenv/server/http/http_server.py
View File

@@ -370,14 +370,6 @@ class _Handler(BaseHTTPRequestHandler):
header_line)
raise ServerError("Header " + header_line + ' received')
def RejectHeaderField(self, header_fields_obj):
rej_header_fields = header_fields_obj.header_fields
for field in rej_header_fields:
if field in self.headers:
self.send_error(400, 'Blacklisted Header Field %s received' %
field)
raise ServerError('Header Field %s received' % field)
def __log_request(self, method):
req = method + " " + self.path
self.server.request_headers.append(req)