From 886ac1a89bf89371ef54d0d0186c78b30d9e331b Mon Sep 17 00:00:00 2001 From: Darshit Shah Date: Sat, 7 Sep 2013 11:15:14 +0530 Subject: [PATCH] Shift to new Threading Model from Multiprocessing model This eliminated the use of Global Variables from the HTTPServer module and uses the more correct threading.Threads module to spawn a new server. However, on multi-core systems, the performance may deteriorate when too many servers are launched due to CPython implementation of GIL. Many artefacts from the old model still remain, sometimes as commenst, else as executable code. These will be cleaned up soon --- testenv/ChangeLog | 28 +++++++++++++++++++ testenv/HTTPServer.py | 65 ++++++++++++++++++++++++++++--------------- testenv/WgetTest.py | 15 +++++----- 3 files changed, 77 insertions(+), 31 deletions(-) diff --git a/testenv/ChangeLog b/testenv/ChangeLog index 2b095353..a00e0dc0 100644 --- a/testenv/ChangeLog +++ b/testenv/ChangeLog @@ -1,3 +1,31 @@ +2013-09-07 Darshit Shah + + * HTTPServer.py (StoppableHTTPServer.server_conf): Change global + variable fileSys to an object variable. This is good programming + practice and required for parallel-wget support. + (StoppableHTTPServer.server_forever): Edit overridden method to remove + the global queue variable. No longer required under the new working + (WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue + (_Handler): Rename class __Handler to _Handler to match Python's + encapsulation rules + (_Handler.do_POST): fileSys is now an object variable of the server + (_Handler.do_PUT): Same + (_Handler.send_put): Same + (_Handler.send_head): Same + (HTTPd): New class that wraps around the server for Threading + (create_server): Make new object of HTTPd. + (spawn_server): Start the thread created through create_server + (ret_fileSys): Removed method. No longer required. + * WgetTest.py (HTTPTest.__init__): Don't explicitly set + self.act_retcode. Instead toggle tests_passed boolean to set the + correct return code. + (HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to + start a new instance of the server. + (HTTPTest.init_HTTP_server): We no longer call the old + create_server(), spawn_server() methods. Instead use the new HTTPd + class interface to create new instances of the server + (HTTPTest.stop_HTTP_server): Don't ask server to return fileSys. + 2013-09-07 Darshit Shah * Test-Post.py: Test basic functionality for sending HTTP POST diff --git a/testenv/HTTPServer.py b/testenv/HTTPServer.py index 063bde34..22e6a675 100644 --- a/testenv/HTTPServer.py +++ b/testenv/HTTPServer.py @@ -3,6 +3,7 @@ from http.server import HTTPServer, BaseHTTPRequestHandler from base64 import b64encode from random import random from hashlib import md5 +import threading import os import re @@ -32,17 +33,22 @@ class StoppableHTTPServer (HTTPServer): """ self.server_configs = conf_dict - global fileSys - fileSys = filelist + self.fileSys = filelist - def serve_forever (self, q): - """ Override method allowing for programmatical shutdown process. """ + def serve_forever (self): + self.stop = False + while not self.stop: + self.handle_request () + + +""" def serve_forever (self, q): + # Override method allowing for programmatical shutdown process. global queue queue = q self.stop = False while not self.stop: self.handle_request () - +""" class WgetHTTPRequestHandler (BaseHTTPRequestHandler): @@ -76,13 +82,13 @@ class WgetHTTPRequestHandler (BaseHTTPRequestHandler): return r_list def do_QUIT (self): - queue.put (fileSys) + #queue.put (self.server.fileSys) self.send_response (200) self.end_headers () self.server.stop = True -class __Handler (WgetHTTPRequestHandler): +class _Handler (WgetHTTPRequestHandler): """ Define Handler Methods for different Requests. """ @@ -107,13 +113,13 @@ class __Handler (WgetHTTPRequestHandler): self.rules = self.server.server_configs.get (path) if not self.custom_response (): return (None, None) - if path in fileSys: + if path in self.server.fileSys: body_data = self.get_body_data () self.send_response (200) self.send_header ("Content-type", "text/plain") - content = fileSys.pop (path) + "\n" + body_data + content = self.server.fileSys.pop (path) + "\n" + body_data total_length = len (content) - fileSys[path] = content + self.server.fileSys[path] = content self.send_header ("Content-Length", total_length) self.finish_headers () try: @@ -128,7 +134,7 @@ class __Handler (WgetHTTPRequestHandler): self.rules = self.server.server_configs.get (path) if not self.custom_response (): return (None, None) - fileSys.pop (path, None) + self.server.fileSys.pop (path, None) self.send_put (path) """ End of HTTP Request Method Handlers. """ @@ -156,7 +162,7 @@ class __Handler (WgetHTTPRequestHandler): def send_put (self, path): body_data = self.get_body_data () self.send_response (201) - fileSys[path] = body_data + self.server.fileSys[path] = body_data self.send_header ("Content-type", "text/plain") self.send_header ("Content-Length", len (body_data)) self.finish_headers () @@ -374,7 +380,7 @@ class __Handler (WgetHTTPRequestHandler): """ path = self.path[1:] - if path in fileSys: + if path in self.server.fileSys: self.rules = self.server.server_configs.get (path) for rule_name in self.rules: @@ -397,7 +403,7 @@ class __Handler (WgetHTTPRequestHandler): else: return (None, None) - content = fileSys.get (path) + content = self.server.fileSys.get (path) content_length = len (content) try: self.range_begin = self.parse_range_header ( @@ -428,20 +434,33 @@ class __Handler (WgetHTTPRequestHandler): self.send_error (404, "Not Found") return (None, None) +class HTTPd (threading.Thread): + server_class = StoppableHTTPServer + handler = _Handler + def __init__ (self, addr=None): + threading.Thread.__init__ (self) + if addr is None: + addr = ('localhost', 0) + self.server = self.server_class (addr, self.handler) + self.server_address = self.server.socket.getsockname()[:2] + + def run (self): + self.server.serve_forever () + + def server_conf (self, file_list, server_rules): + self.server.server_conf (file_list, server_rules) + def create_server (): - server = StoppableHTTPServer (("localhost", 0), __Handler) + server = HTTPd () return server def spawn_server (server): - global q - q = Queue() - server_process = Process (target=server.serve_forever, args=(q,)) - server_process.start () - - -def ret_fileSys (): - return (q.get (True)) + #global q + #q = Queue() + #server_process = Process (target=server.serve_forever, args=(q,)) + #server_process.start () + server.start () # vim: set ts=8 sts=4 sw=3 tw=0 et : diff --git a/testenv/WgetTest.py b/testenv/WgetTest.py index 7e2b2857..41399252 100644 --- a/testenv/WgetTest.py +++ b/testenv/WgetTest.py @@ -204,13 +204,12 @@ class HTTPTest (CommonMethods): self.HTTP_setup (name, pre_hook, test_params, post_hook) except TestFailed as tf: printer ("RED", "Error: " + tf.error) - self.act_retcode = 100 self.tests_passed = False except Exception as ae: printer ("RED", "Unhandled Exception Caught.") print ( ae.__str__ ()) traceback.print_exc () - self.act_retcode = 100 + self.tests_passed = False else: printer ("GREEN", "Test Passed") finally: @@ -239,7 +238,7 @@ class HTTPTest (CommonMethods): getattr (self, test_func) (test_params[test_func]) - HTTPServer.spawn_server (self.server) + #HTTPServer.spawn_server (self.server) self.act_retcode = self.exec_wget (self.options, self.urls, self.domain) self.stop_HTTP_Server () @@ -251,16 +250,16 @@ class HTTPTest (CommonMethods): getattr (self, post_hook_func) (post_hook[post_hook_func]) def init_HTTP_Server (self): - server = HTTPServer.create_server () - return server - #server = HTTPServer.HTTPd () - #server.start (self) + #server = HTTPServer.create_server () #return server + server = HTTPServer.HTTPd () + server.start () + return server def stop_HTTP_Server (self): conn = http.client.HTTPConnection (self.domain.strip ('/')) conn.request ("QUIT", "/") - self.fileSys = HTTPServer.ret_fileSys () + #self.fileSys = HTTPServer.ret_fileSys () conn.getresponse () #server.stop ()