diff options
Diffstat (limited to 'testenv')
28 files changed, 1195 insertions, 323 deletions
diff --git a/testenv/Makefile.am b/testenv/Makefile.am index 39ea76c..c16a6c6 100644 --- a/testenv/Makefile.am +++ b/testenv/Makefile.am @@ -1,5 +1,5 @@ # Makefile for `wget' utility -# Copyright (C) 2013 Free Software Foundation, Inc. +# Copyright (C) 2013, 2015 Free Software Foundation, Inc. # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,39 +26,58 @@ # as that of the covered work. +if METALINK_IS_ENABLED + METALINK_TESTS = Test-metalink-xml.py \ + Test-metalink-http.py +else + METALINK_TESTS = +endif + AUTOMAKE_OPTIONS = parallel-tests AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; MAKE_CHECK=True; export MAKE_CHECK;\ export PYTHONPATH=$$PYTHONPATH:$(srcdir); export VALGRIND_TESTS="@VALGRIND_TESTS@"; + +SSL_TESTS = Test--https.py Test--https-crl.py if HAVE_PYTHON3 - TESTS = Test-auth-basic-fail.py \ - Test-auth-basic.py \ - Test-auth-both.py \ - Test-auth-digest.py \ - Test-auth-no-challenge.py \ - Test-auth-no-challenge-url.py \ - Test-auth-retcode.py \ - Test-auth-with-content-disposition.py \ - Test-c-full.py \ - Test-Content-disposition-2.py \ - Test-Content-disposition.py \ - Test-cookie-401.py \ - Test-cookie-domain-mismatch.py \ - Test-cookie-expires.py \ - Test-cookie.py \ - Test-Head.py \ - Test--https.py \ - Test--https-crl.py \ - Test-O.py \ - Test-Post.py \ - Test-504.py \ - Test--spider-r.py \ - Test-redirect-crash.py + TESTS = Test-auth-basic-fail.py \ + Test-auth-basic.py \ + Test-auth-both.py \ + Test-auth-digest.py \ + Test-auth-no-challenge.py \ + Test-auth-no-challenge-url.py \ + Test-auth-retcode.py \ + Test-auth-with-content-disposition.py \ + Test-c-full.py \ + Test-Content-disposition-2.py \ + Test-Content-disposition.py \ + Test--convert-links--content-on-error.py \ + Test-cookie-401.py \ + Test-cookie-domain-mismatch.py \ + Test-cookie-expires.py \ + Test-cookie.py \ + Test-Head.py \ + Test--https.py \ + Test--https-crl.py \ + Test-hsts.py \ + Test-O.py \ + Test-Post.py \ + Test-504.py \ + Test--spider-r.py \ + Test--rejected-log.py \ + Test-redirect-crash.py \ + Test-reserved-chars.py \ + Test-condget.py \ + $(SSL_TESTS) \ + $(METALINK_TESTS) # added test cases expected to fail here and under TESTS - XFAIL_TESTS = +if !WITH_SSL + XFAIL_TESTS = $(SSL_TESTS) +endif + endif -EXTRA_DIST = certs conf exc misc server test README $(TESTS) $(XFAIL_TESTS) +EXTRA_DIST = certs conf exc misc server test README $(TESTS) TEST_EXTENSIONS = .py PY_LOG_COMPILER = python3 diff --git a/testenv/Makefile.in b/testenv/Makefile.in index ef50588..96518d1 100644 --- a/testenv/Makefile.in +++ b/testenv/Makefile.in @@ -1,7 +1,7 @@ -# Makefile.in generated by automake 1.14.1 from Makefile.am. +# Makefile.in generated by automake 1.15 from Makefile.am. # @configure_input@ -# Copyright (C) 1994-2013 Free Software Foundation, Inc. +# Copyright (C) 1994-2014 Free Software Foundation, Inc. # This Makefile.in is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -15,7 +15,7 @@ @SET_MAKE@ # Makefile for `wget' utility -# Copyright (C) 2013 Free Software Foundation, Inc. +# Copyright (C) 2013, 2015 Free Software Foundation, Inc. # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -41,7 +41,17 @@ # shall include the source code for the parts of OpenSSL used as well # as that of the covered work. VPATH = @srcdir@ -am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)' +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} am__make_running_with_option = \ case $${target_option-} in \ ?) ;; \ @@ -103,10 +113,27 @@ PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ -XFAIL_TESTS = +@HAVE_PYTHON3_TRUE@TESTS = Test-auth-basic-fail.py Test-auth-basic.py \ +@HAVE_PYTHON3_TRUE@ Test-auth-both.py Test-auth-digest.py \ +@HAVE_PYTHON3_TRUE@ Test-auth-no-challenge.py \ +@HAVE_PYTHON3_TRUE@ Test-auth-no-challenge-url.py \ +@HAVE_PYTHON3_TRUE@ Test-auth-retcode.py \ +@HAVE_PYTHON3_TRUE@ Test-auth-with-content-disposition.py \ +@HAVE_PYTHON3_TRUE@ Test-c-full.py \ +@HAVE_PYTHON3_TRUE@ Test-Content-disposition-2.py \ +@HAVE_PYTHON3_TRUE@ Test-Content-disposition.py \ +@HAVE_PYTHON3_TRUE@ Test--convert-links--content-on-error.py \ +@HAVE_PYTHON3_TRUE@ Test-cookie-401.py \ +@HAVE_PYTHON3_TRUE@ Test-cookie-domain-mismatch.py \ +@HAVE_PYTHON3_TRUE@ Test-cookie-expires.py Test-cookie.py \ +@HAVE_PYTHON3_TRUE@ Test-Head.py Test--https.py \ +@HAVE_PYTHON3_TRUE@ Test--https-crl.py Test-hsts.py Test-O.py \ +@HAVE_PYTHON3_TRUE@ Test-Post.py Test-504.py Test--spider-r.py \ +@HAVE_PYTHON3_TRUE@ Test--rejected-log.py \ +@HAVE_PYTHON3_TRUE@ Test-redirect-crash.py \ +@HAVE_PYTHON3_TRUE@ Test-reserved-chars.py Test-condget.py \ +@HAVE_PYTHON3_TRUE@ $(SSL_TESTS) $(am__EXEEXT_1) subdir = testenv -DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ - $(top_srcdir)/build-aux/test-driver README ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/00gnulib.m4 \ $(top_srcdir)/m4/absolute-header.m4 $(top_srcdir)/m4/alloca.m4 \ @@ -168,8 +195,9 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/00gnulib.m4 \ $(top_srcdir)/m4/realloc.m4 $(top_srcdir)/m4/regex.m4 \ $(top_srcdir)/m4/sched_h.m4 $(top_srcdir)/m4/secure_getenv.m4 \ $(top_srcdir)/m4/select.m4 $(top_srcdir)/m4/servent.m4 \ - $(top_srcdir)/m4/sha1.m4 $(top_srcdir)/m4/sig_atomic_t.m4 \ - $(top_srcdir)/m4/sigaction.m4 $(top_srcdir)/m4/signal_h.m4 \ + $(top_srcdir)/m4/sha1.m4 $(top_srcdir)/m4/sha256.m4 \ + $(top_srcdir)/m4/sig_atomic_t.m4 $(top_srcdir)/m4/sigaction.m4 \ + $(top_srcdir)/m4/signal_h.m4 \ $(top_srcdir)/m4/signalblocking.m4 $(top_srcdir)/m4/sigpipe.m4 \ $(top_srcdir)/m4/size_max.m4 $(top_srcdir)/m4/snprintf.m4 \ $(top_srcdir)/m4/socketlib.m4 $(top_srcdir)/m4/sockets.m4 \ @@ -209,6 +237,7 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/00gnulib.m4 \ $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) mkinstalldirs = $(install_sh) -d CONFIG_HEADER = $(top_builddir)/src/config.h CONFIG_CLEAN_FILES = @@ -417,6 +446,8 @@ am__set_TESTS_bases = \ bases=`echo $$bases` RECHECK_LOGS = $(TEST_LOGS) AM_RECURSIVE_TARGETS = check recheck +@METALINK_IS_ENABLED_TRUE@am__EXEEXT_1 = Test-metalink-xml.py \ +@METALINK_IS_ENABLED_TRUE@ Test-metalink-http.py TEST_SUITE_LOG = test-suite.log am__test_logs1 = $(TESTS:=.log) am__test_logs2 = $(am__test_logs1:@EXEEXT@.log=.log) @@ -433,6 +464,8 @@ am__set_b = \ *) \ b='$*';; \ esac +am__DIST_COMMON = $(srcdir)/Makefile.in \ + $(top_srcdir)/build-aux/test-driver README DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) pkglibexecdir = @pkglibexecdir@ ACLOCAL = @ACLOCAL@ @@ -716,6 +749,7 @@ GNULIB_SYMLINKAT = @GNULIB_SYMLINKAT@ GNULIB_SYSTEM_POSIX = @GNULIB_SYSTEM_POSIX@ GNULIB_TIMEGM = @GNULIB_TIMEGM@ GNULIB_TIME_R = @GNULIB_TIME_R@ +GNULIB_TIME_RZ = @GNULIB_TIME_RZ@ GNULIB_TMPFILE = @GNULIB_TMPFILE@ GNULIB_TOWCTRANS = @GNULIB_TOWCTRANS@ GNULIB_TTYNAME_R = @GNULIB_TTYNAME_R@ @@ -778,6 +812,9 @@ GNULIB_WRITE = @GNULIB_WRITE@ GNULIB__EXIT = @GNULIB__EXIT@ GNUTLS_CFLAGS = @GNUTLS_CFLAGS@ GNUTLS_LIBS = @GNUTLS_LIBS@ +GPGME_CFLAGS = @GPGME_CFLAGS@ +GPGME_CONFIG = @GPGME_CONFIG@ +GPGME_LIBS = @GPGME_LIBS@ GREP = @GREP@ HAVE_ACCEPT4 = @HAVE_ACCEPT4@ HAVE_ARPA_INET_H = @HAVE_ARPA_INET_H@ @@ -969,6 +1006,7 @@ HAVE_SYS_TIME_H = @HAVE_SYS_TIME_H@ HAVE_SYS_TYPES_H = @HAVE_SYS_TYPES_H@ HAVE_SYS_UIO_H = @HAVE_SYS_UIO_H@ HAVE_TIMEGM = @HAVE_TIMEGM@ +HAVE_TIMEZONE_T = @HAVE_TIMEZONE_T@ HAVE_TYPE_VOLATILE_SIG_ATOMIC_T = @HAVE_TYPE_VOLATILE_SIG_ATOMIC_T@ HAVE_UNISTD_H = @HAVE_UNISTD_H@ HAVE_UNLINKAT = @HAVE_UNLINKAT@ @@ -1075,6 +1113,8 @@ LTLIBPTH = @LTLIBPTH@ LTLIBSSL = @LTLIBSSL@ LTLIBTHREAD = @LTLIBTHREAD@ MAKEINFO = @MAKEINFO@ +METALINK_CFLAGS = @METALINK_CFLAGS@ +METALINK_LIBS = @METALINK_LIBS@ MKDIR_P = @MKDIR_P@ MSGFMT = @MSGFMT@ MSGFMT_015 = @MSGFMT_015@ @@ -1361,6 +1401,7 @@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ am__include = @am__include@ am__leading_dot = @am__leading_dot@ @@ -1418,35 +1459,17 @@ target_alias = @target_alias@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ +@METALINK_IS_ENABLED_FALSE@METALINK_TESTS = +@METALINK_IS_ENABLED_TRUE@METALINK_TESTS = Test-metalink-xml.py \ +@METALINK_IS_ENABLED_TRUE@ Test-metalink-http.py + AUTOMAKE_OPTIONS = parallel-tests AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; MAKE_CHECK=True; export MAKE_CHECK;\ export PYTHONPATH=$$PYTHONPATH:$(srcdir); export VALGRIND_TESTS="@VALGRIND_TESTS@"; -@HAVE_PYTHON3_TRUE@TESTS = Test-auth-basic-fail.py \ -@HAVE_PYTHON3_TRUE@ Test-auth-basic.py \ -@HAVE_PYTHON3_TRUE@ Test-auth-both.py \ -@HAVE_PYTHON3_TRUE@ Test-auth-digest.py \ -@HAVE_PYTHON3_TRUE@ Test-auth-no-challenge.py \ -@HAVE_PYTHON3_TRUE@ Test-auth-no-challenge-url.py \ -@HAVE_PYTHON3_TRUE@ Test-auth-retcode.py \ -@HAVE_PYTHON3_TRUE@ Test-auth-with-content-disposition.py \ -@HAVE_PYTHON3_TRUE@ Test-c-full.py \ -@HAVE_PYTHON3_TRUE@ Test-Content-disposition-2.py \ -@HAVE_PYTHON3_TRUE@ Test-Content-disposition.py \ -@HAVE_PYTHON3_TRUE@ Test-cookie-401.py \ -@HAVE_PYTHON3_TRUE@ Test-cookie-domain-mismatch.py \ -@HAVE_PYTHON3_TRUE@ Test-cookie-expires.py \ -@HAVE_PYTHON3_TRUE@ Test-cookie.py \ -@HAVE_PYTHON3_TRUE@ Test-Head.py \ -@HAVE_PYTHON3_TRUE@ Test--https.py \ -@HAVE_PYTHON3_TRUE@ Test--https-crl.py \ -@HAVE_PYTHON3_TRUE@ Test-O.py \ -@HAVE_PYTHON3_TRUE@ Test-Post.py \ -@HAVE_PYTHON3_TRUE@ Test-504.py \ -@HAVE_PYTHON3_TRUE@ Test--spider-r.py \ -@HAVE_PYTHON3_TRUE@ Test-redirect-crash.py - -EXTRA_DIST = certs conf exc misc server test README $(TESTS) $(XFAIL_TESTS) +SSL_TESTS = Test--https.py Test--https-crl.py +@HAVE_PYTHON3_TRUE@@WITH_SSL_FALSE@XFAIL_TESTS = $(SSL_TESTS) +EXTRA_DIST = certs conf exc misc server test README $(TESTS) TEST_EXTENSIONS = .py PY_LOG_COMPILER = python3 AM_PY_LOG_FLAGS = -O @@ -1466,7 +1489,6 @@ $(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu testenv/Makefile'; \ $(am__cd) $(top_srcdir) && \ $(AUTOMAKE) --gnu testenv/Makefile -.PRECIOUS: Makefile Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status @case '$?' in \ *config.status*) \ @@ -1521,7 +1543,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) if test -n "$$am__remaking_logs"; then \ echo "fatal: making $(TEST_SUITE_LOG): possible infinite" \ "recursion detected" >&2; \ - else \ + elif test -n "$$redo_logs"; then \ am__remaking_logs=yes $(MAKE) $(AM_MAKEFLAGS) $$redo_logs; \ fi; \ if $(am__make_dryrun); then :; else \ @@ -1794,6 +1816,8 @@ uninstall-am: maintainer-clean-generic mostlyclean mostlyclean-generic pdf \ pdf-am ps ps-am recheck tags-am uninstall uninstall-am +.PRECIOUS: Makefile + @HAVE_PYTHON3_TRUE@ # added test cases expected to fail here and under TESTS diff --git a/testenv/README b/testenv/README index 081a957..50baf3d 100644 --- a/testenv/README +++ b/testenv/README @@ -124,7 +124,7 @@ WgetFile (str name, str contents, str timestamp, dict rules) None except name is a mandatory paramter, one may pass only those parameters that are required by the File object. -The timestamp string should be a valid Unix Timestamp as defined in RFC xxxx. +The timestamp string should be in a format: "YYYY-MM-DD HH:MM:SS" in UTC zone. The rules object is a dictionary element, with the key as the Rule Name and value as the Rule Data. In most cases, the Rule Data is another dictionary. @@ -184,7 +184,8 @@ This section lists the currently supported File Rules and their structure. * SendHeader : This list of Headers will be sent in EVERY response to a request for the respective file. It follows the same value format as - ExpectHeader. + ExpectHeader. Additionally you can specify a list of strings as <Header Data> + if you want the header repeated with multiple values. * Response : The HTTP Response Code to send to a request for this File. The value is an Integer that represents a valid HTTP Response Code. diff --git a/testenv/Test--convert-links--content-on-error.py b/testenv/Test--convert-links--content-on-error.py new file mode 100755 index 0000000..bfa9d9b --- /dev/null +++ b/testenv/Test--convert-links--content-on-error.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + +""" + This test ensures that Wget link conversion works also on HTTP error pages. +""" +TEST_NAME = "Test--convert-links--content-on-error" +############# File Definitions ############################################### +a_x_FileContent = """ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title></title> +</head> +<body> + <a href="/b/y.html"></a> +</body> +</html> +""" +a_x_LocalFileContent = """ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title></title> +</head> +<body> + <a href="../b/y.html"></a> +</body> +</html> +""" + +error_FileContent = '404 page content' +error_FileRules = { + 'Response': 404 , + 'SendHeader': { + 'Content-Length': len(error_FileContent), + 'Content-Type': 'text/plain', + } +} + +a_x_File = WgetFile ("a/x.html", a_x_FileContent) +robots_File = WgetFile ("robots.txt", '') +error_File = WgetFile ("b/y.html", error_FileContent, rules=error_FileRules) + +B_File = WgetFile ("a/x.html", a_x_LocalFileContent) + +WGET_OPTIONS = "--no-host-directories -r -l2 --convert-links --content-on-error" +WGET_URLS = [["a/x.html"]] + +Files = [[a_x_File, robots_File, error_File]] + +ExpectedReturnCode = 8 +ExpectedDownloadedFiles = [B_File, robots_File, error_File] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +err = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test +).begin () + +exit (err) diff --git a/testenv/Test--rejected-log.py b/testenv/Test--rejected-log.py new file mode 100755 index 0000000..a94ae3b --- /dev/null +++ b/testenv/Test--rejected-log.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + +""" + This test executed Wget in recursive mode with a rejected log outputted. +""" +TEST_NAME = "Rejected Log" +############# File Definitions ############################################### +mainpage = """ +<html> +<head> + <title>Main Page</title> +</head> +<body> + <p> + Recurse to a <a href="http://127.0.0.1:{{port}}/secondpage.html">second page</a>. + </p> +</body> +</html> +""" + +secondpage = """ +<html> +<head> + <title>Second Page</title> +</head> +<body> + <p> + Recurse to a <a href="http://127.0.0.1:{{port}}/thirdpage.html">third page</a>. + Try the blacklisted <a href="http://127.0.0.1:{{port}}/index.html">main page</a>. + </p> +</body> +</html> +""" + +thirdpage = """ +<html> +<head> + <title>Third Page</title> +</head> +<body> + <p> + Try a hidden <a href="http://127.0.0.1:{{port}}/dummy.txt">dummy file</a>. + Try to leave to <a href="http://no.such.domain/">another domain</a>. + </p> +</body> +</html> +""" + +robots = """ +User-agent: * +Disallow: /dummy.txt +""" + +log = """\ +REASON U_URL U_SCHEME U_HOST U_PORT U_PATH U_PARAMS U_QUERY U_FRAGMENT P_URL P_SCHEME P_HOST P_PORT P_PATH P_PARAMS P_QUERY P_FRAGMENT +BLACKLIST http%3A//127.0.0.1%3A{{port}}/index.html SCHEME_HTTP 127.0.0.1 {{port}} index.html http%3A//127.0.0.1%3A{{port}}/secondpage.html SCHEME_HTTP 127.0.0.1 {{port}} secondpage.html +ROBOTS http%3A//127.0.0.1%3A{{port}}/dummy.txt SCHEME_HTTP 127.0.0.1 {{port}} dummy.txt http%3A//127.0.0.1%3A{{port}}/thirdpage.html SCHEME_HTTP 127.0.0.1 {{port}} thirdpage.html +SPANNEDHOST http%3A//no.such.domain/ SCHEME_HTTP no.such.domain 80 http%3A//127.0.0.1%3A{{port}}/thirdpage.html SCHEME_HTTP 127.0.0.1 {{port}} thirdpage.html +""" + +dummyfile = "Don't care." + + +index_html = WgetFile ("index.html", mainpage) +secondpage_html = WgetFile ("secondpage.html", secondpage) +thirdpage_html = WgetFile ("thirdpage.html", thirdpage) +robots_txt = WgetFile ("robots.txt", robots) +dummy_txt = WgetFile ("dummy.txt", dummyfile) +log_csv = WgetFile ("log.csv", log) + +WGET_OPTIONS = "-nd -r --rejected-log log.csv" +WGET_URLS = [["index.html"]] + +Files = [[index_html, secondpage_html, thirdpage_html, robots_txt, dummy_txt]] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [index_html, secondpage_html, thirdpage_html, robots_txt, log_csv] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +err = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test +).begin () + +exit (err) diff --git a/testenv/Test-auth-both.py b/testenv/Test-auth-both.py index 91d72b8..23bdaf5 100755 --- a/testenv/Test-auth-both.py +++ b/testenv/Test-auth-both.py @@ -18,7 +18,10 @@ File1_rules = { "Authentication" : { "Type" : "Both", "User" : "Sauron", - "Pass" : "TheEye" + "Pass" : "TheEye", + "Parm" : { + "qop" : "auth" + } }, "RejectHeader" : { "Authorization" : "Basic U2F1cm9uOlRoZUV5ZQ==" @@ -28,7 +31,10 @@ File2_rules = { "Authentication" : { "Type" : "Both_inline", "User" : "Sauron", - "Pass" : "TheEye" + "Pass" : "TheEye", + "Parm" : { + "qop" : "auth" + } }, "RejectHeader" : { "Authorization" : "Basic U2F1cm9uOlRoZUV5ZQ==" @@ -38,7 +44,11 @@ File3_rules = { "Authentication" : { "Type" : "Digest", "User" : "Sauron", - "Pass" : "TheEye" + "Pass" : "TheEye", + "Parm" : { + "qop" : "auth" + } + } } diff --git a/testenv/Test-auth-digest.py b/testenv/Test-auth-digest.py index 8a73c0d..6f58daa 100755 --- a/testenv/Test-auth-digest.py +++ b/testenv/Test-auth-digest.py @@ -9,23 +9,39 @@ from misc.wget_file import WgetFile TEST_NAME = "Digest Authorization" ############# File Definitions ############################################### File1 = "Need a cookie?" +File2 = "Want cookies with milk!" File1_rules = { "Authentication" : { "Type" : "Digest", "User" : "Pacman", - "Pass" : "Omnomnom" + "Pass" : "Omnomnom", + "Parm" : { + "qop" : "auth" + } + } +} + +File2_rules = { + "Authentication" : { + "Type" : "Digest", + "User" : "Pacman", + "Pass" : "Omnomnom", + "Parm" : { + "qop" : None + } } } A_File = WgetFile ("File1", File1, rules=File1_rules) +B_File = WgetFile ("File2", File2, rules=File2_rules) WGET_OPTIONS = "--user=Pacman --password=Omnomnom" -WGET_URLS = [["File1"]] +WGET_URLS = [["File1", "File2"]] -Files = [[A_File]] +Files = [[A_File, B_File]] ExpectedReturnCode = 0 -ExpectedDownloadedFiles = [A_File] +ExpectedDownloadedFiles = [A_File, B_File] ################ Pre and Post Test Hooks ##################################### pre_test = { diff --git a/testenv/Test-condget.py b/testenv/Test-condget.py new file mode 100755 index 0000000..c9e8b2a --- /dev/null +++ b/testenv/Test-condget.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + +""" + Simple test for HTTP Conditional-GET Requests using the -N command +""" +TEST_NAME = "HTTP Conditional-GET Requests" +############# File Definitions ############################################### +# Keep same length ! +Cont1 = """THIS IS 1 FILE""" +Cont2 = """THIS IS 2 FILE""" +Cont3 = """THIS IS 3 FILE""" +Cont4 = """THIS IS 4 FILE""" + +# Local Wget files + +# These have same timestamp as remote files +UpToDate_Local_File1 = WgetFile ("UpToDateFile1", Cont1, timestamp="1995-01-01 00:00:00") +UpToDate_Local_File2 = WgetFile ("UpToDateFile2", Cont1, timestamp="1995-01-01 00:00:00") +UpToDate_Local_File3 = WgetFile ("UpToDateFile3", Cont1, timestamp="1995-01-01 00:00:00") + +# This is newer than remote (expected same behaviour as for above files) +Newer_Local_File = WgetFile ("NewerFile", Cont1, timestamp="1995-02-02 02:02:02") + +# This is older than remote - should be clobbered +Outdated_Local_File = WgetFile ("UpdatedFile", Cont2, timestamp="1990-01-01 00:00:00") + +UpToDate_Rules1 = { + "SendHeader" : { + # RFC1123 format + "Last-Modified" : "Sun, 01 Jan 1995 00:00:00 GMT", + }, + "Response": 304, + "ExpectHeader" : { + "If-Modified-Since" : "Sun, 01 Jan 1995 00:00:00 GMT" + }, +} + +UpToDate_Rules2 = { + "SendHeader" : { + # RFC850 format + "Last-Modified" : "Sunday, 01-Jan-95 00:00:00 GMT", + }, + "Response": 304, + "ExpectHeader" : { + "If-Modified-Since" : "Sun, 01 Jan 1995 00:00:00 GMT" + }, +} + +UpToDate_Rules3 = { + "SendHeader" : { + # Asctime format + "Last-Modified" : "Sun Jan 01 00:00:00 1995", + }, + "Response": 304, + "ExpectHeader" : { + "If-Modified-Since" : "Sun, 01 Jan 1995 00:00:00 GMT" + }, +} + +Newer_Rules = { + "SendHeader" : { + # Asctime format + "Last-Modified" : "Sun Jan 01 00:00:00 1995", + }, + "Response": 304, + "ExpectHeader" : { + "If-Modified-Since" : "Thu, 02 Feb 1995 02:02:02 GMT" + }, +} + +Outdated_Rules = { + "SendHeader" : { + # RFC850 format + "Last-Modified" : "Thursday, 01-Jan-15 00:00:00 GMT", + }, + "ExpectHeader" : { + "If-Modified-Since" : "Mon, 01 Jan 1990 00:00:00 GMT", + }, +} + +UpToDate_Server_File1 = WgetFile ("UpToDateFile1", Cont3, rules=UpToDate_Rules1) +UpToDate_Server_File2 = WgetFile ("UpToDateFile2", Cont3, rules=UpToDate_Rules2) +UpToDate_Server_File3 = WgetFile ("UpToDateFile3", Cont3, rules=UpToDate_Rules3) +Newer_Server_File = WgetFile ("NewerFile", Cont3, rules=Newer_Rules) +Updated_Server_File = WgetFile ("UpdatedFile", Cont4, rules=Outdated_Rules) + +WGET_OPTIONS = "-N" +WGET_URLS = [["UpToDateFile1", "UpToDateFile2", "UpToDateFile3", "NewerFile", + "UpdatedFile", ]] + +Files = [[UpToDate_Server_File1, UpToDate_Server_File2, UpToDate_Server_File3, + Newer_Server_File, Updated_Server_File, ]] + +Existing_Files = [UpToDate_Local_File1, UpToDate_Local_File2, + UpToDate_Local_File3, Newer_Local_File, Outdated_Local_File] + +ExpectedReturnCode = 0 + +# The uptodate file should not be downloaded +ExpectedDownloadedFiles = [UpToDate_Local_File1, UpToDate_Local_File2, + UpToDate_Local_File3, Newer_Local_File, + Updated_Server_File] + +# Kind of hack to ensure proper request types +Request_List = [ + [ + "GET /UpToDateFile1", + "GET /UpToDateFile2", + "GET /UpToDateFile3", + "GET /NewerFile", + "GET /UpdatedFile", + ] +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode, + "FilesCrawled" : Request_List, +} + +err = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test +).begin () + +exit (err) diff --git a/testenv/Test-hsts.py b/testenv/Test-hsts.py new file mode 100755 index 0000000..4290929 --- /dev/null +++ b/testenv/Test-hsts.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from test.base_test import HTTP, HTTPS +from misc.wget_file import WgetFile +import time +import os + +""" +This test makes sure Wget can parse a given HSTS database and apply the indicated HSTS policy. +""" +def hsts_database_path(): + hsts_file = ".wget-hsts-testenv" + return os.path.abspath(hsts_file) + +def create_hsts_database(path, host, port): + # we want the current time as an integer, + # not as a floating point + curtime = int(time.time()) + max_age = "123456" + + f = open(path, "w") + + f.write("# dummy comment\n") + f.write(host + "\t" + str(port) + "\t0\t" + str(curtime) + "\t" + max_age + "\n") + f.close() + +TEST_NAME = "HSTS basic test" + +File_Name = "hw" +File_Content = "Hello, world!" +File = WgetFile(File_Name, File_Content) + +Hsts_File_Path = hsts_database_path() + +CAFILE = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs', 'ca-cert.pem')) + +WGET_OPTIONS = "--hsts-file=" + Hsts_File_Path + " --ca-certificate=" + CAFILE +WGET_URLS = [[File_Name]] + +Files = [[File]] +Servers = [HTTPS] +Requests = ["http"] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [File] + +pre_test = { + "ServerFiles" : Files, + "Domains" : ["localhost"] +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetCode" : ExpectedReturnCode, +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} + +test = HTTPTest( + name = TEST_NAME, + pre_hook = pre_test, + post_hook = post_test, + test_params = test_options, + protocols = Servers, + req_protocols = Requests +) + +# start the web server and create the temporary HSTS database +test.setup() +create_hsts_database(Hsts_File_Path, 'localhost', test.port) + +err = test.begin() + +# remove the temporary HSTS database +os.unlink(hsts_database_path()) +exit(err) diff --git a/testenv/Test-metalink-http.py b/testenv/Test-metalink-http.py new file mode 100755 index 0000000..53b0801 --- /dev/null +++ b/testenv/Test-metalink-http.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import re +import hashlib +from base64 import b64encode + +""" + This is to test Metalink as HTTP file support in Wget. +""" +TEST_NAME = "Metalink in HTTP" + +# Helper function for hostname, port and digest substitution +def SubstituteServerInfo (text, host, port, digest): + text = re.sub (r'{{FILE1_HASH}}', digest, text) + text = re.sub (r'{{SRV_HOST}}', host, text) + text = re.sub (r'{{SRV_PORT}}', str (port), text) + return text + +############# File Definitions ############################################### +File1 = "Would you like some Tea?" +File1_corrupted = "Would you like some Coffee?" +File1_lowPref = "Do not take this" +File1_sha256 = b64encode (hashlib.sha256 (File1.encode ('UTF-8')).digest ()).decode ('ascii') +Signature = '''-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.7 (GNU/Linux) + +This is no valid signature. But it should be downloaded. +The attempt to verify should fail but should not prevent +a successful metalink resource retrieval (the sig failure +should not be fatal). +-----END PGP SIGNATURE----- +''' +File2 = "No meta data for this file." + +LinkHeaders = [ + # This file has low priority and should not be picked. + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref>; rel=duplicate; pri=9; geo=pl", + # This file should be picked second, after hash failure. + "<http://this.is.no.good.example/File1_try2_badconnection>; rel =duplicate;pref; pri=7", + # This signature download will fail. + "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig2.asc>; rel=describedby; type=application/pgp-signature", + # Two good signatures + "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig.asc>; rel=describedby; type=application/pgp-signature", + "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig.asc>; rel=describedby; type=application/pgp-signature", + # Bad URL scheme + "<invalid_url>; rel=duplicate; pri=4", + # rel missing + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1>; pri=1; pref", + # invalid rel + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1>; rel=strange; pri=4", + # This file should be picked first, because it has the lowest pri among preferred. + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_try1_corrupted>; rel=duplicate; geo=su; pri=4; pref", + # This file should NOT be picked third due to preferred location set to 'uk' + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_badgeo>; rel =duplicate;pri=5", + # This file should be picked as third try, and it should succeed + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_try3_ok>; rel=duplicate; pri=5;geo=uk" + ] +DigestHeader = "SHA-256={{FILE1_HASH}}" + +# This will be filled as soon as we know server hostname and port +MetaFileRules = {'SendHeader' : {}} + +FileOkServer = WgetFile ("File1_try3_ok", File1) +FileBadPref = WgetFile ("File1_lowPref", File1_lowPref) +FileBadHash = WgetFile ("File1_try1_corrupted", File1_corrupted) +MetaFile = WgetFile ("test.meta", rules=MetaFileRules) +# In case of Metalink over HTTP, the local file name is +# derived from the URL suffix. +FileOkLocal = WgetFile ("test.meta", File1) +SigFile = WgetFile ("Sig.asc", Signature) +FileNoMeta = WgetFile ("File2", File2) + +WGET_OPTIONS = "--metalink-over-http --preferred-location=uk" +WGET_URLS = [["test.meta", "File2"]] + +Files = [[FileOkServer, FileBadPref, FileBadHash, MetaFile, SigFile, FileNoMeta]] +Existing_Files = [] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [FileNoMeta, FileOkLocal] + +RequestList = [ + [ + "HEAD /test.meta", + "GET /Sig2.asc", + "GET /Sig.asc", + "GET /File1_try1_corrupted", + "GET /File1_try3_ok", + "HEAD /File2", + "GET /File2", + ] +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode, + "FilesCrawled" : RequestList, +} + +http_test = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaFileRules["SendHeader"] = { + 'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port, File1_sha256) + for LinkHeader in LinkHeaders ], + 'Digest': SubstituteServerInfo (DigestHeader, srv_host, srv_port, File1_sha256), +} + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-metalink-xml.py b/testenv/Test-metalink-xml.py new file mode 100755 index 0000000..eded04d --- /dev/null +++ b/testenv/Test-metalink-xml.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import re +import hashlib + +""" + This is to test Metalink as XML file support in Wget. +""" +TEST_NAME = "Metalink in XML" +############# File Definitions ############################################### +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () +MetaXml = \ +"""<?xml version="1.0" encoding="utf-8"?> +<metalink version="3.0" xmlns="http://www.metalinker.org/"> + <publisher> + <name>GNU Wget</name> + </publisher> + <license> + <name>GNU GPL</name> + <url>http://www.gnu.org/licenses/gpl.html</url> + </license> + <identity>Wget Test File 1</identity> + <version>1.2.3</version> + <description>Wget Test File 1 description</description> + <files> + <file name="File1"> + <verification> + <hash type="sha256">{{FILE1_HASH}}</hash> + </verification> + <resources> + <url type="http" preference="40">http://broken.example/File1</url> + <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url> + <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url> + </resources> + </file> + </files> +</metalink> +""" + +A_File = WgetFile ("File1", File1) +B_File = WgetFile ("File1_lowPref", File1_lowPref) +MetaFile = WgetFile ("test.meta4", MetaXml) + +WGET_OPTIONS = "--input-metalink test.meta4" +WGET_URLS = [[]] + +Files = [[A_File, B_File]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [A_File, MetaFile] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = re.sub (r'{{FILE1_HASH}}', File1_sha256, MetaXml) +MetaXml = re.sub (r'{{SRV_HOST}}', srv_host, MetaXml) +MetaXml = re.sub (r'{{SRV_PORT}}', str (srv_port), MetaXml) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) diff --git a/testenv/Test-reserved-chars.py b/testenv/Test-reserved-chars.py new file mode 100755 index 0000000..e5d33d0 --- /dev/null +++ b/testenv/Test-reserved-chars.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +from sys import exit +from os import environ # to set LC_ALL +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + +""" +This test ensures that Wget keeps reserved characters in URLs in non-UTF-8 charsets. +""" +# This bug only happened with ASCII charset, +# so we need to set LC_ALL="C" in order to reproduce it. +environ["LC_ALL"] = "C" + +TEST_NAME = "URLs with reserved characters" +######### File Definitions ######### +RequestList = [ + [ + "HEAD /base.html", + "GET /base.html", + "GET /robots.txt", + "HEAD /a%2Bb.html", + "GET /a%2Bb.html" + ] +] +A_File_Name = "base.html" +B_File_Name = "a%2Bb.html" +A_File = WgetFile (A_File_Name, "<a href=\"a%2Bb.html\">") +B_File = WgetFile (B_File_Name, "this is file B") + +WGET_OPTIONS = " --spider -r" +WGET_URLS = [[A_File_Name]] + +Files = [[A_File, B_File]] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [] + +######### Pre and Post Test Hooks ######### +pre_test = { + "ServerFiles" : Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode, + "FilesCrawled" : RequestList +} + +err = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test +).begin () + +exit (err) diff --git a/testenv/conf/__init__.py b/testenv/conf/__init__.py index 4b5ddc4..55433c9 100644 --- a/testenv/conf/__init__.py +++ b/testenv/conf/__init__.py @@ -3,6 +3,7 @@ import os # this file implements the mechanism of conf class auto-registration, # don't modify this file if you have no idea what you're doing + def gen_hook(): hook_table = {} diff --git a/testenv/conf/authentication.py b/testenv/conf/authentication.py index c87994a..ca5149c 100644 --- a/testenv/conf/authentication.py +++ b/testenv/conf/authentication.py @@ -16,7 +16,8 @@ that. @rule() class Authentication: - def __init__ (self, auth_obj): + def __init__(self, auth_obj): self.auth_type = auth_obj['Type'] self.auth_user = auth_obj['User'] self.auth_pass = auth_obj['Pass'] + self.auth_parm = auth_obj.get('Parm', None) diff --git a/testenv/conf/domains.py b/testenv/conf/domains.py new file mode 100644 index 0000000..ac03fe1 --- /dev/null +++ b/testenv/conf/domains.py @@ -0,0 +1,9 @@ +from conf import hook + +@hook(alias='Domains') +class Domains: + def __init__(self, domains): + self.domains = domains + + def __call__(self, test_obj): + test_obj.domains = self.domains diff --git a/testenv/conf/expected_files.py b/testenv/conf/expected_files.py index 2c8d632..5362771 100644 --- a/testenv/conf/expected_files.py +++ b/testenv/conf/expected_files.py @@ -37,9 +37,10 @@ class ExpectedFiles: for file in self.expected_fs: if file.name in local_fs: local_file = local_fs.pop(file.name) - if file.content != local_file['content']: + formatted_content = test_obj._replace_substring(file.content) + if formatted_content != local_file['content']: for line in unified_diff(local_file['content'], - file.content, + formatted_content, fromfile='Actual', tofile='Expected'): print(line, file=sys.stderr) diff --git a/testenv/conf/files_crawled.py b/testenv/conf/files_crawled.py index 334e596..7db8392 100644 --- a/testenv/conf/files_crawled.py +++ b/testenv/conf/files_crawled.py @@ -23,5 +23,5 @@ class FilesCrawled: diff = headers.symmetric_difference(remaining) if diff: - print_red (str(diff)) + print_red(str(diff)) raise TestFailed('Not all files were crawled correctly.') diff --git a/testenv/conf/hook_sample.py b/testenv/conf/hook_sample.py index 6230a70..591ec3b 100644 --- a/testenv/conf/hook_sample.py +++ b/testenv/conf/hook_sample.py @@ -18,5 +18,5 @@ class SampleHook: # implement hook here # if you need the test case instance, refer to test_obj if False: - raise TestFailed ("Reason") + raise TestFailed("Reason") pass diff --git a/testenv/conf/local_files.py b/testenv/conf/local_files.py index 5f9c8fa..908ced1 100644 --- a/testenv/conf/local_files.py +++ b/testenv/conf/local_files.py @@ -1,3 +1,6 @@ +from os import utime +from time import strptime +from calendar import timegm from conf import hook """ Pre-Test Hook: LocalFiles @@ -16,3 +19,8 @@ class LocalFiles: for f in self.local_files: with open(f.name, 'w') as fp: fp.write(f.content) + if f.timestamp is not None: + tstamp = timegm(strptime(f.timestamp, '%Y-%m-%d %H:%M:%S')) + atime = tstamp + mtime = tstamp + utime(f.name, (atime, mtime)) diff --git a/testenv/conf/reject_header.py b/testenv/conf/reject_header.py index 53e237d..0dcf463 100644 --- a/testenv/conf/reject_header.py +++ b/testenv/conf/reject_header.py @@ -9,5 +9,5 @@ requests. @rule() class RejectHeader: - def __init__ (self, header_obj): + def __init__(self, header_obj): self.headers = header_obj diff --git a/testenv/conf/server_files.py b/testenv/conf/server_files.py index 1e9d346..eaa9cd0 100644 --- a/testenv/conf/server_files.py +++ b/testenv/conf/server_files.py @@ -3,7 +3,7 @@ from conf import hook """ Pre-Test Hook: ServerFiles This hook is used to define a set of files on the server's virtual filesystem. server_files is expected to be dictionary that maps filenames to their -contents. In the future, this can be used to add additional metadat to the +contents. In the future, this can be used to add additional metadata to the files using the WgetFile class too. This hook also does some additional processing on the contents of the file. Any @@ -19,8 +19,8 @@ class ServerFiles: def __call__(self, test_obj): for server, files in zip(test_obj.servers, self.server_files): - rules = {f.name: test_obj.get_server_rules(f) - for f in files} - files = {f.name: test_obj._replace_substring(f.content) - for f in files} - server.server_conf(files, rules) + files_content = {f.name: test_obj._replace_substring(f.content) + for f in files} + files_rules = {f.name: test_obj.get_server_rules(f) + for f in files} + server.server_conf(files_content, files_rules) diff --git a/testenv/exc/server_error.py b/testenv/exc/server_error.py index b8a37ce..fe359f5 100644 --- a/testenv/exc/server_error.py +++ b/testenv/exc/server_error.py @@ -3,5 +3,17 @@ class ServerError (Exception): """ A custom exception which is raised by the test servers. Often used to handle control flow. """ - def __init__ (self, err_message): + def __init__(self, err_message): self.err_message = err_message + +class NoBodyServerError (Exception): + """ A custom exception which is raised by the test servers. + Used if no body should be sent in response. """ + + def __init__(self, err_message): + self.err_message = err_message + +class AuthError (ServerError): + """ A custom exception raised byt he servers when authentication of the + request fails. """ + pass diff --git a/testenv/exc/test_failed.py b/testenv/exc/test_failed.py index de5e02a..89f7960 100644 --- a/testenv/exc/test_failed.py +++ b/testenv/exc/test_failed.py @@ -3,5 +3,5 @@ class TestFailed(Exception): """ A Custom Exception raised by the Test Environment. """ - def __init__ (self, error): + def __init__(self, error): self.error = error diff --git a/testenv/misc/colour_terminal.py b/testenv/misc/colour_terminal.py index b0849c1..bc549a2 100644 --- a/testenv/misc/colour_terminal.py +++ b/testenv/misc/colour_terminal.py @@ -1,6 +1,7 @@ from functools import partial import platform from os import getenv +import sys """ This module allows printing coloured output to the terminal when running a Wget Test under certain conditions. @@ -17,22 +18,23 @@ codes on;y add clutter. """ T_COLORS = { - 'PURPLE' : '\033[95m', - 'BLUE' : '\033[94m', - 'GREEN' : '\033[92m', - 'YELLOW' : '\033[93m', - 'RED' : '\033[91m', - 'ENDC' : '\033[0m' + 'PURPLE': '\033[95m', + 'BLUE': '\033[94m', + 'GREEN': '\033[92m', + 'YELLOW': '\033[93m', + 'RED': '\033[91m', + 'ENDC': '\033[0m' } -system = True if platform.system() == 'Linux' else False +system = True if platform.system() in ('Linux', 'Darwin') else False check = False if getenv("MAKE_CHECK") == 'True' else True -def printer (color, string): - if system and check: - print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC')) + +def printer(color, string): + if sys.stdout.isatty() and system and check: + print(T_COLORS.get(color) + string + T_COLORS.get('ENDC')) else: - print (string) + print(string) print_blue = partial(printer, 'BLUE') diff --git a/testenv/misc/wget_file.py b/testenv/misc/wget_file.py index 027dced..c2a7239 100644 --- a/testenv/misc/wget_file.py +++ b/testenv/misc/wget_file.py @@ -3,7 +3,7 @@ class WgetFile: """ WgetFile is a File Data Container object """ - def __init__ ( + def __init__( self, name, content="Test Contents", diff --git a/testenv/server/http/http_server.py b/testenv/server/http/http_server.py index 9128b3e..78aa605 100644 --- a/testenv/server/http/http_server.py +++ b/testenv/server/http/http_server.py @@ -1,5 +1,5 @@ from http.server import HTTPServer, BaseHTTPRequestHandler -from exc.server_error import ServerError +from exc.server_error import ServerError, AuthError, NoBodyServerError from socketserver import BaseServer from posixpath import basename, splitext from base64 import b64encode @@ -10,50 +10,55 @@ import socket import os -class StoppableHTTPServer (HTTPServer): +class StoppableHTTPServer(HTTPServer): """ This class extends the HTTPServer class from default http.server library in Python 3. The StoppableHTTPServer class is capable of starting an HTTP server that serves a virtual set of files made by the WgetFile class and has most of its properties configurable through the server_conf() method. """ - request_headers = list () + request_headers = list() """ Define methods for configuring the Server. """ - def server_conf (self, filelist, conf_dict): + def server_conf(self, filelist, conf_dict): """ Set Server Rules and File System for this instance. """ self.server_configs = conf_dict self.fileSys = filelist - def get_req_headers (self): + def get_req_headers(self): return self.request_headers -class HTTPSServer (StoppableHTTPServer): +class HTTPSServer(StoppableHTTPServer): """ The HTTPSServer class extends the StoppableHTTPServer class with additional support for secure connections through SSL. """ - def __init__ (self, address, handler): + def __init__(self, address, handler): import ssl - BaseServer.__init__ (self, address, handler) - # step one up because test suite change directory away from $srcdir (don't do that !!!) - CERTFILE = os.path.abspath(os.path.join('..', os.getenv('srcdir', '.'), 'certs', 'server-cert.pem')) - KEYFILE = os.path.abspath(os.path.join('..', os.getenv('srcdir', '.'), 'certs', 'server-key.pem')) - fop = open (CERTFILE) - print (fop.readline()) - self.socket = ssl.wrap_socket ( - sock = socket.socket (self.address_family, self.socket_type), - ssl_version = ssl.PROTOCOL_TLSv1, - certfile = CERTFILE, - keyfile = KEYFILE, - server_side = True + BaseServer.__init__(self, address, handler) + # step one up because test suite change directory away from $srcdir + # (don't do that !!!) + CERTFILE = os.path.abspath(os.path.join('..', + os.getenv('srcdir', '.'), + 'certs', + 'server-cert.pem')) + KEYFILE = os.path.abspath(os.path.join('..', + os.getenv('srcdir', '.'), + 'certs', + 'server-key.pem')) + self.socket = ssl.wrap_socket( + sock=socket.socket(self.address_family, self.socket_type), + ssl_version=ssl.PROTOCOL_TLSv1, + certfile=CERTFILE, + keyfile=KEYFILE, + server_side=True ) self.server_bind() self.server_activate() -class _Handler (BaseHTTPRequestHandler): +class _Handler(BaseHTTPRequestHandler): """ This is a private class which tells the server *HOW* to handle each request. For each HTTP Request Command that the server should be capable of responding to, there must exist a do_REQUESTNAME() method which details the @@ -61,7 +66,7 @@ class _Handler (BaseHTTPRequestHandler): in this class are auxilliary methods created to help in processing certain requests. """ - def get_rule_list (self, name): + def get_rule_list(self, name): return self.rules.get(name) # The defailt protocol version of the server we run is HTTP/1.1 not @@ -70,23 +75,23 @@ class _Handler (BaseHTTPRequestHandler): """ Define functions for various HTTP Requests. """ - def do_HEAD (self): - self.send_head ("HEAD") + def do_HEAD(self): + self.send_head("HEAD") - def do_GET (self): + def do_GET(self): """ Process HTTP GET requests. This is the same as processing HEAD requests and then actually transmitting the data to the client. If send_head() does not specify any "start" offset, we send the complete data, else transmit only partial data. """ - content, start = self.send_head ("GET") + content, start = self.send_head("GET") if content: if start is None: - self.wfile.write (content.encode ('utf-8')) + self.wfile.write(content.encode('utf-8')) else: - self.wfile.write (content.encode ('utf-8')[start:]) + self.wfile.write(content.encode('utf-8')[start:]) - def do_POST (self): + def do_POST(self): """ According to RFC 7231 sec 4.3.3, if the resource requested in a POST request does not exist on the server, the first POST request should create that resource. PUT requests are otherwise used to create a @@ -100,70 +105,70 @@ class _Handler (BaseHTTPRequestHandler): path = self.path[1:] if path in self.server.fileSys: - self.rules = self.server.server_configs.get (path) + self.rules = self.server.server_configs.get(path) if not self.rules: - self.rules = dict () + self.rules = dict() - if not self.custom_response (): - return (None, None) + if not self.custom_response(): + return(None, None) - body_data = self.get_body_data () - self.send_response (200) - self.send_header ("Content-type", "text/plain") - content = self.server.fileSys.pop (path) + "\n" + body_data - total_length = len (content) + body_data = self.get_body_data() + self.send_response(200) + self.add_header("Content-type", "text/plain") + content = self.server.fileSys.pop(path) + "\n" + body_data + total_length = len(content) self.server.fileSys[path] = content - self.send_header ("Content-Length", total_length) - self.send_header ("Location", self.path) - self.finish_headers () + self.add_header("Content-Length", total_length) + self.add_header("Location", self.path) + self.finish_headers() try: - self.wfile.write (content.encode ('utf-8')) + self.wfile.write(content.encode('utf-8')) except Exception: pass else: - self.send_put (path) + self.send_put(path) - def do_PUT (self): + def do_PUT(self): path = self.path[1:] - self.rules = self.server.server_configs.get (path) - if not self.custom_response (): - return (None, None) - self.send_put (path) + self.rules = self.server.server_configs.get(path) + if not self.custom_response(): + return(None, None) + self.send_put(path) """ End of HTTP Request Method Handlers. """ """ Helper functions for the Handlers. """ - def parse_range_header (self, header_line, length): + def parse_range_header(self, header_line, length): import re if header_line is None: return None - if not header_line.startswith ("bytes="): - raise ServerError ("Cannot parse header Range: %s" % - (header_line)) - regex = re.match (r"^bytes=(\d*)\-$", header_line) - range_start = int (regex.group (1)) + if not header_line.startswith("bytes="): + raise ServerError("Cannot parse header Range: %s" % + (header_line)) + regex = re.match(r"^bytes=(\d*)\-$", header_line) + range_start = int(regex.group(1)) if range_start >= length: - raise ServerError ("Range Overflow") + raise ServerError("Range Overflow") return range_start - def get_body_data (self): - cLength_header = self.headers.get ("Content-Length") - cLength = int (cLength_header) if cLength_header is not None else 0 - body_data = self.rfile.read (cLength).decode ('utf-8') + def get_body_data(self): + cLength_header = self.headers.get("Content-Length") + cLength = int(cLength_header) if cLength_header is not None else 0 + body_data = self.rfile.read(cLength).decode('utf-8') return body_data - def send_put (self, path): + def send_put(self, path): if path in self.server.fileSys: - self.server.fileSys.pop (path, None) - self.send_response (204) + self.server.fileSys.pop(path, None) + self.send_response(204) else: - self.rules = dict () - self.send_response (201) - body_data = self.get_body_data () + self.rules = dict() + self.send_response(201) + body_data = self.get_body_data() self.server.fileSys[path] = body_data - self.send_header ("Location", self.path) - self.finish_headers () + self.add_header("Location", self.path) + self.finish_headers() """ This empty method is called automatically when all the rules are processed for a given request. However, send_header() should only be called @@ -173,68 +178,94 @@ class _Handler (BaseHTTPRequestHandler): finish_headers() instead of end_headers(). The finish_headers() method takes care of sending the appropriate headers before completing the response. """ - def SendHeader (self, header_obj): + def SendHeader(self, header_obj): pass - def send_cust_headers (self): - header_obj = self.get_rule_list ('SendHeader') + def send_cust_headers(self): + header_obj = self.get_rule_list('SendHeader') if header_obj: for header in header_obj.headers: - self.send_header (header, header_obj.headers[header]) + self.add_header(header, header_obj.headers[header]) - def finish_headers (self): - self.send_cust_headers () - self.end_headers () - - def Response (self, resp_obj): - self.send_response (resp_obj.response_code) - self.finish_headers () - raise ServerError ("Custom Response code sent.") - - def custom_response (self): - codes = self.get_rule_list ('Response') + def finish_headers(self): + self.send_cust_headers() + try: + for keyword, value in self._headers_dict.items(): + if isinstance(value, list): + for value_el in value: + self.send_header(keyword, value_el) + else: + self.send_header(keyword, value) + # Clear the dictionary of existing headers for the next request + self._headers_dict.clear() + except AttributeError: + pass + self.end_headers() + + def Response(self, resp_obj): + self.send_response(resp_obj.response_code) + self.finish_headers() + if resp_obj.response_code == 304: + raise NoBodyServerError("Conditional get falling to head") + raise ServerError("Custom Response code sent.") + + def custom_response(self): + codes = self.get_rule_list('Response') if codes: - self.send_response (codes.response_code) - self.finish_headers () + self.send_response(codes.response_code) + self.finish_headers() return False else: return True - def base64 (self, data): - string = b64encode (data.encode ('utf-8')) - return string.decode ('utf-8') + def add_header(self, keyword, value): + if not hasattr(self, "_headers_dict"): + self._headers_dict = dict() + self._headers_dict[keyword.lower()] = value + + def base64(self, data): + string = b64encode(data.encode('utf-8')) + return string.decode('utf-8') - def send_challenge (self, auth_type): + """ Send an authentication challenge. + This method calls self.send_header() directly instead of using the + add_header() method because sending multiple WWW-Authenticate headers + actually makes sense and we do use that feature in some tests. """ + def send_challenge(self, auth_type, auth_parm): auth_type = auth_type.lower() if auth_type == "both": - self.send_challenge ("basic") - self.send_challenge ("digest") + self.send_challenge("basic", auth_parm) + self.send_challenge("digest", auth_parm) return if auth_type == "basic": challenge_str = 'BasIc realm="Wget-Test"' elif auth_type == "digest" or auth_type == "both_inline": - self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest() - self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest() + self.nonce = md5(str(random()).encode('utf-8')).hexdigest() + self.opaque = md5(str(random()).encode('utf-8')).hexdigest() # 'DIgest' to provoke a Wget failure with turkish locales challenge_str = 'DIgest realm="Test", nonce="%s", opaque="%s"' % ( self.nonce, self.opaque) - challenge_str += ', qop="auth"' + try: + if auth_parm['qop']: + challenge_str += ', qop="%s"' % auth_parm['qop'] + except KeyError: + pass if auth_type == "both_inline": # 'BasIc' to provoke a Wget failure with turkish locales challenge_str = 'BasIc realm="Wget-Test", ' + challenge_str - self.send_header ("WWW-Authenticate", challenge_str) + self.send_header("WWW-Authenticate", challenge_str) - def authorize_basic (self, auth_header, auth_rule): + def authorize_basic(self, auth_header, auth_rule): if auth_header is None or auth_header.split(' ')[0].lower() != 'basic': return False else: self.user = auth_rule.auth_user self.passw = auth_rule.auth_pass - auth_str = "basic " + self.base64 (self.user + ":" + self.passw) + auth_str = "basic " + self.base64(self.user + ":" + self.passw) return True if auth_str.lower() == auth_header.lower() else False - def parse_auth_header (self, auth_header): + def parse_auth_header(self, auth_header): n = len("digest ") auth_header = auth_header[n:].strip() items = auth_header.split(", ") @@ -242,109 +273,113 @@ class _Handler (BaseHTTPRequestHandler): keyvals = [(k.strip(), v.strip().replace('"', '')) for k, v in keyvals] return dict(keyvals) - def KD (self, secret, data): - return self.H (secret + ":" + data) + def KD(self, secret, data): + return self.H(secret + ":" + data) - def H (self, data): - return md5 (data.encode ('utf-8')).hexdigest () + def H(self, data): + return md5(data.encode('utf-8')).hexdigest() - def A1 (self): + def A1(self): return "%s:%s:%s" % (self.user, "Test", self.passw) - def A2 (self, params): + def A2(self, params): return "%s:%s" % (self.command, params["uri"]) - def check_response (self, params): + def check_response(self, params): if "qop" in params: data_str = params['nonce'] \ + ":" + params['nc'] \ + ":" + params['cnonce'] \ + ":" + params['qop'] \ - + ":" + self.H (self.A2 (params)) + + ":" + self.H(self.A2(params)) else: - data_str = params['nonce'] + ":" + self.H (self.A2 (params)) - resp = self.KD (self.H (self.A1 ()), data_str) + data_str = params['nonce'] + ":" + self.H(self.A2(params)) + resp = self.KD(self.H(self.A1()), data_str) return True if resp == params['response'] else False - def authorize_digest (self, auth_header, auth_rule): - if auth_header is None or auth_header.split(' ')[0].lower() != 'digest': + def authorize_digest(self, auth_header, auth_rule): + if auth_header is None or \ + auth_header.split(' ')[0].lower() != 'digest': return False else: self.user = auth_rule.auth_user self.passw = auth_rule.auth_pass - params = self.parse_auth_header (auth_header) - pass_auth = True + params = self.parse_auth_header(auth_header) if self.user != params['username'] or \ - self.nonce != params['nonce'] or \ - self.opaque != params['opaque']: - pass_auth = False + self.nonce != params['nonce'] or \ + self.opaque != params['opaque']: + return False req_attribs = ['username', 'realm', 'nonce', 'uri', 'response'] for attrib in req_attribs: if attrib not in params: - pass_auth = False - if not self.check_response (params): - pass_auth = False - return pass_auth + return False + if not self.check_response(params): + return False - def authorize_both (self, auth_header, auth_rule): + def authorize_both(self, auth_header, auth_rule): return False - def authorize_both_inline (self, auth_header, auth_rule): + def authorize_both_inline(self, auth_header, auth_rule): return False - def Authentication (self, auth_rule): + def Authentication(self, auth_rule): try: - self.handle_auth (auth_rule) - except ServerError as se: - self.send_response (401, "Authorization Required") - self.send_challenge (auth_rule.auth_type) - self.finish_headers () - raise ServerError (se.__str__()) - - def handle_auth (self, auth_rule): + self.handle_auth(auth_rule) + except AuthError as se: + self.send_response(401, "Authorization Required") + self.send_challenge(auth_rule.auth_type, auth_rule.auth_parm) + self.finish_headers() + raise se + + def handle_auth(self, auth_rule): is_auth = True - auth_header = self.headers.get ("Authorization") + auth_header = self.headers.get("Authorization") required_auth = auth_rule.auth_type.lower() if required_auth == "both" or required_auth == "both_inline": - auth_type = auth_header.split(' ')[0].lower() if auth_header else required_auth + if auth_header: + auth_type = auth_header.split(' ')[0].lower() + else: + auth_type = required_auth else: auth_type = required_auth try: - assert hasattr (self, "authorize_" + auth_type) - is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule) + assert hasattr(self, "authorize_" + auth_type) + is_auth = getattr(self, "authorize_" + auth_type)(auth_header, + auth_rule) except AssertionError: - raise ServerError ("Authentication Mechanism " + auth_type + " not supported") + raise AuthError("Authentication Mechanism %s not supported" % + auth_type) except AttributeError as ae: - raise ServerError (ae.__str__()) + raise AuthError(ae.__str__()) if is_auth is False: - raise ServerError ("Unable to Authenticate") - + raise AuthError("Unable to Authenticate") - def ExpectHeader (self, header_obj): + def ExpectHeader(self, header_obj): exp_headers = header_obj.headers for header_line in exp_headers: - header_recd = self.headers.get (header_line) + header_recd = self.headers.get(header_line) if header_recd is None or header_recd != exp_headers[header_line]: - self.send_error (400, "Expected Header " + header_line + " not found") - self.finish_headers () - raise ServerError ("Header " + header_line + " not found") + self.send_error(400, "Expected Header %s not found" % + header_line) + self.finish_headers() + raise ServerError("Header " + header_line + " not found") - - def RejectHeader (self, header_obj): + def RejectHeader(self, header_obj): rej_headers = header_obj.headers for header_line in rej_headers: - header_recd = self.headers.get (header_line) - if header_recd is not None and header_recd == rej_headers[header_line]: - self.send_error (400, 'Blacklisted Header ' + header_line + ' received') - self.finish_headers () - raise ServerError ("Header " + header_line + ' received') - - def __log_request (self, method): + header_recd = self.headers.get(header_line) + if not header_recd and header_recd == rej_headers[header_line]: + self.send_error(400, 'Blacklisted Header %s received' % + header_line) + self.finish_headers() + raise ServerError("Header " + header_line + ' received') + + def __log_request(self, method): req = method + " " + self.path - self.server.request_headers.append (req) + self.server.request_headers.append(req) - def send_head (self, method): + def send_head(self, method): """ Common code for GET and HEAD Commands. This method is overriden to use the fileSys dict. @@ -358,85 +393,92 @@ class _Handler (BaseHTTPRequestHandler): else: path = self.path[1:] - self.__log_request (method) + self.__log_request(method) if path in self.server.fileSys: - self.rules = self.server.server_configs.get (path) + self.rules = self.server.server_configs.get(path) + content = self.server.fileSys.get(path) + content_length = len(content) for rule_name in self.rules: try: - assert hasattr (self, rule_name) - getattr (self, rule_name) (self.rules [rule_name]) + assert hasattr(self, rule_name) + getattr(self, rule_name)(self.rules[rule_name]) except AssertionError as ae: - msg = "Method " + rule_name + " not defined" - self.send_error (500, msg) - return (None, None) + msg = "Rule " + rule_name + " not defined" + self.send_error(500, msg) + return(None, None) + except AuthError as ae: + print(ae.__str__()) + return(None, None) + except NoBodyServerError as nbse: + print(nbse.__str__()) + return(None, None) except ServerError as se: - print (se.__str__()) - return (None, None) + print(se.__str__()) + return(content, None) - content = self.server.fileSys.get (path) - content_length = len (content) try: - self.range_begin = self.parse_range_header ( - self.headers.get ("Range"), content_length) + self.range_begin = self.parse_range_header( + self.headers.get("Range"), content_length) except ServerError as ae: # self.log_error("%s", ae.err_message) if ae.err_message == "Range Overflow": - self.send_response (416) - self.finish_headers () - return (None, None) + self.send_response(416) + self.finish_headers() + return(None, None) else: self.range_begin = None if self.range_begin is None: - self.send_response (200) + self.send_response(200) else: - self.send_response (206) - self.send_header ("Accept-Ranges", "bytes") - self.send_header ("Content-Range", - "bytes %d-%d/%d" % (self.range_begin, - content_length - 1, - content_length)) + self.send_response(206) + self.add_header("Accept-Ranges", "bytes") + self.add_header("Content-Range", + "bytes %d-%d/%d" % (self.range_begin, + content_length - 1, + content_length)) content_length -= self.range_begin - cont_type = self.guess_type (path) - self.send_header ("Content-type", cont_type) - self.send_header ("Content-Length", content_length) - self.finish_headers () - return (content, self.range_begin) + cont_type = self.guess_type(path) + self.add_header("Content-Type", cont_type) + self.add_header("Content-Length", content_length) + self.finish_headers() + return(content, self.range_begin) else: - self.send_error (404, "Not Found") - return (None, None) + self.send_error(404, "Not Found") + return(None, None) - def guess_type (self, path): - base_name = basename ("/" + path) - name, ext = splitext (base_name) + def guess_type(self, path): + base_name = basename("/" + path) + name, ext = splitext(base_name) extension_map = { - ".txt" : "text/plain", - ".css" : "text/css", - ".html" : "text/html" + ".txt": "text/plain", + ".css": "text/css", + ".html": "text/html" } return extension_map.get(ext, "text/plain") -class HTTPd (threading.Thread): + +class HTTPd(threading.Thread): server_class = StoppableHTTPServer handler = _Handler - def __init__ (self, addr=None): - threading.Thread.__init__ (self) + def __init__(self, addr=None): + threading.Thread.__init__(self) if addr is None: addr = ('localhost', 0) - self.server_inst = self.server_class (addr, self.handler) + self.server_inst = self.server_class(addr, self.handler) self.server_address = self.server_inst.socket.getsockname()[:2] - def run (self): - self.server_inst.serve_forever () + def run(self): + self.server_inst.serve_forever() - def server_conf (self, file_list, server_rules): - self.server_inst.server_conf (file_list, server_rules) + def server_conf(self, file_list, server_rules): + self.server_inst.server_conf(file_list, server_rules) -class HTTPSd (HTTPd): +class HTTPSd(HTTPd): server_class = HTTPSServer -# vim: set ts=4 sts=4 sw=4 tw=80 et : +# vim: set ts=4 sts=4 sw=4 tw=79 et : diff --git a/testenv/test/base_test.py b/testenv/test/base_test.py index 0d98078..c5b82be 100644 --- a/testenv/test/base_test.py +++ b/testenv/test/base_test.py @@ -22,27 +22,37 @@ class BaseTest: * instantiate_server_by(protocol) """ - def __init__(self, name, pre_hook, test_params, post_hook, protocols): + def __init__(self, name, pre_hook, test_params, post_hook, protocols, req_protocols): """ Define the class-wide variables (or attributes). Attributes should not be defined outside __init__. """ self.name = name - self.pre_configs = pre_hook or {} # if pre_hook == None, then - # {} (an empty dict object) is - # passed to self.pre_configs + # if pre_hook == None, then {} (an empty dict object) is passed to + # self.pre_configs + self.pre_configs = pre_hook or {} + self.test_params = test_params or {} self.post_configs = post_hook or {} self.protocols = protocols + if req_protocols is None: + self.req_protocols = map(lambda p: p.lower(), self.protocols) + else: + self.req_protocols = req_protocols + self.servers = [] self.domains = [] + self.ports = [] + + self.addr = None self.port = -1 self.wget_options = '' self.urls = [] self.tests_passed = True + self.ready = False self.init_test_env() self.ret_code = 0 @@ -62,9 +72,12 @@ class BaseTest: def get_domain_addr(self, addr): # TODO if there's a multiple number of ports, wouldn't it be # overridden to the port of the last invocation? + # Set the instance variables 'addr' and 'port' so that + # they can be queried by test cases. + self.addr = str(addr[0]) self.port = str(addr[1]) - return '%s:%s' % (addr[0], self.port) + return [self.addr, self.port] def server_setup(self): print_blue("Running Test %s" % self.name) @@ -76,7 +89,8 @@ class BaseTest: # ports and etc. # so we should record different domains respect to servers. domain = self.get_domain_addr(instance.server_address) - self.domains.append(domain) + self.domains.append(domain[0]) + self.ports.append(domain[1]) def exec_wget(self): cmd_line = self.gen_cmd_line() @@ -101,16 +115,30 @@ class BaseTest: wget_options = '--debug --no-config %s' % self.wget_options valgrind = os.getenv("VALGRIND_TESTS", "") - if valgrind in ("", "0"): - cmd_line = '%s %s ' % (wget_path, wget_options) + gdb = os.getenv("GDB_TESTS", "") + + # GDB has precedence over Valgrind + # If both VALGRIND_TESTS and GDB_TESTS are defined, + # GDB will be executed. + if gdb == "1": + cmd_line = 'gdb --args %s %s ' % (wget_path, wget_options) elif valgrind == "1": - cmd_line = 'valgrind --error-exitcode=301 --leak-check=yes --track-origins=yes %s %s ' % (wget_path, wget_options) + cmd_line = 'valgrind --error-exitcode=301 ' \ + '--leak-check=yes ' \ + '--track-origins=yes ' \ + '--suppressions=../valgrind-suppression-ssl ' \ + '%s %s ' % (wget_path, wget_options) + elif valgrind not in ("", "0"): + cmd_line = '%s %s %s ' % (os.getenv("VALGRIND_TESTS", ""), + wget_path, + wget_options) else: - cmd_line = '%s %s %s ' % (os.getenv("VALGRIND_TESTS", ""), wget_path, wget_options) + cmd_line = '%s %s ' % (wget_path, wget_options) - for protocol, urls, domain in zip(self.protocols, - self.urls, - self.domains): + for req_protocol, urls, domain, port in zip(self.req_protocols, + self.urls, + self.domains, + self.ports): # zip is function for iterating multiple lists at the same time. # e.g. for item1, item2 in zip([1, 5, 3], # ['a', 'e', 'c']): @@ -120,7 +148,8 @@ class BaseTest: # 5 e # 3 c for url in urls: - cmd_line += '%s://%s/%s ' % (protocol.lower(), domain, url) + cmd_line += '%s://%s:%s/%s ' % (req_protocol, domain, port, url) + print(cmd_line) @@ -132,12 +161,12 @@ class BaseTest: if not os.getenv("NO_CLEANUP"): shutil.rmtree(self.get_test_dir()) except: - print ("Unknown Exception while trying to remove Test Environment.") + print("Unknown Exception while trying to remove Test Environment.") - def _exit_test (self): + def _exit_test(self): self.__test_cleanup() - def begin (self): + def begin(self): return 0 if self.tests_passed else 100 def call_test(self): @@ -174,13 +203,17 @@ class BaseTest: def post_hook_call(self): self.hook_call(self.post_configs, 'Post Test Function') - def _replace_substring (self, string): - pattern = re.compile ('\{\{\w+\}\}') - match_obj = pattern.search (string) + def _replace_substring(self, string): + """ + Replace first occurrence of "{{name}}" in @string with + "getattr(self, name)". + """ + pattern = re.compile(r'\{\{\w+\}\}') + match_obj = pattern.search(string) if match_obj is not None: rep = match_obj.group() - temp = getattr (self, rep.strip ('{}')) - string = string.replace (rep, temp) + temp = getattr(self, rep.strip('{}')) + string = string.replace(rep, temp) return string def instantiate_server_by(self, protocol): diff --git a/testenv/test/http_test.py b/testenv/test/http_test.py index 230eff8..5a13d4f 100644 --- a/testenv/test/http_test.py +++ b/testenv/test/http_test.py @@ -17,17 +17,34 @@ class HTTPTest(BaseTest): pre_hook=None, test_params=None, post_hook=None, - protocols=(HTTP,)): + protocols=(HTTP,), + req_protocols=None): super(HTTPTest, self).__init__(name, pre_hook, test_params, post_hook, - protocols) + protocols, + req_protocols) + + def setup(self): + self.server_setup() + self.ready = True + + def begin(self): + if not self.ready: + # this is to maintain compatibility with scripts that + # don't call setup() + self.setup() with self: - # if any exception occurs, self.__exit__ will be immediately called - self.server_setup() - self.do_test() - print_green('Test Passed.') + # If any exception occurs, self.__exit__ will be immediately called. + # We must call the parent method in the end in order to verify + # whether the tests succeeded or not. + if self.ready: + self.do_test() + print_green("Test Passed.") + else: + self.tests_passed = False + super(HTTPTest, self).begin() def instantiate_server_by(self, protocol): server = {HTTP: HTTPd, |