W3C home > Mailing lists > Public > www-validator-cvs@w3.org > February 2008

validator/misc/testsuite harness.py,NONE,1.1

From: Olivier Thereaux via cvs-syncmail <cvsmail@w3.org>
Date: Wed, 27 Feb 2008 20:30:36 +0000
To: www-validator-cvs@w3.org
Message-Id: <E1JUSvE-0007VN-9e@lionel-hutz.w3.org>

Update of /sources/public/validator/misc/testsuite
In directory hutz:/tmp/cvs-serv28808

Added Files:
Log Message:
test suite script for the W3C markup validator
(and in longer term, to compare validators) 
This work is an evolution from earlier work on the Link Test Suite ( http://dev.w3.org/cvsweb/2008/link-testsuite/ )
with a number of small differences:
* instead of a nested directory structure, the test suite is described in a single XML file
* comparison of results and expected results are more flexible, e.g it is possible to specify
  "results should have some warning(s)" rather than "results should have 12 warnings exactly"
* easy selection of the tested validator instance from the command line
* possibility of running only a subset of the test suite with a commandline option

--- NEW FILE: harness.py ---
#!/usr/bin/env python
# encoding: utf-8
Run or Generate test suite for link checkers

Created by olivier Thereaux on 2008-01-23.
Copyright (c) 2008 W3C. Licensed under the W3C Software License

import os
import sys
import glob
import getopt
import unittest
import re

basedir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(basedir, "lib"))
from ValidatorsAPIs import W3CValidatorHTTP, W3CValidatorHTTP_test
from TestCase import ValidatorTestCase, ValidatorTestSuite, ValidatorTestCase_UT, ValidatorTestSuite_UT
#from Documentation import Documentation
help_message = '''

Run or Generate test suite for markup validator(s)

Usage: harness.py [options] [run|sanity|doc]
        -h, --help: this manual you are reading
        -v, --verbose: verbose output
        -q, --quiet: suppress all output except errors
        --validator_uri: use a specific validator instance
          e.g http://validator.w3.org/check
        --id=collection_id: run a single collection from the test suite
          In run mode only, this filters the test suite and uses only
          a collection identified with id="..." 
          with its children tests and collections
        run: run the test suite 
        sanity: check that this code is still working 
            useful after using test cases or modifying code
        @@ TBD @@ doc: generate an HTML index of the test cases

class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg

class TestRun(unittest.TestCase):
    def test_1_1_readTestCase(self):
        """Opening and parsing a Test Case"""
        basedir = getBaseDir()
        test_file = os.path.join(basedir, 'sample', 'collection_1.xml')

def main(argv=None):
    collection_id = None
    checker = None
    check_URI = None
    if argv is None:
        argv = sys.argv
            opts, args = getopt.getopt(argv[1:], "ho:vq", ["help", "verbose", "quiet", "id=", "validator=", "validator_uri="])
            for (opt, value) in opts:
                if opt == "h" or opt == "--help":
                    raise Usage(msg)
        except getopt.error, msg:
            raise Usage(msg)
        # option processing
        for option, value in opts:
            if option == "-v" or opt == "--verbose":
                verbose = 2
            if option == "-q" or opt == "--quiet":
                verbose = 0
            if option in ("-h", "--help"):
                raise Usage(help_message)
            if option in ("-o", "--output"):
                output = value
            if option == "--id":
                collection_id = value
            if option == "--validator":
                pass # TODO add selection of different validators
            if option == "--validator_uri":
                check_URI = value

        if len(args) == 0:
            raise Usage(help_message)
    except Usage, err:
        print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg)
        print >> sys.stderr, "\t for help use --help"
        return 2

    if args[0] == "run":
        if checker == None:
            checker = W3CValidatorHTTP(check_URI=check_URI)
            pass # TODO add selection of different validators
        testsuite = buildTestSuite(collection_id=collection_id, checker=checker)
        runTestSuite(testsuite, verbose)
    elif args[0] == "sanity":
        suite1 = unittest.TestLoader().loadTestsFromTestCase(ValidatorTestCase_UT)
        suite2 = unittest.TestLoader().loadTestsFromTestCase(ValidatorTestSuite_UT)
        suite3 = unittest.TestLoader().loadTestsFromTestCase(TestRun)
        suite4 = unittest.TestLoader().loadTestsFromTestCase(W3CValidatorHTTP_test)
        suite = unittest.TestSuite([suite1, suite2, suite3, suite4])
    elif args[0] == "doc":

def getBaseDir():
    basedir = os.path.dirname(os.path.abspath(__file__))
    return basedir
def buildTestSuite(ts_file=None, collection_id=None, checker=None):
    """read test suite XML file and build it"""
    test_suite = ValidatorTestSuite(checker=checker)
    if ts_file == None:
        basedir = getBaseDir()
        ts_file = os.path.join(basedir, 'catalog.xml')
    parsed_ts = test_suite.readTestSuiteFile(ts_file)
    filtered_ts = parsed_ts
    if collection_id != None:
        for collection in parsed_ts.findall(".//collection"):
            if collection.attrib.has_key("id"):
                if collection.attrib["id"] == collection_id: # this is the one we want, we keep this and its children
                    filtered_ts = collection
    return test_suite

def runTestSuite(testsuite, verbose):
    """Describe each test collection in the test suite and run them"""
    for collection in testsuite.collections:
        print "*** Running Collection #%s: %s" % (collection.collection_id, collection.title)
        if collection.countTestCases() > 0:

# def generateIndex():
#     index = Documentation('index')
#     for testcollection_file in (glob.glob(os.path.join(basedir, 'testcases', '**', '*.collection'))):
#         colldir = os.path.dirname(os.path.abspath(testcollection_file))
#         colldir = os.path.split(colldir)[-1]        
#         testcollection = readCollectionMeta(testcollection_file)
#         index.addCollection(testcollection)
#     print index.generate(template_path=os.path.join(basedir, "templates")).encode('utf-8')

if __name__ == "__main__":
Received on Wednesday, 27 February 2008 20:30:52 GMT

This archive was generated by hypermail 2.2.0+W3C-0.50 : Thursday, 26 April 2012 12:55:00 GMT