Controlling test coverage

I have updated the changes.xml file at

https://github.com/qt4cg/qt4tests/blob/master/changes.xml

to contain a more complete list, reviewed against the GitHub list of applied PRs.

Many new tests have a changes40 attribute (on either the test case or the test set element) that links to the ID of an entry in the changes file. I would suggest we try to move towards making these links finer grained by including the actual PR number.

Test case Catalog014 checks that for every test marked as dependent on 4.0, there is a covers40 attribute. (At present of course, this is not always the case).

Test case Catalog015 checks how many tests there are for each category in the changes.xml file. Here are the current results:

   <tests change="NameTestUnion" count="37" description="Name test unions"/>
   <tests change="annotation-namespaces"
          count="0"
          description="Annotation namespaces"/>
   <tests change="arity-coercion"
          count="11"
          description="Reducing arity in coercion rules"/>
   <tests change="array-build"
          count="4"
          description="New function array:build"/>
   <tests change="array-foot" count="9" description="New function array:foot"/>
   <tests change="array-get"
          count="7"
          description="Extra argument to array:get"/>
   <tests change="array-index-of"
          count="10"
          description="New function array:index-where"/>
   <tests change="array-index-where"
          count="20"
          description="New function array:index-where"/>
   <tests change="array-members"
          count="6"
          description="New function array:members"/>
   <tests change="array-of-members"
          count="6"
          description="New function array:of-members"/>
   <tests change="array-slice"
          count="71"
          description="New function array:slice"/>
   <tests change="array-sort"
          count="0"
          description="Align array:sort with fn:sort"/>
   <tests change="array-split"
          count="11"
          description="New function array:split"/>
   <tests change="array-trunk"
          count="6"
          description="New function array:trunk"/>
   <tests change="array-values"
          count="0"
          description="New function array:values"/>
   <tests change="binary-literals"
          count="8"
          description="Binary notation in numeric literals"/>
   <tests change="binary-promotion"
          count="19"
          description="hexBinary and base64Binary are now interoperable"/>
   <tests change="coercion-in-variables"
          count="30"
          description="Coercion rules are applied to variables"/>
   <tests change="constructors"
          count="0"
          description="Changes to constructor functions"/>
   <tests change="context-value"
          count="0"
          description="context item becomes context value"/>
   <tests change="control-chars"
          count="0"
          description="Clarify serialization rules for control chars"/>
   <tests change="deep-lookup" count="0" description="New ?? operator"/>
   <tests change="default-namespace-fixed"
          count="5"
          description="Default namespace not overridable in direct element constructor"/>
   <tests change="defaulted-params"
          count="36"
          description="Default params in function declarations"/>
   <tests change="downcasting"
          count="10"
          description="Downcasting in coercion rules"/>
   <tests change="escape-solidus"
          count="3"
          description="New serialization parameter escape-solidus"/>
   <tests change="extended-annotations"
          count="20"
          description="Boolean and negative annotation values"/>
   <tests change="filter-expression"
          count="0"
          description="Predicate can be a sequence of integers"/>
   <tests change="flwor-while" count="0" description="New FLWOR while clause"/>
   <tests change="fn-all-different"
          count="34"
          description="New function fn:all-different"/>
   <tests change="fn-all-equal"
          count="33"
          description="New function fn:all-equal"/>
   <tests change="fn-atomic-equal"
          count="30"
          description="New function fn:atomic-equal"/>
   <tests change="fn-build-uri"
          count="45"
          description="New function fn:build-uri"/>
   <tests change="fn-chain" count="0" description="New function fn:char"/>
   <tests change="fn-char" count="32" description="New function fn:char"/>
   <tests change="fn-characters"
          count="12"
          description="New function fn:characters"/>
   <tests change="fn-codepoints-to-string"
          count="2"
          description="Function fn:codepoints-to-string becomes variadic (TODO???)"/>
   <tests change="fn-compare"
          count="0"
          description="fn:compare to handle all data types"/>
   <tests change="fn-concat"
          count="14"
          description="Function fn:concat allows sequence-valued arguments (TODO???)"/>
   <tests change="fn-contains-subsequence"
          count="32"
          description="New function fn:contains-subsequence"/>
   <tests change="fn-decode-from-uri"
          count="26"
          description="New function fn:decode-from-uri"/>
   <tests change="fn-deep-equal"
          count="139"
          description="New options argument to function fn:deep-equal"/>
   <tests change="fn-do-until"
          count="23"
          description="New function fn:do-until"/>
   <tests change="fn-doc" count="0" description="fn:doc error handling"/>
   <tests change="fn-document-uri"
          count="0"
          description="fn:doc error handling"/>
   <tests change="fn-duplicate-values"
          count="127"
          description="New function fn:duplicate-values"/>
   <tests change="fn-ends-with-subsequence"
          count="33"
          description="New function fn:ends-with-subsequence"/>
   <tests change="fn-every" count="31" description="New function fn:every"/>
   <tests change="fn-expanded-QName"
          count="9"
          description="PR 207: New function fn:expanded-QName"/>
   <tests change="fn-foot" count="8" description="New function fn:foot"/>
   <tests change="fn-format-integer"
          count="17"
          description="Changes to fn:format-integer, for example hex and binary output"/>
   <tests change="fn-format-number"
          count="2"
          description="Changes to fn:format-number, decimal format supplied as QName"/>
   <tests change="fn-function-annotations"
          count="18"
          description="New function fn:function-annotations"/>
   <tests change="fn-function-name"
          count="0"
          description="fn:function-name applied to partial functions"/>
   <tests change="fn-hash" count="0" description="New function fn:hash"/>
   <tests change="fn-highest"
          count="32"
          description="New function fn:highest"/>
   <tests change="fn-identity"
          count="5"
          description="New function fn:identity"/>
   <tests change="fn-in-scope-namespaces"
          count="9"
          description="New function fn:in-scope-namespaces"/>
   <tests change="fn-index-where"
          count="19"
          description="New function fn:index-where"/>
   <tests change="fn-intersperse"
          count="13"
          description="New function fn:intersperse"/>
   <tests change="fn-invisible-xml"
          count="0"
          description="New function fn:intersperse"/>
   <tests change="fn-is-NaN" count="16" description="New function fn:is-NaN"/>
   <tests change="fn-items-at"
          count="24"
          description="New function fn:items-at"/>
   <tests change="fn-json-doc"
          count="1"
          description="Extra options to fn:json-doc"/>
   <tests change="fn-json-to-xml"
          count="0"
          description="Extra options to fn:json-doc"/>
   <tests change="fn-label" count="0" description="Labeled values"/>
   <tests change="fn-load-xquery-module"
          count="0"
          description="Changes to fn:load-xquery-module"/>
   <tests change="fn-lowest" count="32" description="New function fn:lowest"/>
   <tests change="fn-message" count="1" description="New function fn:message"/>
   <tests change="fn-op" count="34" description="New function fn:op"/>
   <tests change="fn-parse-QName"
          count="19"
          description="New function fn:parse-QName"/>
   <tests change="fn-parse-csv"
          count="0"
          description="New function fn:parse-csv"/>
   <tests change="fn-parse-html"
          count="1379"
          description="New function fn:parse-html"/>
   <tests change="fn-parse-integer"
          count="31"
          description="New function fn:parse-integer"/>
   <tests change="fn-parse-json"
          count="0"
          description="New options for function fn:parse-json"/>
   <tests change="fn-parse-uri"
          count="49"
          description="New function fn:parse-uri"/>
   <tests change="fn-partition"
          count="23"
          description="New function fn:partition"/>
   <tests change="fn-parts"
          count="94"
          description="New function fn:parts (TODO: not currently in spec)"/>
   <tests change="fn-pin" count="0" description="Pinned values"/>
   <tests change="fn-remove"
          count="6"
          description="Changes to fn:remove (remove multiple items)"/>
   <tests change="fn-replace"
          count="0"
          description="Changes to fn:replace (new substitute argument)"/>
   <tests change="fn-replicate"
          count="21"
          description="New function fn:replicate"/>
   <tests change="fn-resolve-uri"
          count="2"
          description="Changes to fn:resolve-uri (empty sequence in arg 2; fragment id)"/>
   <tests change="fn-scan-left"
          count="0"
          description="New function fn:scan-left"/>
   <tests change="fn-scan-right"
          count="0"
          description="New function fn:scan-right"/>
   <tests change="fn-seconds"
          count="0"
          description="New function fn:scan-right"/>
   <tests change="fn-slice" count="73" description="New function fn:slice"/>
   <tests change="fn-some" count="31" description="New function fn:some"/>
   <tests change="fn-sort" count="0" description="fn:sort descending"/>
   <tests change="fn-sort-with" count="14" description="fn:sort-with"/>
   <tests change="fn-starts-with-subsequence"
          count="33"
          description="New function fn:starts-with-subsequence"/>
   <tests change="fn-subsequence-after"
          count="0"
          description="New function fn:subsequence-after"/>
   <tests change="fn-subsequence-ending-where"
          count="0"
          description="New function fn:subsequence-ending-where"/>
   <tests change="fn-subsequence-starting-where"
          count="0"
          description="New function fn:subsequence-starting-where"/>
   <tests change="fn-subsequence-where"
          count="92"
          description="New function fn:subsequence-where"/>
   <tests change="fn-take-while"
          count="16"
          description="New function fn:take-while"/>
   <tests change="fn-transform"
          count="2"
          description="Changes to function fn:transform Changes to function fn:transform"/>
   <tests change="fn-transitive-closure"
          count="10"
          description="Changes to function fn:transform"/>
   <tests change="fn-trunk" count="8" description="New function fn:trunk"/>
   <tests change="fn-unparsed-text"
          count="0"
          description="Changes to fn:unparsed-text: line endings"/>
   <tests change="fn-void" count="5" description="New function fn:void"/>
   <tests change="fn-while-do"
          count="28"
          description="New function fn:while-do"/>
   <tests change="fn-xdm-to-json"
          count="152"
          description="Extra options to fn:xdm-to-json"/>
   <tests change="fn-xml-to-json"
          count="4"
          description="Changes to function fn:xml-to-json"/>
   <tests change="hex-literals"
          count="23"
          description="Hex notation in numeric literals"/>
   <tests change="html-ascii-case-insensitive-ordered"
          count="3"
          description="HTML collation now defines an ordering"/>
   <tests change="if-curlies"
          count="42"
          description="PR 284: Curly braces in if expression"/>
   <tests change="keywords"
          count="42"
          description="Keywords in static function calls"/>
   <tests change="map-build" count="37" description="New function map:build"/>
   <tests change="map-entries"
          count="17"
          description="New function map:entries"/>
   <tests change="map-filter"
          count="14"
          description="Changes to function map:filter"/>
   <tests change="map-get" count="6" description="Extra argument to map:get"/>
   <tests change="map-keys"
          count="0"
          description="Extra argument to map:keys"/>
   <tests change="map-of-pairs"
          count="18"
          description="New function map:of-pairs"/>
   <tests change="map-pair" count="9" description="New function map:pair"/>
   <tests change="map-pairs" count="17" description="New function map:pair"/>
   <tests change="map-values"
          count="15"
          description="New function map:values"/>
   <tests change="meta-elements"
          count="0"
          description="Revise serialization rules for meta elements"/>
   <tests change="methods-in-maps" count="0" description="New =?&gt; operator"/>
   <tests change="misc-callbacks"
          count="0"
          description="Add position argument to callbacks"/>
   <tests change="misc-collation-optional"
          count="0"
          description="$collation argument may be empty"/>
   <tests change="misc-optional-args"
          count="0"
          description="Optional arguments become emptiable"/>
   <tests change="multiple-for"
          count="1"
          description="Multiple for clauses in an expression"/>
   <tests change="multiple-let"
          count="2"
          description="Multiple let clauses in an expression"/>
   <tests change="numeric-promotion"
          count="0"
          description="Comparing and converting decimals and doubles"/>
   <tests change="numeric-underscores"
          count="11"
          description="Underscores in numeric literals"/>
   <tests change="operator-symbols"
          count="29"
          description="Non-ASCII characters in operator tokens"/>
   <tests change="otherwise" count="8" description="Otherwise operator"/>
   <tests change="plausibility"
          count="0"
          description="PR 603: Implausible expressions"/>
   <tests change="predeclare-prefixes"
          count="0"
          description="Predeclare well-known namespace prefixes"/>
   <tests change="prod-EnumerationType"
          count="14"
          description="Enumeration types"/>
   <tests change="prod-ForClause.map"
          count="56"
          description="for expression with keywords &#34;key&#34; and &#34;value&#34;"/>
   <tests change="prod-ForClause.member"
          count="42"
          description="For-member in for expressions"/>
   <tests change="prod-InlineFunctionExpr.focus"
          count="29"
          description="abbreviated inline functions - focus notation"/>
   <tests change="prod-ItemTypeDecl"
          count="15"
          description="Named item type declarations"/>
   <tests change="prod-LambdaExpr"
          count="9"
          description="abbreviated inline functions - lambda notation"/>
   <tests change="prod-LocalUnionType"
          count="9"
          description="local union types"/>
   <tests change="prod-MappingArrow"
          count="49"
          description="Introduces the =!&gt; operator"/>
   <tests change="prod-StringTemplate"
          count="53"
          description="String templates"/>
   <tests change="prod-ThickArrow"
          count="0"
          description="Inline functions after arrow operator"/>
   <tests change="prod-ThinArrow"
          count="0"
          description="Thin arrow expressions"/>
   <tests change="prod-UnionNodeTest"
          count="12"
          description="Union node test"/>
   <tests change="record-constructors"
          count="0"
          description="Record constructors"/>
   <tests change="record-test" count="14" description="Record tests"/>
   <tests change="regex-comments"
          count="0"
          description="Comments in regular expressions"/>
   <tests change="schema-compatibility"
          count="0"
          description="Compatibility of schemas across modules"/>
   <tests change="switch"
          count="8"
          description="Changes to switch expressions"/>
   <tests change="try-catch-variables"
          count="5"
          description="New variable for error information"/>
   <tests change="typeswitch"
          count="0"
          description="Allow braces in typeswitch expressions"/>
   <tests change="typeswitch-braces"
          count="1"
          description="Curly braces in typeswitch expressions"/>
   <tests change="uri-promotion"
          count="1"
          description="anyURI and string are now interoperable"/>
   <tests change="window"
          count="25"
          description="Changes to FLWOR window clause"/>
   <tests change="xs-string"
          count="0"
          description="Non-XML characters in strings"/>

Of course there is immense scope for cleaning up this data but it does give us a mechanism for controlling test coverage if we use it effectively,

Michael Kay
 

Received on Tuesday, 12 March 2024 19:17:03 UTC