W3C home > Mailing lists > Public > www-validator-cvs@w3.org > February 2009

2002/css-validator/org/w3c/css/parser/analyzer CssParser.java,1.52,1.53 CssParser.jj,1.52,1.53 CssParserTokenManager.java,1.31,1.32

From: Yves Lafon via cvs-syncmail <cvsmail@w3.org>
Date: Fri, 13 Feb 2009 21:50:18 +0000
To: www-validator-cvs@w3.org
Message-Id: <E1LY5vO-0002tR-7y@lionel-hutz.w3.org>

Update of /sources/public/2002/css-validator/org/w3c/css/parser/analyzer
In directory hutz:/tmp/cvs-serv11022/org/w3c/css/parser/analyzer

Modified Files:
	CssParser.java CssParser.jj CssParserTokenManager.java 
Log Message:
multi-byte handling, more comprehensive use of charsets, while trying to preserve BOM support

Index: CssParserTokenManager.java
===================================================================
RCS file: /sources/public/2002/css-validator/org/w3c/css/parser/analyzer/CssParserTokenManager.java,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- CssParserTokenManager.java	12 Feb 2009 21:26:35 -0000	1.31
+++ CssParserTokenManager.java	13 Feb 2009 21:50:15 -0000	1.32
@@ -1,6 +1,8 @@
 /* Generated By:JavaCC: Do not edit this line. CssParserTokenManager.java */
 package org.w3c.css.parser.analyzer;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
 import java.util.Vector;
 import java.util.Enumeration;
 import java.net.URL;
@@ -782,6 +784,9 @@
    jjCheckNAdd(jjnextStates[start + 1]);
 }
 static final long[] jjbitVec0 = {
+   0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+static final long[] jjbitVec2 = {
    0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
 };
 private final int jjMoveNfa_0(int startState, int curPos)
@@ -5564,6 +5569,9 @@
       }
       else
       {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
          MatchLoop: do
@@ -5571,189 +5579,207 @@
             switch(jjstateSet[--i])
             {
                case 7:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 34)
                      kind = 34;
                   jjCheckNAddStates(1672, 1676);
                   break;
                case 1:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1687, 1688);
                   break;
                case 3:
                case 4:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(4, 5);
                   break;
                case 22:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1689, 1690);
                   break;
                case 24:
                case 25:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(25, 26);
                   break;
                case 40:
                case 41:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 81)
+                     kind = 81;
+                  jjCheckNAddTwoStates(41, 42);
+                  break;
                case 43:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 81)
                      kind = 81;
                   jjCheckNAddTwoStates(41, 42);
                   break;
                case 64:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1691, 1692);
                   break;
                case 66:
                case 67:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(67, 68);
                   break;
                case 72:
                case 73:
-               case 171:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_1(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(242, 247);
                   break;
                case 76:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1693, 1694);
                   break;
                case 78:
                case 79:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(79, 80);
                   break;
                case 84:
                case 85:
-               case 101:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_1(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(260, 262);
                   break;
                case 88:
                case 89:
-               case 120:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_1(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(268, 273);
                   break;
                case 92:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1695, 1696);
                   break;
                case 94:
                case 95:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(95, 96);
                   break;
+               case 101:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjCheckNAddStates(260, 262);
+                  break;
+               case 120:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjCheckNAddStates(268, 273);
+                  break;
                case 139:
                case 145:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(483, 485);
                   break;
                case 155:
                case 161:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(532, 534);
                   break;
+               case 171:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjCheckNAddStates(242, 247);
+                  break;
                case 192:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1713, 1714);
                   break;
                case 194:
                case 195:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(195, 196);
                   break;
                case 200:
                case 201:
-               case 213:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_1(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(706, 709);
                   break;
                case 204:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1715, 1716);
                   break;
                case 206:
                case 207:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(207, 208);
                   break;
+               case 213:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjCheckNAddStates(706, 709);
+                  break;
                case 234:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1717, 1718);
                   break;
                case 236:
                case 237:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(237, 238);
                   break;
                case 243:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1719, 1720);
                   break;
                case 245:
                case 246:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(246, 247);
                   break;
                case 254:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1721, 1722);
                   break;
                case 256:
                case 257:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(257, 258);
                   break;
                case 263:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1723, 1724);
                   break;
                case 265:
                case 266:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(266, 267);
                   break;
                case 272:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1725, 1726);
                   break;
                case 274:
                case 275:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(275, 276);
                   break;
                case 281:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1727, 1728);
                   break;
                case 283:
                case 284:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(284, 285);
                   break;
                case 290:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1729, 1730);
                   break;
                case 292:
                case 293:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(293, 294);
                   break;
                case 301:
                case 307:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(867, 869);
                   break;
                case 316:
                case 321:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 33)
                      kind = 33;
@@ -5761,12 +5787,12 @@
                   break;
                case 331:
                case 337:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(946, 948);
                   break;
                case 346:
                case 351:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 33)
                      kind = 33;
@@ -5774,8 +5800,14 @@
                   break;
                case 361:
                case 362:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 34)
+                     kind = 34;
+                  jjCheckNAddTwoStates(362, 363);
+                  break;
                case 364:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 34)
                      kind = 34;
@@ -5783,22 +5815,37 @@
                   break;
                case 374:
                case 375:
+                  if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     jjCheckNAddStates(1060, 1062);
+                  break;
                case 378:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(1060, 1062);
                   break;
                case 407:
                case 408:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 35)
+                     kind = 35;
+                  jjCheckNAddTwoStates(408, 409);
+                  break;
                case 410:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 35)
                      kind = 35;
                   jjCheckNAddTwoStates(408, 409);
                   break;
                case 428:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 36)
+                     kind = 36;
+                  jjCheckNAddTwoStates(428, 429);
+                  break;
                case 430:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 36)
                      kind = 36;
@@ -5806,41 +5853,50 @@
                   break;
                case 443:
                case 459:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(1291, 1293);
                   break;
                case 447:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1749, 1750);
                   break;
                case 449:
                case 450:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(450, 451);
                   break;
                case 469:
                case 475:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(1344, 1346);
                   break;
                case 484:
+                  if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     jjCheckNAddStates(1388, 1391);
+                  break;
                case 486:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(1388, 1391);
                   break;
                case 498:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjAddStates(1761, 1762);
                   break;
                case 500:
                case 501:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddTwoStates(501, 502);
                   break;
                case 623:
                case 624:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 62)
+                     kind = 62;
+                  jjCheckNAddTwoStates(624, 625);
+                  break;
                case 626:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 62)
                      kind = 62;
@@ -5848,8 +5904,14 @@
                   break;
                case 648:
                case 649:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 92)
+                     kind = 92;
+                  jjCheckNAddTwoStates(649, 650);
+                  break;
                case 651:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 92)
                      kind = 92;
@@ -6004,6 +6066,28 @@
    350, 351, 352, 430, 431, 447, 448, 456, 458, 459, 460, 472, 474, 475, 476, 486, 
    487, 498, 499, 
 };
+private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
+{
+   switch(hiByte)
+   {
+      case 0:
+         return ((jjbitVec2[i2] & l2) != 0L);
+      default : 
+         if ((jjbitVec0[i1] & l1) != 0L)
+            return true;
+         return false;
+   }
+}
+private static final boolean jjCanMove_1(int hiByte, int i1, int i2, long l1, long l2)
+{
+   switch(hiByte)
+   {
+      case 0:
+         return ((jjbitVec2[i2] & l2) != 0L);
+      default : 
+         return false;
+   }
+}
 public static final String[] jjstrLiteralImages = {
 "", null, null, null, null, null, null, null, null, null, null, null, null, 
 null, null, null, null, null, null, null, null, null, "\74\41\55\55", "\55\55\76", 

Index: CssParser.java
===================================================================
RCS file: /sources/public/2002/css-validator/org/w3c/css/parser/analyzer/CssParser.java,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- CssParser.java	13 Feb 2009 14:03:36 -0000	1.52
+++ CssParser.java	13 Feb 2009 21:50:15 -0000	1.53
@@ -2,6 +2,8 @@
 package org.w3c.css.parser.analyzer;
 
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
 import java.util.Vector;
 import java.util.Enumeration;
 import java.net.URL;
@@ -160,8 +162,17 @@
      * @param stream the stream data to parse.
      * @param ac  the new ac to use for parsing.
      */
-    public void ReInitWithAc(InputStream stream, ApplContext ac) {
-        ReInit(new CommentSkipperInputStream(stream));
+    public void ReInitWithAc(InputStream stream, ApplContext ac, String charset) {
+        InputStream is = new  CommentSkipperInputStream(stream);
+        if (charset == null) {
+            charset = "iso-8859-1";
+        }
+        InputStreamReader isr = null;
+        try {
+            isr = new InputStreamReader(is, charset);
+        } catch (UnsupportedEncodingException uex) {}; // reinit, it can not happen...
+        // ...in theory ;)
+        ReInit(isr);
         markRule = false;
         setApplContext(ac);
     }
@@ -3605,21 +3616,21 @@
     finally { jj_save(0, xla); }
   }
 
-  final private boolean jj_3R_83() {
+  final private boolean jj_3_1() {
     Token xsp;
     xsp = jj_scanpos;
-    if (jj_scan_token(34)) {
-    jj_scanpos = xsp;
-    if (jj_scan_token(47)) return true;
-    }
+    if (jj_3R_83()) jj_scanpos = xsp;
+    if (jj_scan_token(98)) return true;
     return false;
   }
 
-  final private boolean jj_3_1() {
+  final private boolean jj_3R_83() {
     Token xsp;
     xsp = jj_scanpos;
-    if (jj_3R_83()) jj_scanpos = xsp;
-    if (jj_scan_token(98)) return true;
+    if (jj_scan_token(34)) {
+    jj_scanpos = xsp;
+    if (jj_scan_token(47)) return true;
+    }
     return false;
   }
 

Index: CssParser.jj
===================================================================
RCS file: /sources/public/2002/css-validator/org/w3c/css/parser/analyzer/CssParser.jj,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- CssParser.jj	13 Feb 2009 14:03:36 -0000	1.52
+++ CssParser.jj	13 Feb 2009 21:50:15 -0000	1.53
@@ -10,7 +10,8 @@
 options {
     IGNORE_CASE  = true;
     STATIC = false;
-    /*
+    UNICODE_INPUT = true;
+    /*    
           DEBUG_TOKEN_MANAGER = true; 
 	  DEBUG_PARSER = true;      
     */
@@ -21,6 +22,8 @@
 package org.w3c.css.parser.analyzer;
 
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
 import java.util.Vector;
 import java.util.Enumeration;
 import java.net.URL;
@@ -179,8 +182,17 @@
      * @param stream the stream data to parse.
      * @param ac  the new ac to use for parsing.
      */
-    public void ReInitWithAc(InputStream stream, ApplContext ac) {
-    	ReInit(new CommentSkipperInputStream(stream));
+    public void ReInitWithAc(InputStream stream, ApplContext ac, String charset) {
+	InputStream is = new  CommentSkipperInputStream(stream);
+	if (charset == null) {
+	    charset = "iso-8859-1";
+	}
+	InputStreamReader isr = null;
+	try {
+	    isr = new InputStreamReader(is, charset);
+	} catch (UnsupportedEncodingException uex) {}; // reinit, it can not happen...
+	// ...in theory ;)
+    	ReInit(isr);
 	markRule = false;
 	setApplContext(ac);
     }
Received on Friday, 13 February 2009 21:50:32 GMT

This archive was generated by hypermail 2.2.0+W3C-0.50 : Thursday, 26 April 2012 12:55:08 GMT