- From: Yves Lafon via cvs-syncmail <cvsmail@w3.org>
- Date: Fri, 13 Feb 2009 21:50:18 +0000
- To: www-validator-cvs@w3.org
Update of /sources/public/2002/css-validator/org/w3c/css/parser/analyzer
In directory hutz:/tmp/cvs-serv11022/org/w3c/css/parser/analyzer
Modified Files:
CssParser.java CssParser.jj CssParserTokenManager.java
Log Message:
multi-byte handling, more comprehensive use of charsets, while trying to preserve BOM support
Index: CssParserTokenManager.java
===================================================================
RCS file: /sources/public/2002/css-validator/org/w3c/css/parser/analyzer/CssParserTokenManager.java,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- CssParserTokenManager.java 12 Feb 2009 21:26:35 -0000 1.31
+++ CssParserTokenManager.java 13 Feb 2009 21:50:15 -0000 1.32
@@ -1,6 +1,8 @@
/* Generated By:JavaCC: Do not edit this line. CssParserTokenManager.java */
package org.w3c.css.parser.analyzer;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
import java.util.Vector;
import java.util.Enumeration;
import java.net.URL;
@@ -782,6 +784,9 @@
jjCheckNAdd(jjnextStates[start + 1]);
}
static final long[] jjbitVec0 = {
+ 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+static final long[] jjbitVec2 = {
0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
};
private final int jjMoveNfa_0(int startState, int curPos)
@@ -5564,6 +5569,9 @@
}
else
{
+ int hiByte = (int)(curChar >> 8);
+ int i1 = hiByte >> 6;
+ long l1 = 1L << (hiByte & 077);
int i2 = (curChar & 0xff) >> 6;
long l2 = 1L << (curChar & 077);
MatchLoop: do
@@ -5571,189 +5579,207 @@
switch(jjstateSet[--i])
{
case 7:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
break;
if (kind > 34)
kind = 34;
jjCheckNAddStates(1672, 1676);
break;
case 1:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1687, 1688);
break;
case 3:
case 4:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(4, 5);
break;
case 22:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1689, 1690);
break;
case 24:
case 25:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(25, 26);
break;
case 40:
case 41:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 81)
+ kind = 81;
+ jjCheckNAddTwoStates(41, 42);
+ break;
case 43:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 81)
kind = 81;
jjCheckNAddTwoStates(41, 42);
break;
case 64:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1691, 1692);
break;
case 66:
case 67:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(67, 68);
break;
case 72:
case 73:
- case 171:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(242, 247);
break;
case 76:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1693, 1694);
break;
case 78:
case 79:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(79, 80);
break;
case 84:
case 85:
- case 101:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(260, 262);
break;
case 88:
case 89:
- case 120:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(268, 273);
break;
case 92:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1695, 1696);
break;
case 94:
case 95:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(95, 96);
break;
+ case 101:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(260, 262);
+ break;
+ case 120:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(268, 273);
+ break;
case 139:
case 145:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(483, 485);
break;
case 155:
case 161:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(532, 534);
break;
+ case 171:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(242, 247);
+ break;
case 192:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1713, 1714);
break;
case 194:
case 195:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(195, 196);
break;
case 200:
case 201:
- case 213:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(706, 709);
break;
case 204:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1715, 1716);
break;
case 206:
case 207:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(207, 208);
break;
+ case 213:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(706, 709);
+ break;
case 234:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1717, 1718);
break;
case 236:
case 237:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(237, 238);
break;
case 243:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1719, 1720);
break;
case 245:
case 246:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(246, 247);
break;
case 254:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1721, 1722);
break;
case 256:
case 257:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(257, 258);
break;
case 263:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1723, 1724);
break;
case 265:
case 266:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(266, 267);
break;
case 272:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1725, 1726);
break;
case 274:
case 275:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(275, 276);
break;
case 281:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1727, 1728);
break;
case 283:
case 284:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(284, 285);
break;
case 290:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1729, 1730);
break;
case 292:
case 293:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(293, 294);
break;
case 301:
case 307:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(867, 869);
break;
case 316:
case 321:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 33)
kind = 33;
@@ -5761,12 +5787,12 @@
break;
case 331:
case 337:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(946, 948);
break;
case 346:
case 351:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 33)
kind = 33;
@@ -5774,8 +5800,14 @@
break;
case 361:
case 362:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 34)
+ kind = 34;
+ jjCheckNAddTwoStates(362, 363);
+ break;
case 364:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 34)
kind = 34;
@@ -5783,22 +5815,37 @@
break;
case 374:
case 375:
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(1060, 1062);
+ break;
case 378:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(1060, 1062);
break;
case 407:
case 408:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 35)
+ kind = 35;
+ jjCheckNAddTwoStates(408, 409);
+ break;
case 410:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 35)
kind = 35;
jjCheckNAddTwoStates(408, 409);
break;
case 428:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 36)
+ kind = 36;
+ jjCheckNAddTwoStates(428, 429);
+ break;
case 430:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 36)
kind = 36;
@@ -5806,41 +5853,50 @@
break;
case 443:
case 459:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(1291, 1293);
break;
case 447:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1749, 1750);
break;
case 449:
case 450:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(450, 451);
break;
case 469:
case 475:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(1344, 1346);
break;
case 484:
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(1388, 1391);
+ break;
case 486:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(1388, 1391);
break;
case 498:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(1761, 1762);
break;
case 500:
case 501:
- if ((jjbitVec0[i2] & l2) != 0L)
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjCheckNAddTwoStates(501, 502);
break;
case 623:
case 624:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 62)
+ kind = 62;
+ jjCheckNAddTwoStates(624, 625);
+ break;
case 626:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 62)
kind = 62;
@@ -5848,8 +5904,14 @@
break;
case 648:
case 649:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 92)
+ kind = 92;
+ jjCheckNAddTwoStates(649, 650);
+ break;
case 651:
- if ((jjbitVec0[i2] & l2) == 0L)
+ if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
break;
if (kind > 92)
kind = 92;
@@ -6004,6 +6066,28 @@
350, 351, 352, 430, 431, 447, 448, 456, 458, 459, 460, 472, 474, 475, 476, 486,
487, 498, 499,
};
+private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
+{
+ switch(hiByte)
+ {
+ case 0:
+ return ((jjbitVec2[i2] & l2) != 0L);
+ default :
+ if ((jjbitVec0[i1] & l1) != 0L)
+ return true;
+ return false;
+ }
+}
+private static final boolean jjCanMove_1(int hiByte, int i1, int i2, long l1, long l2)
+{
+ switch(hiByte)
+ {
+ case 0:
+ return ((jjbitVec2[i2] & l2) != 0L);
+ default :
+ return false;
+ }
+}
public static final String[] jjstrLiteralImages = {
"", null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null, null, "\74\41\55\55", "\55\55\76",
Index: CssParser.java
===================================================================
RCS file: /sources/public/2002/css-validator/org/w3c/css/parser/analyzer/CssParser.java,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- CssParser.java 13 Feb 2009 14:03:36 -0000 1.52
+++ CssParser.java 13 Feb 2009 21:50:15 -0000 1.53
@@ -2,6 +2,8 @@
package org.w3c.css.parser.analyzer;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
import java.util.Vector;
import java.util.Enumeration;
import java.net.URL;
@@ -160,8 +162,17 @@
* @param stream the stream data to parse.
* @param ac the new ac to use for parsing.
*/
- public void ReInitWithAc(InputStream stream, ApplContext ac) {
- ReInit(new CommentSkipperInputStream(stream));
+ public void ReInitWithAc(InputStream stream, ApplContext ac, String charset) {
+ InputStream is = new CommentSkipperInputStream(stream);
+ if (charset == null) {
+ charset = "iso-8859-1";
+ }
+ InputStreamReader isr = null;
+ try {
+ isr = new InputStreamReader(is, charset);
+ } catch (UnsupportedEncodingException uex) {}; // reinit, it can not happen...
+ // ...in theory ;)
+ ReInit(isr);
markRule = false;
setApplContext(ac);
}
@@ -3605,21 +3616,21 @@
finally { jj_save(0, xla); }
}
- final private boolean jj_3R_83() {
+ final private boolean jj_3_1() {
Token xsp;
xsp = jj_scanpos;
- if (jj_scan_token(34)) {
- jj_scanpos = xsp;
- if (jj_scan_token(47)) return true;
- }
+ if (jj_3R_83()) jj_scanpos = xsp;
+ if (jj_scan_token(98)) return true;
return false;
}
- final private boolean jj_3_1() {
+ final private boolean jj_3R_83() {
Token xsp;
xsp = jj_scanpos;
- if (jj_3R_83()) jj_scanpos = xsp;
- if (jj_scan_token(98)) return true;
+ if (jj_scan_token(34)) {
+ jj_scanpos = xsp;
+ if (jj_scan_token(47)) return true;
+ }
return false;
}
Index: CssParser.jj
===================================================================
RCS file: /sources/public/2002/css-validator/org/w3c/css/parser/analyzer/CssParser.jj,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- CssParser.jj 13 Feb 2009 14:03:36 -0000 1.52
+++ CssParser.jj 13 Feb 2009 21:50:15 -0000 1.53
@@ -10,7 +10,8 @@
options {
IGNORE_CASE = true;
STATIC = false;
- /*
+ UNICODE_INPUT = true;
+ /*
DEBUG_TOKEN_MANAGER = true;
DEBUG_PARSER = true;
*/
@@ -21,6 +22,8 @@
package org.w3c.css.parser.analyzer;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
import java.util.Vector;
import java.util.Enumeration;
import java.net.URL;
@@ -179,8 +182,17 @@
* @param stream the stream data to parse.
* @param ac the new ac to use for parsing.
*/
- public void ReInitWithAc(InputStream stream, ApplContext ac) {
- ReInit(new CommentSkipperInputStream(stream));
+ public void ReInitWithAc(InputStream stream, ApplContext ac, String charset) {
+ InputStream is = new CommentSkipperInputStream(stream);
+ if (charset == null) {
+ charset = "iso-8859-1";
+ }
+ InputStreamReader isr = null;
+ try {
+ isr = new InputStreamReader(is, charset);
+ } catch (UnsupportedEncodingException uex) {}; // reinit, it can not happen...
+ // ...in theory ;)
+ ReInit(isr);
markRule = false;
setApplContext(ac);
}
Received on Friday, 13 February 2009 21:50:32 UTC