Bug 3140 - allow UTF-8 BOM in slice files

1 files changed, 26 insertions, 1 deletions

diff --git a/cpp/src/Slice/Scanner.l b/cpp/src/Slice/Scanner.l
index 0cdeeea50ca..ac465f1d3c3 100644
--- a/cpp/src/Slice/Scanner.l
+++ b/cpp/src/Slice/Scanner.l
@@ -62,6 +62,8 @@ fractional_constant	(\+|-)?(([[:digit:]]*\.[[:digit:]]+)|([[:digit:]]+\.))
 exponent_part		(e|E)(\+|-)?[[:digit:]]+
 floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{exponent_part}))[fF]?
 
+%s MAINSCAN
+
 %%
 
 ^"#"[[:blank:]]*[[:digit:]]+[[:blank:]]*$ {
@@ -82,6 +84,7 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 
 "//" {
     // C++-style comment
+    BEGIN(MAINSCAN);
     int c;
     do
     {
@@ -96,6 +99,7 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 
 "/*" {
     // C-style comment
+    BEGIN(MAINSCAN);
     string comment = yytext + 2;
     while(true)
     {
@@ -135,26 +139,32 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 }
 
 "::" {
+    BEGIN(MAINSCAN);
     return ICE_SCOPE_DELIMITER;
 }
 
 "[" {
+    BEGIN(MAINSCAN);
     return ICE_METADATA_OPEN;
 }
 
 "]" {
+    BEGIN(MAINSCAN);
     return ICE_METADATA_CLOSE;
 }
 
 "[[" {
+    BEGIN(MAINSCAN);
     return ICE_GLOBAL_METADATA_OPEN;
 }
 
 "]]" {
+    BEGIN(MAINSCAN);
     return ICE_GLOBAL_METADATA_CLOSE;
 }
 
 {identifier}[[:space:]]*"(" {
+    BEGIN(MAINSCAN);
     StringTokPtr ident = new StringTok;
     ident->v = *yytext == '\\' ? yytext + 1 : yytext;
     ident->v.erase(ident->v.find_first_of(" \t\v\n\r\f("));
@@ -168,6 +178,7 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 }
 
 {identifier} {
+    BEGIN(MAINSCAN);
     StringTokPtr ident = new StringTok;
     ident->v = *yytext == '\\' ? yytext + 1 : yytext;
     *yylvalp = ident;
@@ -176,6 +187,7 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 }
 
 \" {
+    BEGIN(MAINSCAN);
     StringTokPtr str = new StringTok;
     str->literal = "\"";
     while(true)
@@ -336,6 +348,7 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 }
 
 {integer_constant} {
+    BEGIN(MAINSCAN);
     IntegerTokPtr itp = new IntegerTok;
     itp->literal = string(yytext);
     *yylvalp = itp;
@@ -351,6 +364,7 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 }
 
 {floating_literal} {
+    BEGIN(MAINSCAN);
     errno = 0;
     FloatingTokPtr ftp = new FloatingTok;
     *yylvalp = ftp;
@@ -380,15 +394,26 @@ floating_literal	(({fractional_constant}{exponent_part}?)|((\+|-)?[[:digit:]]+{e
 }
 
 [[:space:]] {
-    // Igore white-space
+    // Ignore white-space
     
+    if(unit->currentLine() != 0)
+    {
+        BEGIN(MAINSCAN);
+    }
     if(yytext[0] == '\n')
     {
 	unit->nextLine();
     }
 }
 
+<INITIAL>^"\357\273\277" {
+    // Ignore UTF-8 BOM, rule only active when parsing start of file.
+    
+    BEGIN(MAINSCAN);
+}
+
 . {
+    BEGIN(MAINSCAN);
     if(yytext[0] < 32 || yytext[0] > 126)
     {
 	stringstream s;