using System; using System.Text; using System.Collections; using System.IO; using System.Globalization; using System.Reflection; using Mono.CSharp; //%implements yyInput %% %namespace Mono.CSharp %public %class Tokenizer %implements yyParser.yyInput %type int %ignore_token Token.NONE %eofval{ return Token.EOF; %eofval} %{ /* this is where the interface to mcs is done (which is seperate from the core tokenizer */ /* instantation wrappers */ //public Tokenizer(System.IO.TextReader reader) : base (reader) {} //public Tokenizer(System.IO.FileStream instream) : base (instream) {} /* used by the parser */ public Tokenizer (StreamReader input, string fname, ArrayList defs) : this(input) { this.ref_line_offset=0; any_token_seen = false; this.ref_name = fname; caller_is_taking= true; if (defs != null) { defines = new Hashtable (); foreach (string def in defs) { define (def); } } Mono.CSharp.Location.Push (fname); } /* variables called by parser */ public int col { get { return yychar - yyEndOfLine ; } } public bool PropertyParsing { get { return handle_get_set; } set { handle_get_set = value; } } public bool AssemblyTargetParsing { get { return handle_assembly; } set { handle_assembly = value; } } public bool EventParsing { get { return handle_remove_add; } set { handle_remove_add = value; } } string error_details; public string error { get { return error_details; } } public string ref_name; // what on earth is this? public string location { get { string det; if (((int) current_token) == Token.ERROR) det = "detail: " + error_details; else det = ""; // return "Line: "+line+" Col: "+col + "\n" + // "VirtLine: "+ref_line + // " Token: "+current_token + " " + det; string current_token_name = TokenValueName [current_token] as string; if (current_token_name == null) current_token_name = current_token.ToString (); return String.Format ("{0} ({1},{2}), Token: {3} {4}", ref_name, line, col, current_token_name, det); } } /* defines and compiler directives */ Hashtable defines; string preprocessor_cmd; string preprocessor_args; Stack ifstack; bool caller_is_taking; public int ref_line_offset = 0; bool any_token_seen = false; const int TAKING = 1; const int TAKEN_BEFORE = 2; const int ELSE_SEEN = 4; const int PARENT_TAKING = 8; public int ref_line { get { return line - ref_line_offset; } set { ref_line_offset = line - value + 1; } } void define (string def) { if (!RootContext.AllDefines.Contains (def)) { RootContext.AllDefines [def] = true; } if (defines.Contains (def)) { return; } defines [def] = true; } bool is_identifier_start_character (char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || Char.IsLetter (c); } bool is_identifier_part_character (char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9') || Char.IsLetter (c); } // // Handles the #line directive // bool PreProcessLine (string arg) { if (arg == "") return false; if (arg == "default"){ ref_line = line; return false; } try { int pos; if ((pos = arg.IndexOf (' ')) != -1 && pos != 0){ ref_line = System.Int32.Parse (arg.Substring (0, pos)); pos++; char [] quotes = { '\"' }; ref_name = arg.Substring (pos). Trim (quotes); } else { ref_line = System.Int32.Parse (arg); } } catch { return false; } return true; } // // Handles #define and #undef // void PreProcessDefinition (bool is_define, string arg) { if (arg == "" || arg == "true" || arg == "false"){ Report.Error (1001, Location, "Missing identifer to pre-processor directive"); return; } char[] whitespace = { ' ', '\t' }; if (arg.IndexOfAny (whitespace) != -1){ Report.Error (1025, Location, "Single-line comment or end-of-line expected"); return; } foreach (char c in arg){ if (!Char.IsLetter (c) && (c != '_')){ Report.Error (1001, Location, "Identifier expected"); return; } } if (is_define){ if (defines == null) defines = new Hashtable (); define (arg); } else { if (defines == null) return; if (defines.Contains (arg)) defines.Remove (arg); } } bool eval_val (string s) { if (s == "true") return true; if (s == "false") return false; if (defines == null) return false; if (defines.Contains (s)) return true; return false; } bool pp_primary (ref string s) { s = s.Trim (); int len = s.Length; if (len > 0){ char c = s [0]; if (c == '('){ s = s.Substring (1); bool val = pp_expr (ref s); if (s.Length > 0 && s [0] == ')'){ s = s.Substring (1); return val; } Error_InvalidDirective (); return false; } if (is_identifier_start_character (c)){ int j = 1; while (j < len){ c = s [j]; if (is_identifier_part_character (c)){ j++; continue; } bool v = eval_val (s.Substring (0, j)); s = s.Substring (j); return v; } bool vv = eval_val (s); s = ""; return vv; } } Error_InvalidDirective (); return false; } bool pp_unary (ref string s) { s = s.Trim (); int len = s.Length; if (len > 0){ if (s [0] == '!'){ if (len > 1 && s [1] == '='){ Error_InvalidDirective (); return false; } s = s.Substring (1); return ! pp_primary (ref s); } else return pp_primary (ref s); } else { Error_InvalidDirective (); return false; } } bool pp_eq (ref string s) { bool va = pp_unary (ref s); s = s.Trim (); int len = s.Length; if (len > 0){ if (s [0] == '='){ if (len > 2 && s [1] == '='){ s = s.Substring (2); return va == pp_unary (ref s); } else { Error_InvalidDirective (); return false; } } else if (s [0] == '!' && len > 1 && s [1] == '='){ s = s.Substring (2); return va != pp_unary (ref s); } } return va; } bool pp_and (ref string s) { bool va = pp_eq (ref s); s = s.Trim (); int len = s.Length; if (len > 0){ if (s [0] == '&'){ if (len > 2 && s [1] == '&'){ s = s.Substring (2); return (va & pp_eq (ref s)); } else { Error_InvalidDirective (); return false; } } } return va; } // // Evaluates an expression for `#if' or `#elif' // bool pp_expr (ref string s) { bool va = pp_and (ref s); s = s.Trim (); int len = s.Length; if (len > 0){ char c = s [0]; if (c == '|'){ if (len > 2 && s [1] == '|'){ s = s.Substring (2); return va | pp_expr (ref s); } else { Error_InvalidDirective (); return false; } } } return va; } bool eval (string s) { bool v = pp_expr (ref s); s = s.Trim (); if (s.Length != 0){ Error_InvalidDirective (); return false; } return v; } void Error_InvalidDirective () { Report.Error (1517, Location, "Invalid pre-processor directive"); } void Error_UnexpectedDirective (string extra) { Report.Error ( 1028, Location, "Unexpected processor directive (" + extra + ")"); } void Error_TokensSeen () { Report.Error ( 1032, Location, "Cannot define or undefine pre-processor symbols after a token in the file"); } // // if true, then the code continues processing the code // if false, the code stays in a loop until another directive is // reached. // bool handle_preprocessing_directive (string cmd, string arg) { char [] blank = { ' ', '\t' }; // Eat any trailing whitespaces and single-line comments if (arg.IndexOf ("//") != -1) arg = arg.Substring (0, arg.IndexOf ("//")); arg = arg.TrimEnd (' ', '\t'); // // The first group of pre-processing instructions is always processed // switch (cmd){ case "line": if (!PreProcessLine (arg)) Report.Error ( 1576, Location, "Argument to #line directive is missing or invalid"); return true; case "region": arg = "true"; goto case "if"; case "endregion": goto case "endif"; case "if": if (arg == ""){ Error_InvalidDirective (); return true; } bool taking = false; if (ifstack == null) ifstack = new Stack (); if (ifstack.Count == 0){ taking = true; } else { int state = (int) ifstack.Peek (); if ((state & TAKING) != 0) taking = true; } if (eval (arg) && taking){ ifstack.Push (TAKING | TAKEN_BEFORE | PARENT_TAKING); return true; } else { ifstack.Push (taking ? PARENT_TAKING : 0); return false; } case "endif": if (ifstack == null || ifstack.Count == 0){ Error_UnexpectedDirective ("no #if for this #endif"); return true; } else { ifstack.Pop (); if (ifstack.Count == 0) return true; else { int state = (int) ifstack.Peek (); if ((state & TAKING) != 0) return true; else return false; } } case "elif": if (ifstack == null || ifstack.Count == 0){ Error_UnexpectedDirective ("no #if for this #elif"); return true; } else { int state = (int) ifstack.Peek (); if ((state & ELSE_SEEN) != 0){ Error_UnexpectedDirective ("#elif not valid after #else"); return true; } if ((state & (TAKEN_BEFORE | TAKING)) != 0) return false; if (eval (arg) && ((state & PARENT_TAKING) != 0)){ state = (int) ifstack.Pop (); ifstack.Push (state | TAKING | TAKEN_BEFORE); return true; } else return false; } case "else": if (ifstack == null || ifstack.Count == 0){ Report.Error ( 1028, Location, "Unexpected processor directive (no #if for this #else)"); return true; } else { int state = (int) ifstack.Peek (); if ((state & ELSE_SEEN) != 0){ Error_UnexpectedDirective ("#else within #else"); return true; } ifstack.Pop (); bool ret; if ((state & TAKEN_BEFORE) == 0){ ret = ((state & PARENT_TAKING) != 0); } else ret = false; if (ret) state |= TAKING; else state &= ~TAKING; ifstack.Push (state | ELSE_SEEN); return ret; } } // // These are only processed if we are in a `taking' block // if (!caller_is_taking) return false; switch (cmd){ case "define": if (any_token_seen){ Error_TokensSeen (); return true; } PreProcessDefinition (true, arg); return true; case "undef": if (any_token_seen){ Error_TokensSeen (); return true; } PreProcessDefinition (false, arg); return true; case "error": Report.Error (1029, Location, "#error: '" + arg + "'"); return true; case "warning": Report.Warning (1030, Location, "#warning: '" + arg + "'"); return true; } Report.Error (1024, Location, "Preprocessor directive expected (got: " + cmd + ")"); return true; } /* cheesy token lookup for debugging */ static Hashtable keywords { get { if (keywords == null) { InitTokens(); } return keywords; } set { keywords = value; } } static void InitTokens () { keywords = new Hashtable (); keywords.Add ("abstract", Token.ABSTRACT); keywords.Add ("as", Token.AS); keywords.Add ("add", Token.ADD); keywords.Add ("assembly", Token.ASSEMBLY); keywords.Add ("base", Token.BASE); keywords.Add ("bool", Token.BOOL); keywords.Add ("break", Token.BREAK); keywords.Add ("byte", Token.BYTE); keywords.Add ("case", Token.CASE); keywords.Add ("catch", Token.CATCH); keywords.Add ("char", Token.CHAR); keywords.Add ("checked", Token.CHECKED); keywords.Add ("class", Token.CLASS); keywords.Add ("const", Token.CONST); keywords.Add ("continue", Token.CONTINUE); keywords.Add ("decimal", Token.DECIMAL); keywords.Add ("default", Token.DEFAULT); keywords.Add ("delegate", Token.DELEGATE); keywords.Add ("do", Token.DO); keywords.Add ("double", Token.DOUBLE); keywords.Add ("else", Token.ELSE); keywords.Add ("enum", Token.ENUM); keywords.Add ("event", Token.EVENT); keywords.Add ("explicit", Token.EXPLICIT); keywords.Add ("extern", Token.EXTERN); keywords.Add ("false", Token.FALSE); keywords.Add ("finally", Token.FINALLY); keywords.Add ("fixed", Token.FIXED); keywords.Add ("float", Token.FLOAT); keywords.Add ("for", Token.FOR); keywords.Add ("foreach", Token.FOREACH); keywords.Add ("goto", Token.GOTO); keywords.Add ("get", Token.GET); keywords.Add ("if", Token.IF); keywords.Add ("implicit", Token.IMPLICIT); keywords.Add ("in", Token.IN); keywords.Add ("int", Token.INT); keywords.Add ("interface", Token.INTERFACE); keywords.Add ("internal", Token.INTERNAL); keywords.Add ("is", Token.IS); keywords.Add ("lock", Token.LOCK); keywords.Add ("long", Token.LONG); keywords.Add ("namespace", Token.NAMESPACE); keywords.Add ("new", Token.NEW); keywords.Add ("null", Token.NULL); keywords.Add ("object", Token.OBJECT); keywords.Add ("operator", Token.OPERATOR); keywords.Add ("out", Token.OUT); keywords.Add ("override", Token.OVERRIDE); keywords.Add ("params", Token.PARAMS); keywords.Add ("private", Token.PRIVATE); keywords.Add ("protected", Token.PROTECTED); keywords.Add ("public", Token.PUBLIC); keywords.Add ("readonly", Token.READONLY); keywords.Add ("ref", Token.REF); keywords.Add ("remove", Token.REMOVE); keywords.Add ("return", Token.RETURN); keywords.Add ("sbyte", Token.SBYTE); keywords.Add ("sealed", Token.SEALED); keywords.Add ("set", Token.SET); keywords.Add ("short", Token.SHORT); keywords.Add ("sizeof", Token.SIZEOF); keywords.Add ("stackalloc", Token.STACKALLOC); keywords.Add ("static", Token.STATIC); keywords.Add ("string", Token.STRING); keywords.Add ("struct", Token.STRUCT); keywords.Add ("switch", Token.SWITCH); keywords.Add ("this", Token.THIS); keywords.Add ("throw", Token.THROW); keywords.Add ("true", Token.TRUE); keywords.Add ("try", Token.TRY); keywords.Add ("typeof", Token.TYPEOF); keywords.Add ("uint", Token.UINT); keywords.Add ("ulong", Token.ULONG); keywords.Add ("unchecked", Token.UNCHECKED); keywords.Add ("unsafe", Token.UNSAFE); keywords.Add ("ushort", Token.USHORT); keywords.Add ("using", Token.USING); keywords.Add ("virtual", Token.VIRTUAL); keywords.Add ("void", Token.VOID); keywords.Add ("volatile", Token.VOLATILE); keywords.Add ("while", Token.WHILE); } private static Hashtable GetTokenValueNameHash () { Type t = typeof (Token); FieldInfo [] fields = t.GetFields (); Hashtable hash = new Hashtable (); foreach (FieldInfo field in fields) { if (field.IsLiteral && field.IsStatic && field.FieldType == typeof (int)) hash.Add (field.GetValue (null), field.Name); } return hash; } static Hashtable tokenValues; private static Hashtable TokenValueName { get { if (tokenValues == null) { tokenValues = GetTokenValueNameHash (); } return tokenValues; } } public string tokenName() { string current_token_name = TokenValueName [current_token] as string; return current_token_name; } public int current_token = Token.NONE; /** move on to next token. @returns false if positioned beyond tokens. @throws IOException on input error. */ public bool advance () { //if (!any_token_seen) { // return true; //} if (current_token == Token.EOF) { return false; } return true; } /** classifies current token. Should not be called if advance() returned false. @returns current %token or single character. */ public int token() { current_token = yylex(); if (current_token == Token.NONE) { return token(); } any_token_seen = true; return current_token; } /** associated with current token. Should not be called if advance() returned false. @returns value for token(). */ public Object value () { return val; } public Object Value { get { return val; } } public Location Location { get { return new Location (ref_line); } } public int Line { get { return yyline + 1; } } public int line { get { return yyline + 1; } } /* a few choice state variables :) */ public bool handle_get_set = false; public bool handle_remove_add = false; public bool handle_assembly = false; StringBuilder current_string_literal; void adjust_int (int startpos, int endpos, bool is_unsigned, bool is_long) { try { if ((endpos - startpos) > 9){ integer_type_suffix_signed_long ( System.UInt64.Parse (new string(yy_buffer,startpos,endpos-startpos)), is_unsigned, is_long); return; } else { uint ui = 0; for (int i = startpos; i < endpos; i++){ ui = checked ((ui * 10) + yy_buffer[i] - '0'); } integer_type_suffix_signed_long (ui, is_unsigned, is_long); return; } } catch (OverflowException) { error_details = "Integral constant is too large"; Report.Error (1021, Location, error_details); val = 0ul; } } void integer_type_suffix_signed_long (ulong ul, bool is_unsigned, bool is_long) { if (is_long && is_unsigned) { val = ul; return; } else if (is_unsigned) { // uint if possible, or ulong else. if ((ul & 0xffffffff00000000) == 0) { val = (uint) ul; return; } else { val = ul; return; } } else if (is_long) { // long if possible, ulong otherwise if ((ul & 0x8000000000000000) != 0) { val = ul; return; } else { val = (long) ul; return; } } else { // int, uint, long or ulong in that order if ((ul & 0xffffffff00000000) == 0) { uint ui = (uint) ul; if ((ui & 0x80000000) != 0) { val = ui; return; } else { val = (int) ui; return; } } else { if ((ul & 0x8000000000000000) != 0) { val = ul; return; } else { val = (long) ul; return; } } } } void adjust_real (int t,string s) { switch (t) { case Token.LITERAL_DECIMAL: try { val = System.Decimal.Parse (s, NumberStyles.Float, NumberFormatInfo.InvariantInfo); } catch (OverflowException) { val = 0m; error_details = "Floating-point constant is outside the range of the type 'decimal'"; Report.Error (594, Location, error_details); } break; case Token.LITERAL_FLOAT: try { val = (float) System.Double.Parse (s, NumberStyles.Float, NumberFormatInfo.InvariantInfo); } catch (OverflowException) { val = 0.0f; error_details = "Floating-point constant is outside the range of the type 'float'"; Report.Error (594, Location, error_details); } break; case Token.LITERAL_DOUBLE: try { val = System.Double.Parse (s, NumberStyles.Float, NumberFormatInfo.InvariantInfo); } catch (OverflowException) { val = 0.0; error_details = "Floating-point constant is outside the range of the type 'double'"; Report.Error (594, Location, error_details); } break; } return; } char simple_escape(char c) { switch (c) { case '0': return '\0'; case 'a': return '\a'; case 'b': return '\b'; case 'f': return '\f'; case 'n': return '\n'; case 'r': return '\r'; case 't': return '\t'; case 'v': return '\v'; default: return c; } } Object val; // %full removed to see if it helps? // %char removed to see if it helps? %} %line %full %char %state IN_COMMENT IN_STRING IN_PREPROCESSOR IN_PREPROCESSOR_ARGS IN_HIDDEN_CODE IN_AT_STRING PREPROCESSOR = ^[ \t]*#.* PREPROCESSOR_START = ^[ \t]*#[ \t]* LINE_BREAK = [\n\r] WHITE_SPACE = [ \t\n\r] SINGLE_LINE_COMMENT = "//"[^\r\n]*{LINE_BREAK} DEC_DIGIT = [0-9] HEX_DIGIT = [0-9A-Fa-f] LONG_SUFFIX = [Ll] UNSIGNED_SUFFIX = [Uu] UNSIGNED_LONG_SUFFIX = ([Uu][Ll]|[Ll][Uu]) DEC_LITERAL = ({DEC_DIGIT}+) DEC_LITERAL_LONG = ({DEC_DIGIT}+{LONG_SUFFIX}) DEC_LITERAL_UNSIGNED = ({DEC_DIGIT}+{UNSIGNED_SUFFIX}) DEC_LITERAL_UNSIGNED_LONG = ({DEC_DIGIT}+{UNSIGNED_LONG_SUFFIX}) HEX_LITERAL = (0[xX]?{HEX_DIGIT}+) HEX_LITERAL_LONG = (0[xX]?{HEX_DIGIT}+{LONG_SUFFIX}) HEX_LITERAL_UNSIGNED = (0[xX]?{HEX_DIGIT}+{UNSIGNED_SUFFIX}) HEX_LITERAL_UNSIGNED_LONG = (0[xX]?{HEX_DIGIT}+{UNSIGNED_LONG_SUFFIX}) REAL_SUFFIX = [FfDdMm] SIGN = [+\-] EXPONENT_PART = ([eE]{SIGN}?{DEC_DIGIT}+) WHOLE_REAL1 = ({DEC_DIGIT}+{EXPONENT_PART}{REAL_SUFFIX}?) WHOLE_REAL2 = ({DEC_DIGIT}+{REAL_SUFFIX}) PART_REAL = ({DEC_DIGIT}*\.{DEC_DIGIT}+{EXPONENT_PART}?{REAL_SUFFIX}?) REAL_LITERAL = ({WHOLE_REAL1}|{WHOLE_REAL2}|{PART_REAL}) REAL_WITH_EXPONENT = {DEC_DIGIT}+{EXPONENT_PART} REAL_WITH_POINT = {DEC_DIGIT}*\.{DEC_DIGIT}+{EXPONENT_PART}? REAL_LITERAL_DEFAULT = ({REAL_WITH_EXPONENT}|{REAL_WITH_POINT}) REAL_LITERAL_DOUBLE = ({DEC_DIGIT}+|{REAL_WITH_EXPONENT}|{REAL_WITH_POINT})[Dd] REAL_LITERAL_FLOAT = ({DEC_DIGIT}+|{REAL_WITH_EXPONENT}|{REAL_WITH_POINT})[Ff] REAL_LITERAL_DECIMAL = ({DEC_DIGIT}+|{REAL_WITH_EXPONENT}|{REAL_WITH_POINT})[Mm] SINGLE_CHAR = [^\\\'] SIMPLE_ESC_SEQ = \\[\'\"\\0abfnrtv] UNI_ESC_SEQ1 = \\u{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT} UNI_ESC_SEQ2 = \\U{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT} UNI_ESC_SEQ = {UNI_ESC_SEQ1}|{UNI_ESC_SEQ2} HEX_ESC_SEQ = (\\x{HEX_DIGIT}|\\x{HEX_DIGIT}{HEX_DIGIT}|\\x{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}|\\x{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}) CHARACTER = ({SINGLE_CHAR}|{SIMPLE_ESC_SEQ}|{HEX_ESC_SEQ}|{UNI_ESC_SEQ}) CHARACTER_LITERAL = \'{CHARACTER}\' SINGLE_STRING_CHAR = [^\\\"] REG_STRING_CHAR = ({SINGLE_STRING_CHAR}|{SIMPLE_ESC_SEQ}|{HEX_ESC_SEQ}|{UNI_ESC_SEQ}) REGULAR_STRING = \"{REG_STRING_CHAR}*\" SINGLE_VERBATIM_CHAR = [^\"] QUOTE_ESC_SEQ = \"\" VERB_STRING_CHAR = ({SINGLE_VERBATIM_CHAR}|{QUOTE_ESC_SEQ}) VERBATIM_STRING = @\"{VERB_STRING_CHAR}*\" STRING_LITERAL = ({REGULAR_STRING}|{VERBATIM_STRING}) EASY_STRING_LITERAL = \"{SINGLE_STRING_CHAR}*\" COMPLEX_STRING_BEGIN = \"{SINGLE_STRING_CHAR}* COMPLEX_STRING_END = \" LETTER_CHAR = [A-Za-z] IDENT_CHAR = ({DEC_DIGIT}|{LETTER_CHAR}|"_"|"@") IDENTIFIER = ({LETTER_CHAR}|"_"){IDENT_CHAR}* AT_IDENTIFIER = \@{IDENTIFIER} %% {PREPROCESSOR_START} { yybegin(IN_PREPROCESSOR); break; } {IDENTIFIER} { preprocessor_cmd = yytext(); yybegin(IN_PREPROCESSOR_ARGS); break;} [^\r\n]* { preprocessor_args = yytext().Trim(); //Console.WriteLine("Directive: " + preprocessor_cmd + " ARGS: " + preprocessor_args); bool cont = handle_preprocessing_directive(preprocessor_cmd,preprocessor_args); if (!cont) { caller_is_taking = false; yybegin(IN_HIDDEN_CODE); break; } caller_is_taking = true; yybegin(YYINITIAL); break; } [ \t]*{LINE_BREAK} { preprocessor_args = ""; //Console.WriteLine("Directive: " + preprocessor_cmd); bool cont = handle_preprocessing_directive(preprocessor_cmd,preprocessor_args); if (!cont) { caller_is_taking = false; //Console.WriteLine("In hidden code "); yybegin(IN_HIDDEN_CODE); break; } caller_is_taking = true; yybegin(YYINITIAL); break; } [^#]* { /* ignore */ break; } {WHITE_SPACE} { /* ignore */ break; } {SINGLE_LINE_COMMENT} { ;/* ignore */ break; } "/*" { yybegin(IN_COMMENT); break; } "/*" { break; } [\n\r] { break; } "*/" { yybegin(YYINITIAL); break; } . { break; } {DEC_LITERAL} { adjust_int(yy_buffer_start,yy_buffer_end,false,false);return Token.LITERAL_INTEGER; } {DEC_LITERAL_LONG} { adjust_int(yy_buffer_start,yy_buffer_end -1,false,true);return Token.LITERAL_INTEGER; } {DEC_LITERAL_UNSIGNED} { adjust_int(yy_buffer_start,yy_buffer_end -1,true,false);return Token.LITERAL_INTEGER; } {DEC_LITERAL_UNSIGNED_LONG} { adjust_int(yy_buffer_start,yy_buffer_end -2,true,true);return Token.LITERAL_INTEGER; } {HEX_LITERAL} { integer_type_suffix_signed_long( System.UInt64.Parse ( new string(yy_buffer,yy_buffer_start+2,yy_buffer_end - yy_buffer_start - 2) ,NumberStyles.HexNumber ), false,false); return Token.LITERAL_INTEGER; } {HEX_LITERAL_LONG} { integer_type_suffix_signed_long( System.UInt64.Parse ( new string(yy_buffer,yy_buffer_start+2,yy_buffer_end - yy_buffer_start - 3) ,NumberStyles.HexNumber ), false,true); return Token.LITERAL_INTEGER; } {HEX_LITERAL_UNSIGNED} { integer_type_suffix_signed_long( System.UInt64.Parse ( new string(yy_buffer,yy_buffer_start+2,yy_buffer_end - yy_buffer_start - 3) ,NumberStyles.HexNumber ), true,false); return Token.LITERAL_INTEGER; } {HEX_LITERAL_UNSIGNED_LONG} { val = System.UInt64.Parse ( new string(yy_buffer,yy_buffer_start+2,yy_buffer_end - yy_buffer_start - 4) ,NumberStyles.HexNumber ); return Token.LITERAL_INTEGER; } {REAL_LITERAL_DEFAULT} { adjust_real(Token.LITERAL_DOUBLE,yytext()); return Token.LITERAL_DOUBLE; } {REAL_LITERAL_DOUBLE} { adjust_real(Token.LITERAL_DOUBLE,yytext().Substring(0,yytext().Length - 1)); return Token.LITERAL_DOUBLE; } {REAL_LITERAL_FLOAT} { adjust_real(Token.LITERAL_FLOAT,yytext().Substring(0,yytext().Length - 1)); return Token.LITERAL_FLOAT; } {REAL_LITERAL_DECIMAL} { adjust_real(Token.LITERAL_DECIMAL,yytext().Substring(0,yytext().Length - 1)); return Token.LITERAL_DECIMAL; } \'{SINGLE_CHAR}\' { val = (char) yy_buffer[yy_buffer_start + 1]; return Token.LITERAL_CHARACTER; } \'{SIMPLE_ESC_SEQ}\' { val = simple_escape((char) yy_buffer[yy_buffer_start + 2]); return Token.LITERAL_CHARACTER; } \'({HEX_ESC_SEQ}|{UNI_ESC_SEQ1})\' { // '\x0a0a0a' or '\u0a0a0a' val = (char) System.UInt32.Parse ( new string(yy_buffer,yy_buffer_start+3,yy_buffer_end - yy_buffer_start - 4) ,NumberStyles.HexNumber ); return Token.LITERAL_CHARACTER; } \'({UNI_ESC_SEQ2})\' { // '\x0a0a0a' or '\u0a0a0a' val = (char) System.UInt64.Parse ( new string(yy_buffer,yy_buffer_start+3,yy_buffer_end - yy_buffer_start - 4) ,NumberStyles.HexNumber ); return Token.LITERAL_CHARACTER; } {EASY_STRING_LITERAL} { val = new string(yy_buffer,yy_buffer_start+1,yy_buffer_end - yy_buffer_start - 2); return Token.LITERAL_STRING; } {COMPLEX_STRING_BEGIN} { current_string_literal = new StringBuilder(yytext().Substring(1,yytext().Length - 1)); yybegin(IN_STRING); break; } {SINGLE_STRING_CHAR}* { current_string_literal.Append( yytext()); break;} {SIMPLE_ESC_SEQ} { current_string_literal.Append( simple_escape((char) yy_buffer[yy_buffer_start + 1]) ); break; } {HEX_ESC_SEQ}|{UNI_ESC_SEQ1} { // \x0f0f0f0f0 or \u0f0f0f0f char c = (char) System.UInt32.Parse ( new string(yy_buffer,yy_buffer_start+2,yy_buffer_end - yy_buffer_start - 2) ,NumberStyles.HexNumber ); current_string_literal.Append(c); break; } {UNI_ESC_SEQ2}) { char c = (char) System.UInt64.Parse ( new string(yy_buffer,yy_buffer_start+2,yy_buffer_end - yy_buffer_start - 2) ,NumberStyles.HexNumber ); current_string_literal.Append(c); break; } {COMPLEX_STRING_END} { val = current_string_literal.ToString(); yybegin(YYINITIAL); return Token.LITERAL_STRING; } @\" { current_string_literal = new StringBuilder(); yybegin(IN_AT_STRING); break; } {SINGLE_VERBATIM_CHAR}* { //Console.WriteLine("ADDING : " + yytext()); current_string_literal.Append( yytext()); break; } {QUOTE_ESC_SEQ} { current_string_literal.Append('"' + '"'); break; } "{" { val =null;return Token.OPEN_BRACE; } "}" { val =null;return Token.CLOSE_BRACE ; } "[" { val =null;return Token.OPEN_BRACKET ; } "]" { val =null;return Token.CLOSE_BRACKET ; } "(" { val =null;return Token.OPEN_PARENS ; } ")" { val =null;return Token.CLOSE_PARENS ; } "." { val =null;return Token.DOT ; } "," { val =null;return Token.COMMA ; } ":" { val =null;return Token.COLON ; } ";" { val =null;return Token.SEMICOLON ; } "~" { val =null;return Token.TILDE ; } "+" { val =null;return Token.PLUS ; } "-" { val =null;return Token.MINUS ; } "!" { val =null;return Token.BANG ; } "=" { val =null;return Token.ASSIGN ; } "<" { val =null;return Token.OP_LT ; } ">" { val =null;return Token.OP_GT ; } "&" { val =null;return Token.BITWISE_AND ; } "|" { val =null;return Token.BITWISE_OR ; } "*" { val =null;return Token.STAR ; } "%" { val =null;return Token.PERCENT ; } "/" { val =null;return Token.DIV ; } "^" { val =null;return Token.CARRET ; } "?" { val =null;return Token.INTERR ; } "++" { val =null;return Token.OP_INC ; } "--" { val =null;return Token.OP_DEC ; } "<<" { val =null;return Token.OP_SHIFT_LEFT ; } ">>" { val =null;return Token.OP_SHIFT_RIGHT ; } "<=" { val =null;return Token.OP_LE ; } ">=" { val =null;return Token.OP_GE ; } "==" { val =null;return Token.OP_EQ ; } "!=" { val =null;return Token.OP_NE ; } "&&" { val =null;return Token.OP_AND ; } "||" { val =null;return Token.OP_OR ; } "*=" { val =null;return Token.OP_MULT_ASSIGN ; } "/=" { val =null;return Token.OP_DIV_ASSIGN ; } "%=" { val =null;return Token.OP_MOD_ASSIGN ; } "+=" { val =null;return Token.OP_ADD_ASSIGN ; } "-=" { val =null;return Token.OP_SUB_ASSIGN ; } "<<=" { val =null;return Token.OP_SHIFT_LEFT_ASSIGN ; } ">>=" { val =null;return Token.OP_SHIFT_RIGHT_ASSIGN ; } "&=" { val =null;return Token.OP_AND_ASSIGN ; } "^=" { val =null;return Token.OP_XOR_ASSIGN ; } "|=" { val =null;return Token.OP_OR_ASSIGN ; } "->" { val =null;return Token.OP_PTR ; } "abstract" { val =null;return Token.ABSTRACT ; } "as" { val =null;return Token.AS ; } "base" { val =null;return Token.BASE ; } "bool" { val =null;return Token.BOOL ; } "break" { val =null;return Token.BREAK ; } "byte" { val =null;return Token.BYTE ; } "case" { val =null;return Token.CASE ; } "catch" { val =null;return Token.CATCH ; } "char" { val =null;return Token.CHAR ; } "checked" { val =null;return Token.CHECKED ; } "class" { val =null;return Token.CLASS ; } "const" { val =null;return Token.CONST ; } "continue" { val =null;return Token.CONTINUE ; } "decimal" { val =null;return Token.DECIMAL ; } "default" { val =null;return Token.DEFAULT ; } "delegate" { val =null;return Token.DELEGATE ; } "do" { val =null;return Token.DO ; } "double" { val =null;return Token.DOUBLE ; } "else" { val =null;return Token.ELSE ; } "enum" { val =null;return Token.ENUM ; } "event" { val =null;return Token.EVENT ; } "explicit" { val =null;return Token.EXPLICIT ; } "extern" { val =null;return Token.EXTERN ; } "false" { val =null;return Token.FALSE ; } "finally" { val =null;return Token.FINALLY ; } "fixed" { val =null;return Token.FIXED ; } "float" { val =null;return Token.FLOAT ; } "for" { val =null;return Token.FOR ; } "foreach" { val =null;return Token.FOREACH ; } "goto" { val =null;return Token.GOTO ; } "if" { val =null;return Token.IF ; } "implicit" { val =null;return Token.IMPLICIT ; } "in" { val =null;return Token.IN ; } "int" { val =null;return Token.INT ; } "interface" { val =null;return Token.INTERFACE ; } "internal" { val =null;return Token.INTERNAL ; } "is" { val =null;return Token.IS ; } "lock" { val =null;return Token.LOCK ; } "long" { val =null;return Token.LONG ; } "namespace" { val =null;return Token.NAMESPACE ; } "new" { val =null;return Token.NEW ; } "null" { val =null;return Token.NULL ; } "object" { val =null;return Token.OBJECT ; } "operator" { val =null;return Token.OPERATOR ; } "out" { val =null;return Token.OUT ; } "override" { val =null;return Token.OVERRIDE ; } "params" { val =null;return Token.PARAMS ; } "private" { val =null;return Token.PRIVATE ; } "protected" { val =null;return Token.PROTECTED ; } "public" { val =null;return Token.PUBLIC ; } "readonly" { val =null;return Token.READONLY ; } "ref" { val =null;return Token.REF ; } "return" { val =null;return Token.RETURN ; } "sbyte" { val =null;return Token.SBYTE ; } "sealed" { val =null;return Token.SEALED ; } "short" { val =null;return Token.SHORT ; } "sizeof" { val =null;return Token.SIZEOF ; } "stackalloc" { val =null;return Token.STACKALLOC ; } "static" { val =null;return Token.STATIC ; } "string" { val =null;return Token.STRING ; } "struct" { val =null;return Token.STRUCT ; } "switch" { val =null;return Token.SWITCH ; } "this" { val =null;return Token.THIS ; } "throw" { val =null;return Token.THROW ; } "true" { val =null;return Token.TRUE ; } "try" { val =null;return Token.TRY ; } "typeof" { val =null;return Token.TYPEOF ; } "uint" { val =null;return Token.UINT ; } "ulong" { val =null;return Token.ULONG ; } "unchecked" { val =null;return Token.UNCHECKED ; } "unsafe" { val =null;return Token.UNSAFE ; } "ushort" { val =null;return Token.USHORT ; } "using" { val =null;return Token.USING ; } "virtual" { val =null;return Token.VIRTUAL ; } "void" { val =null;return Token.VOID ; } "volatile" { val =null;return Token.VOLATILE ; } "while" { val =null;return Token.WHILE ; } "get" { if (handle_get_set == false) { val = yytext(); return Token.IDENTIFIER; } val =null; return Token.GET; } "set" { if (handle_get_set == false) { val = yytext(); return Token.IDENTIFIER; } val =null; return Token.SET ; } "remove" { if (handle_remove_add == false) { val = yytext(); return Token.IDENTIFIER; } val =null; return Token.REMOVE ; } "add" { if (handle_remove_add == false) { val = yytext(); return Token.IDENTIFIER; } val =null; return Token.ADD ; } "assembly" { if (handle_assembly == false) { val = yytext(); return Token.IDENTIFIER; } val =null; return Token.ASSEMBLY ; } {IDENTIFIER} { val =yytext();return Token.IDENTIFIER ; } {AT_IDENTIFIER} { val =yytext().Substring(1,yytext().Length -1);return Token.IDENTIFIER; } . { StringBuilder sb = new StringBuilder("Illegal character: <"); String s = yytext(); for (int i = 0; i < s.Length; i++) if (s[i] >= 32) sb.Append(s[i]); else { sb.Append("^"); sb.Append(Convert.ToChar(s[i]+'A'-1)); } sb.Append(">"); Console.WriteLine(sb.ToString()); return Token.ERROR; }