diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java index de55462546..721d217763 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java @@ -26,6 +26,10 @@ public class CustomDescriptors { getLargeLexerDescriptor(), getAtnStatesSizeMoreThan65535Descriptor() }); + descriptors.put("ParserExec", + new RuntimeTestDescriptor[] { + getMultiTokenAlternativeDescriptor() + }); } private static RuntimeTestDescriptor getLineSeparatorLfDescriptor() { @@ -146,4 +150,48 @@ private static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor() { null, false, false, new String[] {"CSharp", "Python2", "Python3", "Go", "PHP", "Swift", "JavaScript", "Dart"}, uri); } + + private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() { + final int tokensCount = 64; + + StringBuilder rule = new StringBuilder("r1: "); + StringBuilder tokens = new StringBuilder(); + StringBuilder input = new StringBuilder(); + StringBuilder output = new StringBuilder(); + + for (int i = 0; i < tokensCount; i++) { + String currentToken = "T" + i; + rule.append(currentToken); + if (i < tokensCount - 1) { + rule.append(" | "); + } else { + rule.append(";"); + } + tokens.append(currentToken).append(": '").append(currentToken).append("';\n"); + input.append(currentToken).append(" "); + output.append(currentToken); + } + String currentToken = "T" + tokensCount; + tokens.append(currentToken).append(": '").append(currentToken).append("';\n"); + input.append(currentToken).append(" "); + output.append(currentToken); + + String grammar = "grammar P;\n" + + "r: (r1 | T" + tokensCount + ")+ EOF {};\n" + + rule + "\n" + + tokens + "\n" + + "WS: [ ]+ -> skip;"; + + return new RuntimeTestDescriptor( + GrammarType.Parser, + "MultiTokenAlternative", + "https://github.com/antlr/antlr4/issues/3698, https://github.com/antlr/antlr4/issues/3703", + input.toString(), + output + "\n", + "", + "r", + "P", + grammar, + null, false, false, null, uri); + } } diff --git a/runtime/Swift/Sources/Antlr4/misc/Utils.swift b/runtime/Swift/Sources/Antlr4/misc/Utils.swift index 0866e20c50..78d70a14de 100644 --- a/runtime/Swift/Sources/Antlr4/misc/Utils.swift +++ b/runtime/Swift/Sources/Antlr4/misc/Utils.swift @@ -2,7 +2,7 @@ /// Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. /// Use of this file is governed by the BSD 3-clause license that /// can be found in the LICENSE.txt file in the project root. -/// +/// import Foundation @@ -39,24 +39,4 @@ public class Utils { } return m } - - public static func bitLeftShift(_ n: Int) -> Int64 { - return (Int64(1) &<< n) - } - - - public static func testBitLeftShiftArray(_ nArray: [Int],_ bitsShift: Int) -> Bool { - let test: Bool = (((nArray[0] - bitsShift) & ~0x3f) == 0) - - var temp: Int64 = Int64(nArray[0] - bitsShift) - temp = (temp < 0) ? (64 + (temp % 64 )) : (temp % 64) - let test1: Int64 = (Int64(1) << temp) - - var test2: Int64 = Utils.bitLeftShift(nArray[1] - bitsShift) - - for i in 1 ..< nArray.count { - test2 = test2 | Utils.bitLeftShift(nArray[i] - bitsShift) - } - return test && (( test1 & test2 ) != 0) - } } diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg index 2df658642f..e0dd9a9887 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg @@ -557,7 +557,7 @@ ErrorHandler.Sync(this); = TokenStream.LT(1); switch (TokenStream.LA(1)) { - + break;}; separator="\n"> default: @@ -569,7 +569,7 @@ LL1OptionalBlock(choice, alts, error) ::= << State = ; ErrorHandler.Sync(this); switch (TokenStream.LA(1)) { - + break;}; separator="\n"> default: @@ -676,7 +676,7 @@ Sync(s) ::= "Sync();" ThrowNoViableAlt(t) ::= "throw new NoViableAltException(this);" TestSetInline(s) ::= << -}; separator=" || "> +}; separator=" || "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test @@ -684,9 +684,9 @@ testShiftInRange(shiftAmount) ::= << (() & ~0x3f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items +// produces smaller bytecode only when bits.tokens contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> && ((1L \<\< ) & ()}; separator=" | ">)) != 0) +})> && ((1L \<\< ) & L) != 0 %> isZero ::= [ @@ -698,13 +698,12 @@ offsetShift(shiftAmount, offset) ::= <% ( - ) %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% -==}; separator=" || "> +==}; separator=" || "> %> -cases(ttypes) ::= << -:}; separator="\n"> +cases(tokens) ::= << +:}; separator="\n"> >> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg index 8d352f0d6c..81a6162a6f 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg @@ -652,7 +652,7 @@ _errHandler->sync(this); LL1AltBlock(choice, preamble, alts, error) = _input->LT(1); switch (_input->LA(1)) { - { + { break; \} @@ -667,7 +667,7 @@ LL1OptionalBlock(choice, alts, error) ::= << setState(); _errHandler->sync(this); switch (_input->LA(1)) { - { + { break; \} @@ -793,7 +793,7 @@ ThrowNoViableAlt(t) ::= "throw NoViableAltException(this);" TestSetInlineHeader(s) ::= "" TestSetInline(s) ::= << -}; separator=" || "> +}; separator=" || "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test @@ -801,10 +801,9 @@ testShiftInRange(shiftAmount) ::= << (( & ~ 0x3fULL) == 0) >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= << -(})> && - ((1ULL \<\< ) & ()}; separator = "\n | ">)) != 0) +})> && + ((1ULL \<\< ) & ) != 0 >> isZero ::= [ @@ -816,13 +815,12 @@ offsetShift(shiftAmount, offset, prefix = false) ::= <% (:: - ):: %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% - == ::}; separator = "\n\n|| "> + == ::}; separator = "\n\n|| "> %> -cases(ttypes) ::= << -:::}; separator="\n"> +cases(tokens) ::= << +:::}; separator="\n"> >> InvokeRuleHeader(r, argExprsChunks) ::= "InvokeRuleHeader" diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Dart/Dart.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Dart/Dart.stg index 769b5cc0ee..7567cb7340 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Dart/Dart.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Dart/Dart.stg @@ -391,7 +391,7 @@ errorHandler.sync(this); = tokenStream.LT(1); switch (tokenStream.LA(1)!) { - + break;}; separator="\n"> default: @@ -403,7 +403,7 @@ LL1OptionalBlock(choice, alts, error) ::= << state = ; errorHandler.sync(this); switch (tokenStream.LA(1)!) { - + break;}; separator="\n"> default: @@ -510,7 +510,7 @@ Sync(s) ::= "sync();" ThrowNoViableAlt(t) ::= "throw NoViableAltException(this);" TestSetInline(s) ::= << -}; separator=" || "> +}; separator=" || "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test @@ -518,9 +518,8 @@ testShiftInRange(shiftAmount) ::= << (() & ~0x3f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> && ((BigInt.one \<\< ) & (}, bits.shift)>)}; separator=" | ">)) != BigInt.zero) +})> && ((1 \<\< ) & ) != 0 %> isZero ::= [ @@ -532,13 +531,12 @@ offsetShift(shiftAmount, offset) ::= <% ( - ) %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% - == TOKEN_}; separator=" || "> + == TOKEN_}; separator=" || "> %> -cases(ttypes) ::= << -:}; separator="\n"> +cases(tokens) ::= << +:}; separator="\n"> >> InvokeRule(r, argExprsChunks) ::=<< diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg index 15df5efbf4..d23802d0ca 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg @@ -553,7 +553,7 @@ p.GetErrorHandler().Sync(p) switch p.GetTokenStream().LA(1) { -}; separator=", ">: +}; separator=", ">: }; separator="\n\n"> @@ -571,7 +571,7 @@ p.GetErrorHandler().Sync(p) switch p.GetTokenStream().LA(1) { -}; separator=", ">: +}; separator=", ">: }; separator="\n\n"> @@ -725,17 +725,16 @@ Sync(s) ::= "Sync()" ThrowNoViableAlt(t) ::= "panic(antlr.NewNoViableAltException(p, nil, nil, nil, nil, nil))" TestSetInline(s) ::= << -}; separator=" || "> +}; separator=" || "> >> // Javascript language spec - shift operators are 32 bits long max testShiftInRange(shiftAmount) ::= << -(() & -(0x1f+1)) == 0 +(int64() & ^0x3f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> && ((1 \<\< uint()) & ()}; separator=" | ">)) != 0) +})> && ((int64(1) \<\< ) & ) != 0 %> isZero ::= [ @@ -751,9 +750,8 @@ offsetShiftType(shiftAmount, offset) ::= <% ( - ) %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% - == }; separator=" || "> + == }; separator=" || "> %> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index c29bcab557..0dc40841e6 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -486,7 +486,7 @@ _errHandler.sync(this); = _input.LT(1); switch (_input.LA(1)) { - + break;}; separator="\n"> default: @@ -498,7 +498,7 @@ LL1OptionalBlock(choice, alts, error) ::= << setState(); _errHandler.sync(this); switch (_input.LA(1)) { - + break;}; separator="\n"> default: @@ -605,7 +605,7 @@ Sync(s) ::= "sync();" ThrowNoViableAlt(t) ::= "throw new NoViableAltException(this);" TestSetInline(s) ::= << -}; separator=" || "> +}; separator=" || "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test @@ -613,9 +613,8 @@ testShiftInRange(shiftAmount) ::= << (() & ~0x3f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> && ((1L \<\< ) & ()}; separator=" | ">)) != 0) +})> && ((1L \<\< ) & L) != 0 %> isZero ::= [ @@ -627,13 +626,12 @@ offsetShift(shiftAmount, offset) ::= <% ( - ) %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% -==}; separator=" || "> +==}; separator=" || "> %> -cases(ttypes) ::= << -:}; separator="\n"> +cases(tokens) ::= << +:}; separator="\n"> >> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg index 9ed2dea76e..1bee0e88ad 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg @@ -327,7 +327,7 @@ this._errHandler.sync(this); = this._input.LT(1); switch(this._input.LA(1)) { - + break;}; separator="\n"> default: @@ -339,7 +339,7 @@ LL1OptionalBlock(choice, alts, error) ::= << this.state = ; this._errHandler.sync(this); switch (this._input.LA(1)) { - + break;}; separator="\n"> default: @@ -449,7 +449,7 @@ Sync(s) ::= "sync()" ThrowNoViableAlt(t) ::= "throw new antlr4.error.NoViableAltException(this);" TestSetInline(s) ::= << -}; separator=" || "> +}; separator=" || "> >> // Javascript language spec - shift operators are 32 bits long max @@ -457,9 +457,8 @@ testShiftInRange(shiftAmount) ::= << (() & ~0x1f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> && ((1 \<\< ) & ()}; separator=" | ">)) !== 0) +(})> && ((1 \<\< ) & ) !== 0) %> isZero ::= [ @@ -475,13 +474,12 @@ offsetShiftType(shiftAmount, offset) ::= <% (. - ). %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% -===.}; separator=" || "> +===}; separator=" || "> %> -cases(ttypes) ::= << -.:}; separator="\n"> +cases(tokens) ::= << +:}; separator="\n"> >> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg index 19df7307b1..8dab298002 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/PHP/PHP.stg @@ -533,7 +533,7 @@ $this->errorHandler->sync($this); switch ($this->input->LA(1)) { - + break;}; separator="\n\n"> @@ -547,7 +547,7 @@ $this->setState(); $this->errorHandler->sync($this); switch ($this->input->LA(1)) { - + break;}; separator="\n\n"> @@ -667,7 +667,7 @@ Sync(s) ::= "sync();" ThrowNoViableAlt(t) ::= "throw new NoViableAltException($this);" TestSetInline(s) ::= << -}; separator=" || "> +}; separator=" || "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test @@ -675,9 +675,8 @@ testShiftInRange(shiftAmount) ::= << (() & ~0x3f) === 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> && ((1 \<\< ) & ()}; separator=" | ">)) !== 0) +})> && ((1 \<\< ) & ) !== 0 %> isZero ::= [ @@ -692,13 +691,12 @@ offsetShiftConst(shiftAmount, offset) ::= <% (self:: - )self:: %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% - === self::}; separator=" || "> + === self::}; separator=" || "> %> -cases(ttypes) ::= << -:}; separator="\n"> +cases(tokens) ::= << +:}; separator="\n"> >> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg index a1da6d4352..defcb040dd 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg @@ -324,7 +324,7 @@ self._errHandler.sync(self) = _input.LT(1) token = self._input.LA(1) - + pass}; separator="\nel"> else: @@ -336,7 +336,7 @@ LL1OptionalBlock(choice, alts, error) ::= << self.state = self._errHandler.sync(self) token = self._input.LA(1) - + pass}; separator="\nel"> else: @@ -443,7 +443,7 @@ Sync(s) ::= "sync()" ThrowNoViableAlt(t) ::= "raise NoViableAltException(self)" TestSetInline(s) ::= << -}; separator=" or "> +}; separator=" or "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test @@ -451,9 +451,8 @@ testShiftInRange(shiftAmount) ::= << (() & ~0x3f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> and ((1 \<\< ) & ()}; separator=" | ">)) != 0) +})> and ((1 \<\< ) & ) != 0 %> isZero ::= [ @@ -469,13 +468,12 @@ offsetShiftType(shiftAmount, offset) ::= <% (. - ). %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% -==.}; separator=" or "> +==}; separator=" or "> %> -cases(ttypes) ::= << -if token in [.}; separator=", ">]: +cases(tokens) ::= << +if token in [}; separator=", ">]: >> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index bb4d53f63c..9a4ad75166 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -337,7 +337,7 @@ self._errHandler.sync(self) = _input.LT(1) token = self._input.LA(1) - + pass}; separator="\nel"> else: @@ -349,7 +349,7 @@ LL1OptionalBlock(choice, alts, error) ::= << self.state = self._errHandler.sync(self) token = self._input.LA(1) - + pass}; separator="\nel"> else: @@ -456,7 +456,7 @@ Sync(s) ::= "sync()" ThrowNoViableAlt(t) ::= "raise NoViableAltException(self)" TestSetInline(s) ::= << -}; separator=" or "> +}; separator=" or "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test @@ -464,9 +464,8 @@ testShiftInRange(shiftAmount) ::= << (() & ~0x3f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= <% -(})> and ((1 \<\< ) & ()}; separator=" | ">)) != 0) +})> and ((1 \<\< ) & ) != 0 %> isZero ::= [ @@ -482,13 +481,12 @@ offsetShiftType(shiftAmount, offset) ::= <% (. - ). %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% -==.}; separator=" or "> +==}; separator=" or "> %> -cases(ttypes) ::= << -if token in [.}; separator=", ">]: +cases(tokens) ::= << +if token in [}; separator=", ">]: >> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg index 9fda44ae9d..ee5835d145 100755 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg @@ -488,7 +488,7 @@ try _errHandler.sync(self) = try _input.LT(1) switch (.Tokens(rawValue: try _input.LA(1))!) { - + break}; separator="\n"> default: @@ -500,7 +500,7 @@ LL1OptionalBlock(choice, alts, error) ::= << setState() try _errHandler.sync(self) switch (.Tokens(rawValue: try _input.LA(1))!) { - + break}; separator="\n"> default: @@ -609,55 +609,36 @@ Sync(s) ::= "sync();" ThrowNoViableAlt(t) ::= "throw ANTLRException.recognition(e: NoViableAltException(self))" TestSetInline(s) ::= << -}; separator=" || ">!> - //closure - { () -> Bool in - varlet testSet: Bool = }> - }; separator="\n"> - return testSet - }() +}; separator=" || "> >> // Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test testShiftInRange(shiftAmount) ::= << -(() & ~0x3f) == 0 +(Int64() & ~0x3f) == 0 >> -// produces smaller bytecode only when bits.ttypes contains more than two items bitsetBitfieldComparison(s, bits) ::= << -})> && ((1 \<\< ) & ()}; separator=" | ">)) != 0)!> -{ () -> Bool in - })>)!> - )!> - - - )}>)!> - )}; separator="\n">!> - let testArray: [Int] = [, }; separator=",">] - - return Utils.testBitLeftShiftArray(testArray, ) -}() +})> && ((Int64(1) \<\< ) & ) != 0 >> isZero ::= [ "0": true, default: false ] -parserName(ttype) ::= <% - .Tokens..rawValue +parserName(t) ::= <% + .Tokens..rawValue %> offsetShift(shiftAmount, offset) ::= <% ( - ) %> -// produces more efficient bytecode when bits.ttypes contains at most two items bitsetInlineComparison(s, bits) ::= <% - == .Tokens..rawValue}; separator=" || "> + == .Tokens..rawValue}; separator=" || "> %> -cases(ttypes) ::= << -:fallthrough} ; separator="\n"> -:} ; separator="\n"> +cases(tokens) ::= << +:fallthrough} ; separator="\n"> +:} ; separator="\n"> >> InvokeRule(r, argExprsChunks) ::= << diff --git a/tool/src/org/antlr/v4/codegen/model/Choice.java b/tool/src/org/antlr/v4/codegen/model/Choice.java index 6b0d279d1d..614f0e50f6 100644 --- a/tool/src/org/antlr/v4/codegen/model/Choice.java +++ b/tool/src/org/antlr/v4/codegen/model/Choice.java @@ -7,10 +7,13 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.codegen.Target; import org.antlr.v4.codegen.model.decl.Decl; import org.antlr.v4.codegen.model.decl.TokenTypeDecl; import org.antlr.v4.misc.Utils; +import org.antlr.v4.runtime.misc.IntegerList; import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.ast.GrammarAST; import java.util.ArrayList; @@ -45,10 +48,17 @@ public void addPreambleOp(SrcOp op) { preamble.add(op); } - public List getAltLookaheadAsStringLists(IntervalSet[] altLookSets) { - List altLook = new ArrayList(); + public List getAltLookaheadAsStringLists(IntervalSet[] altLookSets) { + List altLook = new ArrayList<>(); + Target target = factory.getGenerator().getTarget(); + Grammar grammar = factory.getGrammar(); for (IntervalSet s : altLookSets) { - altLook.add(factory.getGenerator().getTarget().getTokenTypesAsTargetLabels(factory.getGrammar(), s.toArray())); + IntegerList list = s.toIntegerList(); + TokenInfo[] info = new TokenInfo[list.size()]; + for (int i = 0; i < info.length; i++) { + info[i] = new TokenInfo(list.get(i), target.getTokenTypeAsTargetLabel(grammar, list.get(i))); + } + altLook.add(info); } return altLook; } diff --git a/tool/src/org/antlr/v4/codegen/model/LL1Choice.java b/tool/src/org/antlr/v4/codegen/model/LL1Choice.java index 745ae4e608..2befecc1a3 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1Choice.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1Choice.java @@ -13,7 +13,7 @@ public abstract class LL1Choice extends Choice { /** Token names for each alt 0..n-1 */ - public List altLook; + public List altLook; @ModelElement public ThrowNoViableAlt error; public LL1Choice(OutputModelFactory factory, GrammarAST blkAST, diff --git a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java index 19ecd15ec0..3ddb175a31 100644 --- a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java +++ b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java @@ -7,6 +7,7 @@ package org.antlr.v4.codegen.model; import org.antlr.v4.codegen.OutputModelFactory; +import org.antlr.v4.codegen.Target; import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.tool.ast.GrammarAST; @@ -15,9 +16,9 @@ /** */ public class TestSetInline extends SrcOp { - public int bitsetWordSize; - public String varName; - public Bitset[] bitsets; + public final int bitsetWordSize; + public final String varName; + public final Bitset[] bitsets; public TestSetInline(OutputModelFactory factory, GrammarAST ast, IntervalSet set, int wordSize) { super(factory, ast); @@ -32,29 +33,48 @@ private static Bitset[] createBitsets(OutputModelFactory factory, IntervalSet set, int wordSize, boolean useZeroOffset) { - List bitsetList = new ArrayList(); + List bitsetList = new ArrayList<>(); + Target target = factory.getGenerator().getTarget(); + Bitset current = null; for (int ttype : set.toArray()) { - Bitset current = !bitsetList.isEmpty() ? bitsetList.get(bitsetList.size() - 1) : null; if (current == null || ttype > (current.shift + wordSize-1)) { - current = new Bitset(); + int shift; if (useZeroOffset && ttype >= 0 && ttype < wordSize-1) { - current.shift = 0; + shift = 0; } else { - current.shift = ttype; + shift = ttype; } - + current = new Bitset(shift); bitsetList.add(current); } - current.ttypes.add(factory.getGenerator().getTarget().getTokenTypeAsTargetLabel(factory.getGrammar(), ttype)); + current.addToken(ttype, target.getTokenTypeAsTargetLabel(factory.getGrammar(), ttype)); } return bitsetList.toArray(new Bitset[0]); } public static final class Bitset { - public int shift; - public final List ttypes = new ArrayList(); + public final int shift; + private final List tokens = new ArrayList<>(); + private long calculated; + + public Bitset(int shift) { + this.shift = shift; + } + + public void addToken(int type, String name) { + tokens.add(new TokenInfo(type, name)); + calculated |= 1L << (type - shift); + } + + public List getTokens() { + return tokens; + } + + public long getCalculated() { + return calculated; + } } } diff --git a/tool/src/org/antlr/v4/codegen/model/TokenInfo.java b/tool/src/org/antlr/v4/codegen/model/TokenInfo.java new file mode 100644 index 0000000000..0c64b08cb8 --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/model/TokenInfo.java @@ -0,0 +1,19 @@ +package org.antlr.v4.codegen.model; + +public class TokenInfo { + public final int type; + public final String name; + + public TokenInfo(int type, String name) { + this.type = type; + this.name = name; + } + + @Override + public String toString() { + return "TokenInfo{" + + "type=" + type + + ", name='" + name + '\'' + + '}'; + } +} diff --git a/tool/src/org/antlr/v4/codegen/target/GoTarget.java b/tool/src/org/antlr/v4/codegen/target/GoTarget.java index 8c102431b7..7366f27286 100644 --- a/tool/src/org/antlr/v4/codegen/target/GoTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/GoTarget.java @@ -95,11 +95,6 @@ private void gofmt(File fileName) { } } - @Override - public int getInlineTestSetWordSize() { - return 32; - } - public String getRecognizerFileName(boolean header) { CodeGenerator gen = getCodeGenerator(); Grammar g = gen.g;