fix: slurp underscores when processing string literals (#1)

Author: disintegrator

pkg/jsonpath/token.go

@@ -485,7 +485,7 @@ func (t *Tokenizer) Tokenize() Tokens {
 			t.scanString(rune(ch))
 		case isDigit(ch):
 			t.scanNumber()
-		case isLetter(ch):
+		case isLiteralChar(ch):
 			t.scanLiteral()
 		default:
 			t.addToken(ILLEGAL, 1, string(ch))
@@ -546,7 +546,7 @@ func (t *Tokenizer) scanNumber() {
 func (t *Tokenizer) scanLiteral() {
 	start := t.pos
 	for i := start; i < len(t.input); i++ {
-		if !isLetter(t.input[i]) {
+		if !isLiteralChar(t.input[i]) {
 			literal := t.input[start:i]
 			switch literal {
 			case "true", "false":
@@ -606,8 +606,8 @@ func isDigit(ch byte) bool {
 	return '0' <= ch && ch <= '9'
 }
 
-func isLetter(ch byte) bool {
-	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z'
+func isLiteralChar(ch byte) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
 }
 
 func isSpace(ch byte) bool {

pkg/jsonpath/token_test.go

@@ -252,6 +252,19 @@ func TestTokenizer(t *testing.T) {
 				{Token: BRACKET_RIGHT, Line: 1, Column: 13, Literal: "", Len: 1},
 			},
 		},
+		{
+			name:  "Underscore is string literal character",
+			input: "$.pagination._.next_results_cursor",
+			expected: []TokenInfo{
+				{Token: ROOT, Line: 1, Column: 0, Literal: "", Len: 1},
+				{Token: CHILD, Line: 1, Column: 1, Literal: "", Len: 1},
+				{Token: STRING_LITERAL, Line: 1, Column: 2, Literal: "pagination", Len: 10},
+				{Token: CHILD, Line: 1, Column: 12, Literal: "", Len: 1},
+				{Token: STRING_LITERAL, Line: 1, Column: 13, Literal: "_", Len: 1},
+				{Token: CHILD, Line: 1, Column: 14, Literal: "", Len: 1},
+				{Token: STRING_LITERAL, Line: 1, Column: 15, Literal: "next_results_cursor", Len: 19},
+			},
+		},
 		//{
 		//	name:  "Filter regular expression (illegal right now)",
 		//	input: "$[?(@.child=~/.*/)]",