Fix #229

FasterXML · Nov 17, 2015 · 4195e6e · 4195e6e
1 parent 329756d
commit 4195e6e
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 34 deletions.
diff --git a/release-notes/VERSION b/release-notes/VERSION
@@ -20,6 +20,7 @@ JSON library.
  (via `JsonParseException`, `JsonGenerationException`)
 #211: Typo of function name com.fasterxml.jackson.core.Version.isUknownVersion()
  (reported by timray@github)
+#229: Array element and field token spans include previous comma.
 - Implemented `ReaderBasedJsonParser.nextFieldName(SerializableString)`
   (to improved Afterburner performance over String/char[] sources)
 

diff --git a/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java
@@ -579,26 +579,20 @@ public final JsonToken nextToken() throws IOException
             close();
             return (_currToken = null);
         }
-
-        /* First, need to ensure we know the starting location of token
-         * after skipping leading white space
-         */
-        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
-        _tokenInputRow = _currInputRow;
-        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
-
-        // finally: clear any data retained so far
+        // clear any data retained so far
         _binaryValue = null;
 
         // Closing scope?
         if (i == INT_RBRACKET) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inArray()) {
                 _reportMismatchedEndMarker(i, '}');
             }
             _parsingContext = _parsingContext.getParent();
             return (_currToken = JsonToken.END_ARRAY);
         }
         if (i == INT_RCURLY) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inObject()) {
                 _reportMismatchedEndMarker(i, ']');
             }
@@ -610,6 +604,7 @@ public final JsonToken nextToken() throws IOException
         if (_parsingContext.expectComma()) {
             i = _skipComma(i);
         }
+        _updateLocationFromInputPtr();
 
         /* And should we now have a name? Always true for
          * Object contexts, since the intermediate 'expect-value'
@@ -700,6 +695,9 @@ private final JsonToken _nextAfterName()
         _nameCopied = false; // need to invalidate if it was copied
         JsonToken t = _nextToken;
         _nextToken = null;
+
+// !!! 16-Nov-2015, tatu: TODO: fix [databind#37], copy next location to current here
+
         // Also: may need to start new context?
         if (t == JsonToken.START_ARRAY) {
             _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
@@ -735,11 +733,10 @@ public boolean nextFieldName(SerializableString sstr) throws IOException
             _currToken = null;
             return false;
         }
-        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
-        _tokenInputRow = _currInputRow;
-        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
         _binaryValue = null;
+
         if (i == INT_RBRACKET) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inArray()) {
                 _reportMismatchedEndMarker(i, '}');
             }
@@ -748,6 +745,7 @@ public boolean nextFieldName(SerializableString sstr) throws IOException
             return false;
         }
         if (i == INT_RCURLY) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inObject()) {
                 _reportMismatchedEndMarker(i, ']');
             }
@@ -758,6 +756,7 @@ public boolean nextFieldName(SerializableString sstr) throws IOException
         if (_parsingContext.expectComma()) {
             i = _skipComma(i);
         }
+        _updateLocationFromInputPtr();
 
         if (!_parsingContext.inObject()) {
             _nextTokenNotInObject(i);
@@ -813,11 +812,9 @@ public String nextFieldName() throws IOException
             _currToken = null;
             return null;
         }
-        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
-        _tokenInputRow = _currInputRow;
-        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
         _binaryValue = null;
         if (i == INT_RBRACKET) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inArray()) {
                 _reportMismatchedEndMarker(i, '}');
             }
@@ -826,6 +823,7 @@ public String nextFieldName() throws IOException
             return null;
         }
         if (i == INT_RCURLY) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inObject()) {
                 _reportMismatchedEndMarker(i, ']');
             }
@@ -837,6 +835,7 @@ public String nextFieldName() throws IOException
             i = _skipComma(i);
         }
 
+        _updateLocationFromInputPtr();
         if (!_parsingContext.inObject()) {
             _nextTokenNotInObject(i);
             return null;
@@ -2646,6 +2645,20 @@ protected byte[] _decodeBase64(Base64Variant b64variant) throws IOException
         }
     }
 
+    /*
+    /**********************************************************
+    /* Internal methods, location updating (refactored in 2.7)
+    /**********************************************************
+     */
+
+    // @since 2.7
+    private final void _updateLocationFromInputPtr()
+    {
+        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
+        _tokenInputRow = _currInputRow;
+        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
+    }
+
     /*
     /**********************************************************
     /* Error reporting

diff --git a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java
@@ -697,25 +697,20 @@ public JsonToken nextToken() throws IOException
             close();
             return (_currToken = null);
         }
-
-        // First, need to ensure we know the starting location of token
-        // after skipping leading white space
-        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
-        _tokenInputRow = _currInputRow;
-        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
-
-        // finally: clear any data retained so far
+        // clear any data retained so far
         _binaryValue = null;
 
         // Closing scope?
         if (i == INT_RBRACKET) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inArray()) {
                 _reportMismatchedEndMarker(i, '}');
             }
             _parsingContext = _parsingContext.getParent();
             return (_currToken = JsonToken.END_ARRAY);
         }
         if (i == INT_RCURLY) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inObject()) {
                 _reportMismatchedEndMarker(i, ']');
             }
@@ -730,6 +725,7 @@ public JsonToken nextToken() throws IOException
             }
             i = _skipWS();
         }
+        _updateLocationFromInputPtr();
 
         /* And should we now have a name? Always true for
          * Object contexts, since the intermediate 'expect-value'
@@ -848,6 +844,9 @@ private final JsonToken _nextAfterName()
         _nameCopied = false; // need to invalidate if it was copied
         JsonToken t = _nextToken;
         _nextToken = null;
+
+ // !!! 16-Nov-2015, tatu: TODO: fix [databind#37], copy next location to current here
+
         // Also: may need to start new context?
         if (t == JsonToken.START_ARRAY) {
             _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol);
@@ -867,7 +866,6 @@ private final JsonToken _nextAfterName()
     public boolean nextFieldName(SerializableString str) throws IOException
     {
         // // // Note: most of code below is copied from nextToken()
-
         _numTypesValid = NR_UNKNOWN;
         if (_currToken == JsonToken.FIELD_NAME) { // can't have name right after name
             _nextAfterName();
@@ -882,15 +880,11 @@ public boolean nextFieldName(SerializableString str) throws IOException
             _currToken = null;
             return false;
         }
-        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
-        _tokenInputRow = _currInputRow;
-        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
-
-        // finally: clear any data retained so far
         _binaryValue = null;
 
         // Closing scope?
         if (i == INT_RBRACKET) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inArray()) {
                 _reportMismatchedEndMarker(i, '}');
             }
@@ -899,6 +893,7 @@ public boolean nextFieldName(SerializableString str) throws IOException
             return false;
         }
         if (i == INT_RCURLY) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inObject()) {
                 _reportMismatchedEndMarker(i, ']');
             }
@@ -915,6 +910,7 @@ public boolean nextFieldName(SerializableString str) throws IOException
             i = _skipWS();
         }
 
+        _updateLocationFromInputPtr();
         if (!_parsingContext.inObject()) {
             _nextTokenNotInObject(i);
             return false;
@@ -971,13 +967,10 @@ public String nextFieldName() throws IOException
             _currToken = null;
             return null;
         }
-        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
-        _tokenInputRow = _currInputRow;
-        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
-
         _binaryValue = null;
 
         if (i == INT_RBRACKET) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inArray()) {
                 _reportMismatchedEndMarker(i, '}');
             }
@@ -986,6 +979,7 @@ public String nextFieldName() throws IOException
             return null;
         }
         if (i == INT_RCURLY) {
+            _updateLocationFromInputPtr();
             if (!_parsingContext.inObject()) {
                 _reportMismatchedEndMarker(i, ']');
             }
@@ -1001,7 +995,7 @@ public String nextFieldName() throws IOException
             }
             i = _skipWS();
         }
-
+        _updateLocationFromInputPtr();
         if (!_parsingContext.inObject()) {
             _nextTokenNotInObject(i);
             return null;
@@ -3608,6 +3602,20 @@ protected final byte[] _decodeBase64(Base64Variant b64variant) throws IOExceptio
         }
     }
 
+    /*
+    /**********************************************************
+    /* Internal methods, location updating (refactored in 2.7)
+    /**********************************************************
+     */
+
+    // @since 2.7
+    private final void _updateLocationFromInputPtr()
+    {
+        _tokenInputTotal = _currInputProcessed + _inputPtr - 1;
+        _tokenInputRow = _currInputRow;
+        _tokenInputCol = _inputPtr - _currInputRowStart - 1;
+    }
+
     /*
     /**********************************************************
     /* Internal methods, other

diff --git a/src/test/java/com/fasterxml/jackson/failing/LocationInArrayTest.java b/src/test/java/com/fasterxml/jackson/failing/LocationInArrayTest.java
@@ -39,6 +39,15 @@ private void _testOffsetInArrays(boolean useBytes) throws Exception
         _assertLocation(useBytes, p.getTokenLocation(), 7L, 1, 8);
         assertEquals(251, p.getIntValue()); // just to ensure read proceeds to end
         _assertLocation(useBytes, p.getCurrentLocation(), 10L, 1, 11);
+
+        assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+        _assertLocation(useBytes, p.getTokenLocation(), 15L, 2, 4);
+        assertEquals(3, p.getIntValue());
+        _assertLocation(useBytes, p.getCurrentLocation(), 16L, 2, 5);
+
+        assertToken(JsonToken.END_ARRAY, p.nextToken());
+        _assertLocation(useBytes, p.getTokenLocation(), 18L, 2, 7);
+        _assertLocation(useBytes, p.getCurrentLocation(), 19L, 2, 8);
 
         p.close();
     }