From 4195e6ebd59b9dac4065a1eb090d4343c2f61965 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Mon, 16 Nov 2015 22:58:39 -0800 Subject: [PATCH] Fix #229 --- release-notes/VERSION | 1 + .../core/json/ReaderBasedJsonParser.java | 43 +++++++++++------ .../core/json/UTF8StreamJsonParser.java | 46 +++++++++++-------- .../jackson/failing/LocationInArrayTest.java | 9 ++++ 4 files changed, 65 insertions(+), 34 deletions(-) diff --git a/release-notes/VERSION b/release-notes/VERSION index ef127806ba..9e59eafd7a 100644 --- a/release-notes/VERSION +++ b/release-notes/VERSION @@ -20,6 +20,7 @@ JSON library. (via `JsonParseException`, `JsonGenerationException`) #211: Typo of function name com.fasterxml.jackson.core.Version.isUknownVersion() (reported by timray@github) +#229: Array element and field token spans include previous comma. - Implemented `ReaderBasedJsonParser.nextFieldName(SerializableString)` (to improved Afterburner performance over String/char[] sources) diff --git a/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java index b3fdfb8b96..5c6451693d 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java @@ -579,19 +579,12 @@ public final JsonToken nextToken() throws IOException close(); return (_currToken = null); } - - /* First, need to ensure we know the starting location of token - * after skipping leading white space - */ - _tokenInputTotal = _currInputProcessed + _inputPtr - 1; - _tokenInputRow = _currInputRow; - _tokenInputCol = _inputPtr - _currInputRowStart - 1; - - // finally: clear any data retained so far + // clear any data retained so far _binaryValue = null; // Closing scope? if (i == INT_RBRACKET) { + _updateLocationFromInputPtr(); if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } @@ -599,6 +592,7 @@ public final JsonToken nextToken() throws IOException return (_currToken = JsonToken.END_ARRAY); } if (i == INT_RCURLY) { + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _reportMismatchedEndMarker(i, ']'); } @@ -610,6 +604,7 @@ public final JsonToken nextToken() throws IOException if (_parsingContext.expectComma()) { i = _skipComma(i); } + _updateLocationFromInputPtr(); /* And should we now have a name? Always true for * Object contexts, since the intermediate 'expect-value' @@ -700,6 +695,9 @@ private final JsonToken _nextAfterName() _nameCopied = false; // need to invalidate if it was copied JsonToken t = _nextToken; _nextToken = null; + +// !!! 16-Nov-2015, tatu: TODO: fix [databind#37], copy next location to current here + // Also: may need to start new context? if (t == JsonToken.START_ARRAY) { _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol); @@ -735,11 +733,10 @@ public boolean nextFieldName(SerializableString sstr) throws IOException _currToken = null; return false; } - _tokenInputTotal = _currInputProcessed + _inputPtr - 1; - _tokenInputRow = _currInputRow; - _tokenInputCol = _inputPtr - _currInputRowStart - 1; _binaryValue = null; + if (i == INT_RBRACKET) { + _updateLocationFromInputPtr(); if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } @@ -748,6 +745,7 @@ public boolean nextFieldName(SerializableString sstr) throws IOException return false; } if (i == INT_RCURLY) { + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _reportMismatchedEndMarker(i, ']'); } @@ -758,6 +756,7 @@ public boolean nextFieldName(SerializableString sstr) throws IOException if (_parsingContext.expectComma()) { i = _skipComma(i); } + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _nextTokenNotInObject(i); @@ -813,11 +812,9 @@ public String nextFieldName() throws IOException _currToken = null; return null; } - _tokenInputTotal = _currInputProcessed + _inputPtr - 1; - _tokenInputRow = _currInputRow; - _tokenInputCol = _inputPtr - _currInputRowStart - 1; _binaryValue = null; if (i == INT_RBRACKET) { + _updateLocationFromInputPtr(); if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } @@ -826,6 +823,7 @@ public String nextFieldName() throws IOException return null; } if (i == INT_RCURLY) { + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _reportMismatchedEndMarker(i, ']'); } @@ -837,6 +835,7 @@ public String nextFieldName() throws IOException i = _skipComma(i); } + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _nextTokenNotInObject(i); return null; @@ -2646,6 +2645,20 @@ protected byte[] _decodeBase64(Base64Variant b64variant) throws IOException } } + /* + /********************************************************** + /* Internal methods, location updating (refactored in 2.7) + /********************************************************** + */ + + // @since 2.7 + private final void _updateLocationFromInputPtr() + { + _tokenInputTotal = _currInputProcessed + _inputPtr - 1; + _tokenInputRow = _currInputRow; + _tokenInputCol = _inputPtr - _currInputRowStart - 1; + } + /* /********************************************************** /* Error reporting diff --git a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java index 7bea9d21cb..65aa9aee7e 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java @@ -697,18 +697,12 @@ public JsonToken nextToken() throws IOException close(); return (_currToken = null); } - - // First, need to ensure we know the starting location of token - // after skipping leading white space - _tokenInputTotal = _currInputProcessed + _inputPtr - 1; - _tokenInputRow = _currInputRow; - _tokenInputCol = _inputPtr - _currInputRowStart - 1; - - // finally: clear any data retained so far + // clear any data retained so far _binaryValue = null; // Closing scope? if (i == INT_RBRACKET) { + _updateLocationFromInputPtr(); if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } @@ -716,6 +710,7 @@ public JsonToken nextToken() throws IOException return (_currToken = JsonToken.END_ARRAY); } if (i == INT_RCURLY) { + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _reportMismatchedEndMarker(i, ']'); } @@ -730,6 +725,7 @@ public JsonToken nextToken() throws IOException } i = _skipWS(); } + _updateLocationFromInputPtr(); /* And should we now have a name? Always true for * Object contexts, since the intermediate 'expect-value' @@ -848,6 +844,9 @@ private final JsonToken _nextAfterName() _nameCopied = false; // need to invalidate if it was copied JsonToken t = _nextToken; _nextToken = null; + + // !!! 16-Nov-2015, tatu: TODO: fix [databind#37], copy next location to current here + // Also: may need to start new context? if (t == JsonToken.START_ARRAY) { _parsingContext = _parsingContext.createChildArrayContext(_tokenInputRow, _tokenInputCol); @@ -867,7 +866,6 @@ private final JsonToken _nextAfterName() public boolean nextFieldName(SerializableString str) throws IOException { // // // Note: most of code below is copied from nextToken() - _numTypesValid = NR_UNKNOWN; if (_currToken == JsonToken.FIELD_NAME) { // can't have name right after name _nextAfterName(); @@ -882,15 +880,11 @@ public boolean nextFieldName(SerializableString str) throws IOException _currToken = null; return false; } - _tokenInputTotal = _currInputProcessed + _inputPtr - 1; - _tokenInputRow = _currInputRow; - _tokenInputCol = _inputPtr - _currInputRowStart - 1; - - // finally: clear any data retained so far _binaryValue = null; // Closing scope? if (i == INT_RBRACKET) { + _updateLocationFromInputPtr(); if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } @@ -899,6 +893,7 @@ public boolean nextFieldName(SerializableString str) throws IOException return false; } if (i == INT_RCURLY) { + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _reportMismatchedEndMarker(i, ']'); } @@ -915,6 +910,7 @@ public boolean nextFieldName(SerializableString str) throws IOException i = _skipWS(); } + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _nextTokenNotInObject(i); return false; @@ -971,13 +967,10 @@ public String nextFieldName() throws IOException _currToken = null; return null; } - _tokenInputTotal = _currInputProcessed + _inputPtr - 1; - _tokenInputRow = _currInputRow; - _tokenInputCol = _inputPtr - _currInputRowStart - 1; - _binaryValue = null; if (i == INT_RBRACKET) { + _updateLocationFromInputPtr(); if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } @@ -986,6 +979,7 @@ public String nextFieldName() throws IOException return null; } if (i == INT_RCURLY) { + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _reportMismatchedEndMarker(i, ']'); } @@ -1001,7 +995,7 @@ public String nextFieldName() throws IOException } i = _skipWS(); } - + _updateLocationFromInputPtr(); if (!_parsingContext.inObject()) { _nextTokenNotInObject(i); return null; @@ -3608,6 +3602,20 @@ protected final byte[] _decodeBase64(Base64Variant b64variant) throws IOExceptio } } + /* + /********************************************************** + /* Internal methods, location updating (refactored in 2.7) + /********************************************************** + */ + + // @since 2.7 + private final void _updateLocationFromInputPtr() + { + _tokenInputTotal = _currInputProcessed + _inputPtr - 1; + _tokenInputRow = _currInputRow; + _tokenInputCol = _inputPtr - _currInputRowStart - 1; + } + /* /********************************************************** /* Internal methods, other diff --git a/src/test/java/com/fasterxml/jackson/failing/LocationInArrayTest.java b/src/test/java/com/fasterxml/jackson/failing/LocationInArrayTest.java index f5f0967414..2638f003a6 100644 --- a/src/test/java/com/fasterxml/jackson/failing/LocationInArrayTest.java +++ b/src/test/java/com/fasterxml/jackson/failing/LocationInArrayTest.java @@ -39,6 +39,15 @@ private void _testOffsetInArrays(boolean useBytes) throws Exception _assertLocation(useBytes, p.getTokenLocation(), 7L, 1, 8); assertEquals(251, p.getIntValue()); // just to ensure read proceeds to end _assertLocation(useBytes, p.getCurrentLocation(), 10L, 1, 11); + + assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken()); + _assertLocation(useBytes, p.getTokenLocation(), 15L, 2, 4); + assertEquals(3, p.getIntValue()); + _assertLocation(useBytes, p.getCurrentLocation(), 16L, 2, 5); + + assertToken(JsonToken.END_ARRAY, p.nextToken()); + _assertLocation(useBytes, p.getTokenLocation(), 18L, 2, 7); + _assertLocation(useBytes, p.getCurrentLocation(), 19L, 2, 8); p.close(); }