Skip to content

Commit 59d60fd

Browse files
bbeaudreaultapurtell
bbeaudreault
authored and
apurtell
committed
HBASE-26713 Default to LATEST_TIMESTAMP if no timestamp sent along on Increment/Append (#4075)
Signed-off-by: Andrew Purtell <[email protected]> Signed-off-by: Viraj Jasani <[email protected]>
1 parent f7f1a48 commit 59d60fd

File tree

4 files changed

+164
-72
lines changed

4 files changed

+164
-72
lines changed

‎hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -627,10 +627,7 @@ public static Delete toDelete(final MutationProto proto, final CellScanner cellS
627627
if (qv.hasQualifier()) {
628628
qualifier = qv.getQualifier().toByteArray();
629629
}
630-
long ts = HConstants.LATEST_TIMESTAMP;
631-
if (qv.hasTimestamp()) {
632-
ts = qv.getTimestamp();
633-
}
630+
long ts = cellTimestampOrLatest(qv);
634631
if (deleteType == DeleteType.DELETE_ONE_VERSION) {
635632
delete.addColumn(family, qualifier, ts);
636633
} else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
@@ -694,7 +691,7 @@ private static <T extends Mutation> T toDelta(Function<Bytes, T> supplier, Consu
694691
if (qv.hasTags()) {
695692
tags = qv.getTags().toByteArray();
696693
}
697-
consumer.accept(mutation, CellUtil.createCell(mutation.getRow(), family, qualifier, qv.getTimestamp(),
694+
consumer.accept(mutation, CellUtil.createCell(mutation.getRow(), family, qualifier, cellTimestampOrLatest(qv),
698695
KeyValue.Type.Put, value, tags));
699696
}
700697
}
@@ -706,6 +703,14 @@ private static <T extends Mutation> T toDelta(Function<Bytes, T> supplier, Consu
706703
return mutation;
707704
}
708705

706+
private static long cellTimestampOrLatest(QualifierValue cell) {
707+
if (cell.hasTimestamp()) {
708+
return cell.getTimestamp();
709+
} else {
710+
return HConstants.LATEST_TIMESTAMP;
711+
}
712+
}
713+
709714
/**
710715
* Convert a protocol buffer Mutate to an Append
711716
* @param cellScanner

‎hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -787,10 +787,7 @@ public static Delete toDelete(final MutationProto proto, final CellScanner cellS
787787
if (qv.hasQualifier()) {
788788
qualifier = qv.getQualifier().toByteArray();
789789
}
790-
long ts = HConstants.LATEST_TIMESTAMP;
791-
if (qv.hasTimestamp()) {
792-
ts = qv.getTimestamp();
793-
}
790+
long ts = cellTimestampOrLatest(qv);
794791
if (deleteType == DeleteType.DELETE_ONE_VERSION) {
795792
delete.addColumn(family, qualifier, ts);
796793
} else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
@@ -857,7 +854,7 @@ private static <T extends Mutation> T toDelta(Function<Bytes, T> supplier, Consu
857854
.setRow(mutation.getRow())
858855
.setFamily(family)
859856
.setQualifier(qualifier)
860-
.setTimestamp(qv.getTimestamp())
857+
.setTimestamp(cellTimestampOrLatest(qv))
861858
.setType(KeyValue.Type.Put.getCode())
862859
.setValue(value)
863860
.setTags(tags)
@@ -872,6 +869,14 @@ private static <T extends Mutation> T toDelta(Function<Bytes, T> supplier, Consu
872869
return mutation;
873870
}
874871

872+
private static long cellTimestampOrLatest(QualifierValue cell) {
873+
if (cell.hasTimestamp()) {
874+
return cell.getTimestamp();
875+
} else {
876+
return HConstants.LATEST_TIMESTAMP;
877+
}
878+
}
879+
875880
/**
876881
* Convert a protocol buffer Mutate to an Append
877882
* @param cellScanner

‎hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java‎

Lines changed: 73 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.hadoop.hbase.ExtendedCellBuilder;
3434
import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
3535
import org.apache.hadoop.hbase.HBaseClassTestRule;
36+
import org.apache.hadoop.hbase.HConstants;
3637
import org.apache.hadoop.hbase.KeyValue;
3738
import org.apache.hadoop.hbase.PrivateCellUtil;
3839
import org.apache.hadoop.hbase.Tag;
@@ -294,7 +295,22 @@ public void testToCell() {
294295
*/
295296
@Test
296297
public void testIncrement() throws IOException {
297-
long timeStamp = 111111;
298+
299+
MutationProto proto = getIncrementMutation(111111L);
300+
// default fields
301+
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
302+
303+
// set the default value for equal comparison
304+
MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
305+
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
306+
307+
Increment increment = ProtobufUtil.toIncrement(proto, null);
308+
mutateBuilder.setTimestamp(increment.getTimestamp());
309+
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
310+
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
311+
}
312+
313+
private MutationProto getIncrementMutation(Long timestamp) {
298314
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
299315
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
300316
mutateBuilder.setMutateType(MutationProto.MutationType.INCREMENT);
@@ -303,66 +319,93 @@ public void testIncrement() throws IOException {
303319
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
304320
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
305321
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(11L)));
306-
qualifierBuilder.setTimestamp(timeStamp);
322+
323+
if (timestamp != null) {
324+
qualifierBuilder.setTimestamp(timestamp);
325+
}
326+
307327
valueBuilder.addQualifierValue(qualifierBuilder.build());
308328
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
309329
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(22L)));
310330
valueBuilder.addQualifierValue(qualifierBuilder.build());
311331
mutateBuilder.addColumnValue(valueBuilder.build());
312332

313-
MutationProto proto = mutateBuilder.build();
333+
return mutateBuilder.build();
334+
}
335+
336+
/**
337+
* Older clients may not send along a timestamp in the MutationProto. Check that we
338+
* default correctly.
339+
*/
340+
@Test
341+
public void testIncrementNoTimestamp() throws IOException {
342+
MutationProto mutation = getIncrementMutation(null);
343+
Increment increment = ProtobufUtil.toIncrement(mutation, null);
344+
assertEquals(HConstants.LATEST_TIMESTAMP, increment.getTimestamp());
345+
increment.getFamilyCellMap().values()
346+
.forEach(cells ->
347+
cells.forEach(cell ->
348+
assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
349+
}
350+
351+
/**
352+
* Test Append Mutate conversions.
353+
*
354+
* @throws IOException if converting to an {@link Append} fails
355+
*/
356+
@Test
357+
public void testAppend() throws IOException {
358+
MutationProto proto = getAppendMutation(111111L);
314359
// default fields
315360
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
316361

317362
// set the default value for equal comparison
318-
mutateBuilder = MutationProto.newBuilder(proto);
363+
MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
319364
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
320365

321-
Increment increment = ProtobufUtil.toIncrement(proto, null);
322-
mutateBuilder.setTimestamp(increment.getTimestamp());
323-
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
324-
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
366+
Append append = ProtobufUtil.toAppend(proto, null);
367+
368+
// append always use the latest timestamp,
369+
// reset the timestamp to the original mutate
370+
mutateBuilder.setTimestamp(append.getTimestamp());
371+
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
372+
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
325373
}
326374

327375
/**
328-
* Test Append Mutate conversions.
329-
*
330-
* @throws IOException if converting to an {@link Append} fails
376+
* Older clients may not send along a timestamp in the MutationProto. Check that we
377+
* default correctly.
331378
*/
332379
@Test
333-
public void testAppend() throws IOException {
334-
long timeStamp = 111111;
380+
public void testAppendNoTimestamp() throws IOException {
381+
MutationProto mutation = getAppendMutation(null);
382+
Append append = ProtobufUtil.toAppend(mutation, null);
383+
assertEquals(HConstants.LATEST_TIMESTAMP, append.getTimestamp());
384+
append.getFamilyCellMap().values().forEach(cells -> cells.forEach(cell -> assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
385+
}
386+
387+
private MutationProto getAppendMutation(Long timestamp) {
335388
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
336389
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
337390
mutateBuilder.setMutateType(MutationType.APPEND);
338-
mutateBuilder.setTimestamp(timeStamp);
391+
if (timestamp != null) {
392+
mutateBuilder.setTimestamp(timestamp);
393+
}
339394
ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
340395
valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
341396
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
342397
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
343398
qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
344-
qualifierBuilder.setTimestamp(timeStamp);
399+
if (timestamp != null) {
400+
qualifierBuilder.setTimestamp(timestamp);
401+
}
345402
valueBuilder.addQualifierValue(qualifierBuilder.build());
346403
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
347404
qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
348405
valueBuilder.addQualifierValue(qualifierBuilder.build());
349406
mutateBuilder.addColumnValue(valueBuilder.build());
350407

351-
MutationProto proto = mutateBuilder.build();
352-
// default fields
353-
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
354-
355-
// set the default value for equal comparison
356-
mutateBuilder = MutationProto.newBuilder(proto);
357-
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
358-
359-
Append append = ProtobufUtil.toAppend(proto, null);
360-
361-
// append always use the latest timestamp,
362-
// reset the timestamp to the original mutate
363-
mutateBuilder.setTimestamp(append.getTimestamp());
364-
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
365-
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
408+
return mutateBuilder.build();
366409
}
367410

368411
private static ProcedureProtos.Procedure.Builder createProcedureBuilder(long procId) {

‎hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java‎

Lines changed: 71 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -122,38 +122,57 @@ public void testGet() throws IOException {
122122
*/
123123
@Test
124124
public void testAppend() throws IOException {
125-
long timeStamp = 111111;
125+
MutationProto proto = getAppendMutation(111111L);
126+
// default fields
127+
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
128+
129+
// set the default value for equal comparison
130+
MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
131+
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
132+
133+
Append append = ProtobufUtil.toAppend(proto, null);
134+
135+
// append always use the latest timestamp,
136+
// reset the timestamp to the original mutate
137+
mutateBuilder.setTimestamp(append.getTimestamp());
138+
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
139+
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
140+
}
141+
142+
/**
143+
* Older clients may not send along a timestamp in the MutationProto. Check that we
144+
* default correctly.
145+
*/
146+
@Test
147+
public void testAppendNoTimestamp() throws IOException {
148+
MutationProto mutation = getAppendMutation(null);
149+
Append append = ProtobufUtil.toAppend(mutation, null);
150+
assertEquals(HConstants.LATEST_TIMESTAMP, append.getTimestamp());
151+
append.getFamilyCellMap().values().forEach(cells -> cells.forEach(cell -> assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
152+
}
153+
154+
private MutationProto getAppendMutation(Long timestamp) {
126155
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
127156
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
128157
mutateBuilder.setMutateType(MutationType.APPEND);
129-
mutateBuilder.setTimestamp(timeStamp);
158+
if (timestamp != null) {
159+
mutateBuilder.setTimestamp(timestamp);
160+
}
130161
ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
131162
valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
132163
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
133164
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
134165
qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
135-
qualifierBuilder.setTimestamp(timeStamp);
166+
if (timestamp != null) {
167+
qualifierBuilder.setTimestamp(timestamp);
168+
}
136169
valueBuilder.addQualifierValue(qualifierBuilder.build());
137170
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
138171
qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
139172
valueBuilder.addQualifierValue(qualifierBuilder.build());
140173
mutateBuilder.addColumnValue(valueBuilder.build());
141174

142-
MutationProto proto = mutateBuilder.build();
143-
// default fields
144-
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
145-
146-
// set the default value for equal comparison
147-
mutateBuilder = MutationProto.newBuilder(proto);
148-
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
149-
150-
Append append = ProtobufUtil.toAppend(proto, null);
151-
152-
// append always use the latest timestamp,
153-
// reset the timestamp to the original mutate
154-
mutateBuilder.setTimestamp(append.getTimestamp());
155-
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
156-
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
175+
return mutateBuilder.build();
157176
}
158177

159178
/**
@@ -210,7 +229,36 @@ public void testDelete() throws IOException {
210229
*/
211230
@Test
212231
public void testIncrement() throws IOException {
213-
long timeStamp = 111111;
232+
MutationProto proto = getIncrementMutation(111111L);
233+
// default fields
234+
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
235+
236+
// set the default value for equal comparison
237+
MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
238+
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
239+
240+
Increment increment = ProtobufUtil.toIncrement(proto, null);
241+
mutateBuilder.setTimestamp(increment.getTimestamp());
242+
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
243+
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
244+
}
245+
246+
/**
247+
* Older clients may not send along a timestamp in the MutationProto. Check that we
248+
* default correctly.
249+
*/
250+
@Test
251+
public void testIncrementNoTimestamp() throws IOException {
252+
MutationProto mutation = getIncrementMutation(null);
253+
Increment increment = ProtobufUtil.toIncrement(mutation, null);
254+
assertEquals(HConstants.LATEST_TIMESTAMP, increment.getTimestamp());
255+
increment.getFamilyCellMap().values()
256+
.forEach(cells ->
257+
cells.forEach(cell ->
258+
assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
259+
}
260+
261+
private MutationProto getIncrementMutation(Long timestamp) {
214262
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
215263
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
216264
mutateBuilder.setMutateType(MutationType.INCREMENT);
@@ -219,25 +267,16 @@ public void testIncrement() throws IOException {
219267
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
220268
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
221269
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(11L)));
222-
qualifierBuilder.setTimestamp(timeStamp);
270+
if (timestamp != null) {
271+
qualifierBuilder.setTimestamp(timestamp);
272+
}
223273
valueBuilder.addQualifierValue(qualifierBuilder.build());
224274
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
225275
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(22L)));
226276
valueBuilder.addQualifierValue(qualifierBuilder.build());
227277
mutateBuilder.addColumnValue(valueBuilder.build());
228278

229-
MutationProto proto = mutateBuilder.build();
230-
// default fields
231-
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
232-
233-
// set the default value for equal comparison
234-
mutateBuilder = MutationProto.newBuilder(proto);
235-
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
236-
237-
Increment increment = ProtobufUtil.toIncrement(proto, null);
238-
mutateBuilder.setTimestamp(increment.getTimestamp());
239-
mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
240-
assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
279+
return mutateBuilder.build();
241280
}
242281

243282
/**

0 commit comments

Comments
 (0)