View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.protobuf;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertNotNull;
23  
24  import com.google.protobuf.ByteString;
25  import java.io.IOException;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.ServerName;
34  import org.apache.hadoop.hbase.Tag;
35  import org.apache.hadoop.hbase.client.Append;
36  import org.apache.hadoop.hbase.client.Delete;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.Increment;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.master.RegionState;
41  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
42  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
43  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
44  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
47  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
48  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
50  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.MetaRegionServer;
51  import org.apache.hadoop.hbase.testclassification.SmallTests;
52  import org.apache.hadoop.hbase.util.ByteStringer;
53  import org.apache.hadoop.hbase.util.Bytes;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  /**
58   * Class to test ProtobufUtil.
59   */
60  @Category(SmallTests.class)
61  public class TestProtobufUtil {
62    private static final String TAG_STR = "tag-1";
63    private static final byte TAG_TYPE = (byte)10;
64  
65    @Test
66    public void testException() throws IOException {
67      NameBytesPair.Builder builder = NameBytesPair.newBuilder();
68      final String omg = "OMG!!!";
69      builder.setName("java.io.IOException");
70      builder.setValue(ByteStringer.wrap(Bytes.toBytes(omg)));
71      Throwable t = ProtobufUtil.toException(builder.build());
72      assertEquals(omg, t.getMessage());
73      builder.clear();
74      builder.setName("org.apache.hadoop.ipc.RemoteException");
75      builder.setValue(ByteStringer.wrap(Bytes.toBytes(omg)));
76      t = ProtobufUtil.toException(builder.build());
77      assertEquals(omg, t.getMessage());
78    }
79  
80    /**
81     * Test basic Get conversions.
82     *
83     * @throws IOException
84     */
85    @Test
86    public void testGet() throws IOException {
87      ClientProtos.Get.Builder getBuilder = ClientProtos.Get.newBuilder();
88      getBuilder.setRow(ByteString.copyFromUtf8("row"));
89      Column.Builder columnBuilder = Column.newBuilder();
90      columnBuilder.setFamily(ByteString.copyFromUtf8("f1"));
91      columnBuilder.addQualifier(ByteString.copyFromUtf8("c1"));
92      columnBuilder.addQualifier(ByteString.copyFromUtf8("c2"));
93      getBuilder.addColumn(columnBuilder.build());
94  
95      columnBuilder.clear();
96      columnBuilder.setFamily(ByteString.copyFromUtf8("f2"));
97      getBuilder.addColumn(columnBuilder.build());
98      getBuilder.setLoadColumnFamiliesOnDemand(true);
99  
100     ClientProtos.Get proto = getBuilder.build();
101     // default fields
102     assertEquals(1, proto.getMaxVersions());
103     assertEquals(true, proto.getCacheBlocks());
104 
105     // set the default value for equal comparison
106     getBuilder = ClientProtos.Get.newBuilder(proto);
107     getBuilder.setMaxVersions(1);
108     getBuilder.setCacheBlocks(true);
109 
110     Get get = ProtobufUtil.toGet(proto);
111     assertEquals(getBuilder.build(), ProtobufUtil.toGet(get));
112   }
113 
114   /**
115    * Test Append Mutate conversions.
116    *
117    * @throws IOException
118    */
119   @Test
120   public void testAppend() throws IOException {
121     long timeStamp = 111111;
122     MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
123     mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
124     mutateBuilder.setMutateType(MutationType.APPEND);
125     mutateBuilder.setTimestamp(timeStamp);
126     ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
127     valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
128     QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
129     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
130     qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
131     qualifierBuilder.setTimestamp(timeStamp);
132     valueBuilder.addQualifierValue(qualifierBuilder.build());
133     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
134     qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
135     valueBuilder.addQualifierValue(qualifierBuilder.build());
136     qualifierBuilder.setTimestamp(timeStamp);
137     mutateBuilder.addColumnValue(valueBuilder.build());
138 
139     MutationProto proto = mutateBuilder.build();
140     // default fields
141     assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
142 
143     // set the default value for equal comparison
144     mutateBuilder = MutationProto.newBuilder(proto);
145     mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
146 
147     Append append = ProtobufUtil.toAppend(proto, null);
148 
149     // append always use the latest timestamp,
150     // reset the timestamp to the original mutate
151     mutateBuilder.setTimestamp(append.getTimeStamp());
152     assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
153   }
154 
155   /**
156    * Test Delete Mutate conversions.
157    *
158    * @throws IOException
159    */
160   @Test
161   public void testDelete() throws IOException {
162     MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
163     mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
164     mutateBuilder.setMutateType(MutationType.DELETE);
165     mutateBuilder.setTimestamp(111111);
166     ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
167     valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
168     QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
169     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
170     qualifierBuilder.setDeleteType(DeleteType.DELETE_ONE_VERSION);
171     qualifierBuilder.setTimestamp(111222);
172     valueBuilder.addQualifierValue(qualifierBuilder.build());
173     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
174     qualifierBuilder.setDeleteType(DeleteType.DELETE_MULTIPLE_VERSIONS);
175     qualifierBuilder.setTimestamp(111333);
176     valueBuilder.addQualifierValue(qualifierBuilder.build());
177     mutateBuilder.addColumnValue(valueBuilder.build());
178 
179     MutationProto proto = mutateBuilder.build();
180     // default fields
181     assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
182 
183     // set the default value for equal comparison
184     mutateBuilder = MutationProto.newBuilder(proto);
185     mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
186 
187     Delete delete = ProtobufUtil.toDelete(proto);
188 
189     // delete always have empty value,
190     // add empty value to the original mutate
191     for (ColumnValue.Builder column:
192         mutateBuilder.getColumnValueBuilderList()) {
193       for (QualifierValue.Builder qualifier:
194           column.getQualifierValueBuilderList()) {
195         qualifier.setValue(ByteString.EMPTY);
196       }
197     }
198     assertEquals(mutateBuilder.build(),
199       ProtobufUtil.toMutation(MutationType.DELETE, delete));
200   }
201 
202   /**
203    * Test Increment Mutate conversions.
204    *
205    * @throws IOException
206    */
207   @Test
208   public void testIncrement() throws IOException {
209     MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
210     mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
211     mutateBuilder.setMutateType(MutationType.INCREMENT);
212     ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
213     valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
214     QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
215     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
216     qualifierBuilder.setValue(ByteStringer.wrap(Bytes.toBytes(11L)));
217     valueBuilder.addQualifierValue(qualifierBuilder.build());
218     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
219     qualifierBuilder.setValue(ByteStringer.wrap(Bytes.toBytes(22L)));
220     valueBuilder.addQualifierValue(qualifierBuilder.build());
221     mutateBuilder.addColumnValue(valueBuilder.build());
222 
223     MutationProto proto = mutateBuilder.build();
224     // default fields
225     assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
226 
227     // set the default value for equal comparison
228     mutateBuilder = MutationProto.newBuilder(proto);
229     mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
230 
231     Increment increment = ProtobufUtil.toIncrement(proto, null);
232     assertEquals(mutateBuilder.build(),
233       ProtobufUtil.toMutation(increment, MutationProto.newBuilder(), HConstants.NO_NONCE));
234   }
235 
236   @Test
237   public void testIncrementRequestConversion() throws Exception {
238     Increment inc = new Increment(Bytes.toBytes("row1"));
239     inc.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("q1"), 10);
240     inc.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("q2"), 20);
241     inc.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("q3"), 30);
242     MutationProto proto =
243         ProtobufUtil.toMutation(inc, MutationProto.newBuilder(), HConstants.NO_NONCE);
244     Increment deSerializedIncrement = ProtobufUtil.toIncrement(proto, null);
245     List<Cell> cellList = deSerializedIncrement.getFamilyCellMap().get(Bytes.toBytes("f1"));
246     Collections.sort(cellList, KeyValue.COMPARATOR);
247     Cell cell0 = inc.getFamilyCellMap().get(Bytes.toBytes("f1")).get(0);
248     Cell cell1 = inc.getFamilyCellMap().get(Bytes.toBytes("f2")).get(0);
249     // We do not serialize TS for increment, hence need to compare parts of cell individually
250     // Compare cell0
251     assertEquals(Bytes.compareTo(cell0.getRowArray(), cell0.getRowOffset(), cell0.getRowLength(),
252       cellList.get(0).getRowArray(), cellList.get(0).getRowOffset(),
253       cellList.get(0).getRowLength()), 0);
254     assertEquals(Bytes.compareTo(cell0.getQualifierArray(), cell0.getQualifierOffset(),
255       cell0.getQualifierLength(), cellList.get(0).getQualifierArray(),
256       cellList.get(0).getQualifierOffset(), cellList.get(0).getQualifierLength()), 0);
257     assertEquals(Bytes.compareTo(cell0.getValueArray(), cell0.getValueOffset(),
258       cell0.getValueLength(), cellList.get(0).getValueArray(), cellList.get(0).getValueOffset(),
259       cellList.get(0).getValueLength()), 0);
260     // Compare cell1
261     Cell deSerCell = deSerializedIncrement.getFamilyCellMap().get(Bytes.toBytes("f2")).get(0);
262     assertEquals(Bytes.compareTo(cell1.getRowArray(), cell1.getRowOffset(), cell1.getRowLength(),
263       deSerCell.getRowArray(), deSerCell.getRowOffset(), deSerCell.getRowLength()), 0);
264     assertEquals(Bytes.compareTo(cell1.getQualifierArray(), cell1.getQualifierOffset(),
265       cell1.getQualifierLength(), deSerCell.getQualifierArray(), deSerCell.getQualifierOffset(),
266       deSerCell.getQualifierLength()), 0);
267     assertEquals(
268       Bytes.compareTo(cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength(),
269         deSerCell.getValueArray(), deSerCell.getValueOffset(), deSerCell.getValueLength()),
270       0);
271   }
272 
273   /**
274    * Test Put Mutate conversions.
275    *
276    * @throws IOException
277    */
278   @Test
279   public void testPut() throws IOException {
280     MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
281     mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
282     mutateBuilder.setMutateType(MutationType.PUT);
283     mutateBuilder.setTimestamp(111111);
284     ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
285     valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
286     QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
287     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
288     qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
289     valueBuilder.addQualifierValue(qualifierBuilder.build());
290     qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
291     qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
292     qualifierBuilder.setTimestamp(222222);
293     valueBuilder.addQualifierValue(qualifierBuilder.build());
294     mutateBuilder.addColumnValue(valueBuilder.build());
295 
296     MutationProto proto = mutateBuilder.build();
297     // default fields
298     assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
299 
300     // set the default value for equal comparison
301     mutateBuilder = MutationProto.newBuilder(proto);
302     mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
303 
304     Put put = ProtobufUtil.toPut(proto);
305 
306     // put value always use the default timestamp if no
307     // value level timestamp specified,
308     // add the timestamp to the original mutate
309     long timestamp = put.getTimeStamp();
310     for (ColumnValue.Builder column:
311         mutateBuilder.getColumnValueBuilderList()) {
312       for (QualifierValue.Builder qualifier:
313           column.getQualifierValueBuilderList()) {
314         if (!qualifier.hasTimestamp()) {
315           qualifier.setTimestamp(timestamp);
316         }
317       }
318     }
319     assertEquals(mutateBuilder.build(),
320       ProtobufUtil.toMutation(MutationType.PUT, put));
321   }
322 
323   /**
324    * Test basic Scan conversions.
325    *
326    * @throws IOException
327    */
328   @Test
329   public void testScan() throws IOException {
330     ClientProtos.Scan.Builder scanBuilder = ClientProtos.Scan.newBuilder();
331     scanBuilder.setStartRow(ByteString.copyFromUtf8("row1"));
332     scanBuilder.setStopRow(ByteString.copyFromUtf8("row2"));
333     Column.Builder columnBuilder = Column.newBuilder();
334     columnBuilder.setFamily(ByteString.copyFromUtf8("f1"));
335     columnBuilder.addQualifier(ByteString.copyFromUtf8("c1"));
336     columnBuilder.addQualifier(ByteString.copyFromUtf8("c2"));
337     scanBuilder.addColumn(columnBuilder.build());
338 
339     columnBuilder.clear();
340     columnBuilder.setFamily(ByteString.copyFromUtf8("f2"));
341     scanBuilder.addColumn(columnBuilder.build());
342 
343     ClientProtos.Scan proto = scanBuilder.build();
344 
345     // Verify default values
346     assertEquals(1, proto.getMaxVersions());
347     assertEquals(true, proto.getCacheBlocks());
348 
349     // Verify fields survive ClientProtos.Scan -> Scan -> ClientProtos.Scan
350     // conversion
351     scanBuilder = ClientProtos.Scan.newBuilder(proto);
352     scanBuilder.setMaxVersions(2);
353     scanBuilder.setCacheBlocks(false);
354     scanBuilder.setCaching(1024);
355     scanBuilder.setIncludeStopRow(false);
356     ClientProtos.Scan expectedProto = scanBuilder.build();
357 
358     ClientProtos.Scan actualProto = ProtobufUtil.toScan(
359         ProtobufUtil.toScan(expectedProto));
360     assertEquals(expectedProto, actualProto);
361   }
362 
363   @Test
364   public void testMetaRegionState() throws Exception {
365     ServerName serverName = ServerName.valueOf("localhost", 1234, 5678);
366     // New region state style.
367     for (RegionState.State state: RegionState.State.values()) {
368       RegionState regionState =
369           new RegionState(HRegionInfo.FIRST_META_REGIONINFO, state, serverName);
370       MetaRegionServer metars = MetaRegionServer.newBuilder()
371           .setServer(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toServerName(serverName))
372           .setRpcVersion(HConstants.RPC_CURRENT_VERSION)
373           .setState(state.convert()).build();
374       // Serialize
375       byte[] data = ProtobufUtil.prependPBMagic(metars.toByteArray());
376       ProtobufUtil.prependPBMagic(data);
377       // Deserialize
378       RegionState regionStateNew = ProtobufUtil.parseMetaRegionStateFrom(data, 1);
379       assertEquals(regionState.getServerName(), regionStateNew.getServerName());
380       assertEquals(regionState.getState(), regionStateNew.getState());
381     }
382     // old style.
383     RegionState rs =
384         org.apache.hadoop.hbase.protobuf.ProtobufUtil.parseMetaRegionStateFrom(
385             serverName.getVersionedBytes(), 1);
386     assertEquals(serverName, rs.getServerName());
387     assertEquals(rs.getState(), RegionState.State.OPEN);
388   }
389 
390   /**
391    * Test {@link ProtobufUtil#toCell(Cell, boolean)} and
392    * {@link ProtobufUtil#toCell(CellProtos.Cell, boolean)} conversion
393    * methods when it contains tags and encode/decode tags is set to true.
394    */
395   @Test
396   public void testCellConversionWithTags() {
397     Cell cell = getCellWithTags();
398     CellProtos.Cell protoCell = ProtobufUtil.toCell(cell, true);
399     assertNotNull(protoCell);
400 
401     Cell decodedCell = getCellFromProtoResult(protoCell, true);
402     List<Tag> decodedTags = Tag.asList(decodedCell.getTagsArray(), decodedCell.getTagsOffset(),
403         decodedCell.getTagsLength());
404     assertEquals(1,  decodedTags.size());
405     Tag decodedTag = decodedTags.get(0);
406     assertEquals(TAG_TYPE, decodedTag.getType());
407     assertEquals(TAG_STR, Bytes.toStringBinary(decodedTag.getValue()));
408   }
409 
410   private Cell getCellWithTags() {
411     Tag tag = new Tag(TAG_TYPE, TAG_STR);
412     KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("f1"),
413       Bytes.toBytes("q1"), 10L, Bytes.toBytes("value1"), new Tag[] {tag});
414     return kv;
415   }
416 
417   private Cell getCellFromProtoResult(CellProtos.Cell protoCell, boolean decodeTags) {
418     return ProtobufUtil.toCell(protoCell, decodeTags);
419   }
420 
421   /**
422    * Test {@link ProtobufUtil#toCell(Cell, boolean)} and
423    * {@link ProtobufUtil#toCell(CellProtos.Cell, boolean)} conversion
424    * methods when it contains tags and encode/decode tags is set to false.
425    */
426   @Test
427   public void testCellConversionWithoutTags() {
428     Cell cell = getCellWithTags();
429     CellProtos.Cell protoCell =
430       ProtobufUtil.toCell(cell, false);
431     assertNotNull(protoCell);
432 
433     Cell decodedCell = getCellFromProtoResult(protoCell, false);
434     List<Tag> decodedTags = Tag.asList(decodedCell.getTagsArray(), decodedCell.getTagsOffset(),
435       decodedCell.getTagsLength());
436     assertEquals(0,  decodedTags.size());
437   }
438 
439   /**
440    * Test {@link ProtobufUtil#toCell(Cell, boolean)} and
441    * {@link ProtobufUtil#toCell(CellProtos.Cell, boolean)} conversion
442    * methods when it contains tags and encoding of tags is set to false
443    * and decoding of tags is set to true.
444    */
445   @Test
446   public void testTagEncodeFalseDecodeTrue() {
447     Cell cell = getCellWithTags();
448     CellProtos.Cell protoCell =
449       ProtobufUtil.toCell(cell, false);
450     assertNotNull(protoCell);
451 
452     Cell decodedCell = getCellFromProtoResult(protoCell, true);
453     List<Tag> decodedTags = Tag.asList(decodedCell.getTagsArray(), decodedCell.getTagsOffset(),
454       decodedCell.getTagsLength());
455     assertEquals(0,  decodedTags.size());
456   }
457 
458   /**
459    * Test {@link ProtobufUtil#toCell(Cell, boolean)} and
460    * {@link ProtobufUtil#toCell(CellProtos.Cell, boolean)} conversion
461    * methods when it contains tags and encoding of tags is set to true
462    * and decoding of tags is set to false.
463    */
464   @Test
465   public void testTagEncodeTrueDecodeFalse() {
466     Cell cell = getCellWithTags();
467     CellProtos.Cell protoCell =
468       ProtobufUtil.toCell(cell, true);
469     assertNotNull(protoCell);
470 
471     Cell decodedCell = getCellFromProtoResult(protoCell, false);
472     List<Tag> decodedTags = Tag.asList(decodedCell.getTagsArray(), decodedCell.getTagsOffset(),
473       decodedCell.getTagsLength());
474     assertEquals(0,  decodedTags.size());
475   }
476 }