1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.protobuf;
21
22
23 import com.google.protobuf.ServiceException;
24
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.Iterator;
28 import java.util.List;
29
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellScanner;
33 import org.apache.hadoop.hbase.CellUtil;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.io.SizedCellScanner;
36 import org.apache.hadoop.hbase.ipc.HBaseRpcController;
37 import org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl;
38 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
39 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
40 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
41 import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
42 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
43 import org.apache.hadoop.hbase.util.ByteStringer;
44 import org.apache.hadoop.hbase.util.Pair;
45 import org.apache.hadoop.hbase.wal.WAL.Entry;
46 import org.apache.hadoop.hbase.wal.WALKey;
47
48 @InterfaceAudience.Private
49 public class ReplicationProtbufUtil {
50
51
52
53
54
55
56
57
58
59
60 public static void replicateWALEntry(final AdminService.BlockingInterface admin,
61 final Entry[] entries, String replicationClusterId, Path sourceBaseNamespaceDir,
62 Path sourceHFileArchiveDir) throws IOException {
63 Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner> p =
64 buildReplicateWALEntryRequest(entries, null, replicationClusterId, sourceBaseNamespaceDir,
65 sourceHFileArchiveDir);
66 HBaseRpcController controller = new HBaseRpcControllerImpl(p.getSecond());
67 try {
68 admin.replicateWALEntry(controller, p.getFirst());
69 } catch (ServiceException se) {
70 throw ProtobufUtil.getServiceException(se);
71 }
72 }
73
74
75
76
77
78
79
80
81 public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
82 buildReplicateWALEntryRequest(final Entry[] entries) throws IOException {
83
84 return buildReplicateWALEntryRequest(entries, null, null, null, null);
85 }
86
87
88
89
90
91
92
93
94
95
96
97 public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
98 buildReplicateWALEntryRequest(final Entry[] entries, byte[] encodedRegionName,
99 String replicationClusterId, Path sourceBaseNamespaceDir, Path sourceHFileArchiveDir)
100 throws IOException {
101
102 List<List<? extends Cell>> allCells = new ArrayList<List<? extends Cell>>(entries.length);
103 int size = 0;
104 AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder();
105 AdminProtos.ReplicateWALEntryRequest.Builder builder =
106 AdminProtos.ReplicateWALEntryRequest.newBuilder();
107 for (Entry entry: entries) {
108 entryBuilder.clear();
109 WALKey key = entry.getKey();
110 WALProtos.WALKey.Builder keyBuilder;
111 try {
112 keyBuilder = key.getBuilder(WALCellCodec.getNoneCompressor());
113 } catch (IOException e) {
114 throw new IOException(
115 "There should not throw exception since NoneCompressor do not throw any exceptions", e);
116 }
117 if(encodedRegionName != null){
118 keyBuilder.setEncodedRegionName(
119 ByteStringer.wrap(encodedRegionName));
120 }
121 entryBuilder.setKey(keyBuilder.build());
122 WALEdit edit = entry.getEdit();
123 List<Cell> cells = edit.getCells();
124
125 for (Cell cell: cells) {
126 size += CellUtil.estimatedSerializedSizeOf(cell);
127 }
128
129 allCells.add(cells);
130
131 entryBuilder.setAssociatedCellCount(cells.size());
132 builder.addEntry(entryBuilder.build());
133 }
134
135 if (replicationClusterId != null) {
136 builder.setReplicationClusterId(replicationClusterId);
137 }
138 if (sourceBaseNamespaceDir != null) {
139 builder.setSourceBaseNamespaceDirPath(sourceBaseNamespaceDir.toString());
140 }
141 if (sourceHFileArchiveDir != null) {
142 builder.setSourceHFileArchiveDirPath(sourceHFileArchiveDir.toString());
143 }
144
145 return new Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>(builder.build(),
146 getCellScanner(allCells, size));
147 }
148
149
150
151
152
153 static CellScanner getCellScanner(final List<List<? extends Cell>> cells, final int size) {
154 return new SizedCellScanner() {
155 private final Iterator<List<? extends Cell>> entries = cells.iterator();
156 private Iterator<? extends Cell> currentIterator = null;
157 private Cell currentCell;
158
159 @Override
160 public Cell current() {
161 return this.currentCell;
162 }
163
164 @Override
165 public boolean advance() {
166 if (this.currentIterator == null) {
167 if (!this.entries.hasNext()) return false;
168 this.currentIterator = this.entries.next().iterator();
169 }
170 if (this.currentIterator.hasNext()) {
171 this.currentCell = this.currentIterator.next();
172 return true;
173 }
174 this.currentCell = null;
175 this.currentIterator = null;
176 return advance();
177 }
178
179 @Override
180 public long heapSize() {
181 return size;
182 }
183 };
184 }
185 }