View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.protobuf;
21  
22  
23  import com.google.protobuf.ServiceException;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.Iterator;
28  import java.util.List;
29  
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellScanner;
33  import org.apache.hadoop.hbase.CellUtil;
34  import org.apache.hadoop.hbase.classification.InterfaceAudience;
35  import org.apache.hadoop.hbase.io.SizedCellScanner;
36  import org.apache.hadoop.hbase.ipc.HBaseRpcController;
37  import org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
40  import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
41  import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
42  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
43  import org.apache.hadoop.hbase.util.ByteStringer;
44  import org.apache.hadoop.hbase.util.Pair;
45  import org.apache.hadoop.hbase.wal.WAL.Entry;
46  import org.apache.hadoop.hbase.wal.WALKey;
47  
48  @InterfaceAudience.Private
49  public class ReplicationProtbufUtil {
50    /**
51     * A helper to replicate a list of WAL entries using admin protocol.
52     * @param admin Admin service
53     * @param entries Array of WAL entries to be replicated
54     * @param replicationClusterId Id which will uniquely identify source cluster FS client
55     *          configurations in the replication configuration directory
56     * @param sourceBaseNamespaceDir Path to source cluster base namespace directory
57     * @param sourceHFileArchiveDir Path to the source cluster hfile archive directory
58     * @throws java.io.IOException
59     */
60    public static void replicateWALEntry(final AdminService.BlockingInterface admin,
61        final Entry[] entries, String replicationClusterId, Path sourceBaseNamespaceDir,
62        Path sourceHFileArchiveDir) throws IOException {
63      Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner> p =
64          buildReplicateWALEntryRequest(entries, null, replicationClusterId, sourceBaseNamespaceDir,
65            sourceHFileArchiveDir);
66      HBaseRpcController controller = new HBaseRpcControllerImpl(p.getSecond());
67      try {
68        admin.replicateWALEntry(controller, p.getFirst());
69      } catch (ServiceException se) {
70        throw ProtobufUtil.getServiceException(se);
71      }
72    }
73  
74    /**
75     * Create a new ReplicateWALEntryRequest from a list of WAL entries
76     *
77     * @param entries the WAL entries to be replicated
78     * @return a pair of ReplicateWALEntryRequest and a CellScanner over all the WALEdit values
79     * found.
80     */
81    public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
82        buildReplicateWALEntryRequest(final Entry[] entries) throws IOException {
83      // Accumulate all the Cells seen in here.
84      return buildReplicateWALEntryRequest(entries, null, null, null, null);
85    }
86  
87    /**
88     * Create a new ReplicateWALEntryRequest from a list of HLog entries
89     * @param entries the HLog entries to be replicated
90     * @param encodedRegionName alternative region name to use if not null
91     * @param replicationClusterId Id which will uniquely identify source cluster FS client
92     *          configurations in the replication configuration directory
93     * @param sourceBaseNamespaceDir Path to source cluster base namespace directory
94     * @param sourceHFileArchiveDir Path to the source cluster hfile archive directory
95     * @return a pair of ReplicateWALEntryRequest and a CellScanner over all the WALEdit values found.
96     */
97    public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
98        buildReplicateWALEntryRequest(final Entry[] entries, byte[] encodedRegionName,
99            String replicationClusterId, Path sourceBaseNamespaceDir, Path sourceHFileArchiveDir)
100     throws IOException {
101     // Accumulate all the KVs seen in here.
102     List<List<? extends Cell>> allCells = new ArrayList<List<? extends Cell>>(entries.length);
103     int size = 0;
104     AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder();
105     AdminProtos.ReplicateWALEntryRequest.Builder builder =
106       AdminProtos.ReplicateWALEntryRequest.newBuilder();
107     for (Entry entry: entries) {
108       entryBuilder.clear();
109       WALKey key = entry.getKey();
110       WALProtos.WALKey.Builder keyBuilder;
111       try {
112         keyBuilder = key.getBuilder(WALCellCodec.getNoneCompressor());
113       } catch (IOException e) {
114         throw new IOException(
115           "There should not throw exception since NoneCompressor do not throw any exceptions", e);
116       }
117       if(encodedRegionName != null){
118         keyBuilder.setEncodedRegionName(
119           ByteStringer.wrap(encodedRegionName));
120       }
121       entryBuilder.setKey(keyBuilder.build());
122       WALEdit edit = entry.getEdit();
123       List<Cell> cells = edit.getCells();
124       // Add up the size.  It is used later serializing out the cells.
125       for (Cell cell: cells) {
126         size += CellUtil.estimatedSerializedSizeOf(cell);
127       }
128       // Collect up the cells
129       allCells.add(cells);
130       // Write out how many cells associated with this entry.
131       entryBuilder.setAssociatedCellCount(cells.size());
132       builder.addEntry(entryBuilder.build());
133     }
134 
135     if (replicationClusterId != null) {
136       builder.setReplicationClusterId(replicationClusterId);
137     }
138     if (sourceBaseNamespaceDir != null) {
139       builder.setSourceBaseNamespaceDirPath(sourceBaseNamespaceDir.toString());
140     }
141     if (sourceHFileArchiveDir != null) {
142       builder.setSourceHFileArchiveDirPath(sourceHFileArchiveDir.toString());
143     }
144 
145     return new Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>(builder.build(),
146       getCellScanner(allCells, size));
147   }
148 
149   /**
150    * @param cells
151    * @return <code>cells</code> packaged as a CellScanner
152    */
153   static CellScanner getCellScanner(final List<List<? extends Cell>> cells, final int size) {
154     return new SizedCellScanner() {
155       private final Iterator<List<? extends Cell>> entries = cells.iterator();
156       private Iterator<? extends Cell> currentIterator = null;
157       private Cell currentCell;
158 
159       @Override
160       public Cell current() {
161         return this.currentCell;
162       }
163 
164       @Override
165       public boolean advance() {
166         if (this.currentIterator == null) {
167           if (!this.entries.hasNext()) return false;
168           this.currentIterator = this.entries.next().iterator();
169         }
170         if (this.currentIterator.hasNext()) {
171           this.currentCell = this.currentIterator.next();
172           return true;
173         }
174         this.currentCell = null;
175         this.currentIterator = null;
176         return advance();
177       }
178 
179       @Override
180       public long heapSize() {
181         return size;
182       }
183     };
184   }
185 }