View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.Map;
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HTableDescriptor;
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
33  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
34  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
35  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
36  import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
37  import org.apache.hadoop.hbase.util.FSUtils;
38  import org.apache.hadoop.hbase.wal.WAL;
39  import org.apache.hadoop.hbase.wal.WALKey;
40  
41  import com.google.protobuf.TextFormat;
42  
43  /**
44   * Helper methods to ease Region Server integration with the write ahead log.
45   * Note that methods in this class specifically should not require access to anything
46   * other than the API found in {@link WAL}.
47   */
48  @InterfaceAudience.Private
49  public class WALUtil {
50    private static final Log LOG = LogFactory.getLog(WALUtil.class);
51  
52    private WALUtil() {
53      // Shut down construction of this class.
54    }
55  
56    /**
57     * Write the marker that a compaction has succeeded and is about to be committed.
58     * This provides info to the HMaster to allow it to recover the compaction if
59     * this regionserver dies in the middle (This part is not yet implemented). It also prevents
60     * the compaction from finishing if this regionserver has already lost its lease on the log.
61     * @param mvcc Used by WAL to get sequence Id for the waledit.
62     */
63    public static long writeCompactionMarker(WAL wal, HTableDescriptor htd, HRegionInfo hri,
64        final CompactionDescriptor c, MultiVersionConcurrencyControl mvcc)
65    throws IOException {
66      long trx = writeMarker(wal, htd, hri, WALEdit.createCompaction(hri, c), mvcc, true, null);
67      if (LOG.isTraceEnabled()) {
68        LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
69      }
70      return trx;
71    }
72  
73    /**
74     * Write a flush marker indicating a start / abort or a complete of a region flush
75     */
76    public static long writeFlushMarker(WAL wal, HTableDescriptor htd, HRegionInfo hri,
77        final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc)
78    throws IOException {
79      long trx = writeMarker(wal, htd, hri, WALEdit.createFlushWALEdit(hri, f), mvcc, sync, null);
80      if (LOG.isTraceEnabled()) {
81        LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
82      }
83      return trx;
84    }
85  
86    /**
87     * Write a region open marker indicating that the region is opened
88     */
89    public static long writeRegionEventMarker(WAL wal, HTableDescriptor htd, HRegionInfo hri,
90        final RegionEventDescriptor r, final MultiVersionConcurrencyControl mvcc)
91    throws IOException {
92      long trx = writeMarker(wal, htd, hri, WALEdit.createRegionEventWALEdit(hri, r), mvcc, true,
93        null);
94      if (LOG.isTraceEnabled()) {
95        LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r));
96      }
97      return trx;
98    }
99  
100   /**
101    * Write a log marker that a bulk load has succeeded and is about to be committed.
102    *
103    * @param wal        The log to write into.
104    * @param htd        A description of the table that we are bulk loading into.
105    * @param hri       A description of the region in the table that we are bulk loading into.
106    * @param desc A protocol buffers based description of the client's bulk loading request
107    * @return txid of this transaction or if nothing to do, the last txid
108    * @throws IOException We will throw an IOException if we can not append to the HLog.
109    */
110   public static long writeBulkLoadMarkerAndSync(final WAL wal, final HTableDescriptor htd,
111       final HRegionInfo hri, final WALProtos.BulkLoadDescriptor desc,
112       final MultiVersionConcurrencyControl mvcc)
113   throws IOException {
114     long trx = writeMarker(wal, htd, hri, WALEdit.createBulkLoadEvent(hri, desc), mvcc, true,
115       null);
116     if (LOG.isTraceEnabled()) {
117       LOG.trace("Appended Bulk Load marker " + TextFormat.shortDebugString(desc));
118     }
119     return trx;
120   }
121 
122   private static long writeMarker(final WAL wal, final HTableDescriptor htd, final HRegionInfo hri,
123       final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final boolean sync,
124       final Map<String, byte[]> extendedAttributes)
125   throws IOException {
126     // TODO: Pass in current time to use?
127     WALKey key = new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(),
128       mvcc, extendedAttributes);
129     // Add it to the log but the false specifies that we don't need to add it to the memstore
130     long trx = MultiVersionConcurrencyControl.NONE;
131     try {
132       trx = wal.append(htd, hri, key, edit, false);
133       if (sync) wal.sync(trx);
134     } finally {
135       // If you get hung here, is it a real WAL or a mocked WAL? If the latter, you need to
136       // trip the latch that is inside in getWriteEntry up in your mock. See down in the append
137       // called from onEvent in FSHLog.
138       MultiVersionConcurrencyControl.WriteEntry we = key.getWriteEntry();
139       if (mvcc != null && we != null) mvcc.complete(we);
140     }
141     return trx;
142   }
143 
144   /**
145    * Blocksize returned here is 2x the default HDFS blocksize unless explicitly set in
146    * Configuration. Works in tandem with hbase.regionserver.logroll.multiplier. See comment in
147    * AbstractFSWAL in Constructor where we set blocksize and logrollsize for why.
148    * @return Blocksize to use writing WALs.
149    */
150   public static long getWALBlockSize(Configuration conf, FileSystem fs, Path dir)
151       throws IOException {
152     return getWALBlockSize(conf, fs, dir, false);
153   }
154 
155   /**
156    * Public because of FSHLog. Should be package-private
157    * @param isRecoverEdits the created writer is for recovered edits or WAL. For recovered edits, it
158    *          is true and for WAL it is false.
159    */
160   public static long getWALBlockSize(Configuration conf, FileSystem fs, Path dir,
161       boolean isRecoverEdits) throws IOException {
162     long defaultBlockSize = FSUtils.getDefaultBlockSize(fs, dir) * 2;
163     if (isRecoverEdits) {
164       return conf.getLong("hbase.regionserver.recoverededits.blocksize", defaultBlockSize);
165     }
166     return conf.getLong("hbase.regionserver.hlog.blocksize", defaultBlockSize);
167   }
168 }