View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.querymatcher;
19  
20  import java.io.IOException;
21  
22  import org.apache.hadoop.hbase.Cell;
23  import org.apache.hadoop.hbase.HConstants;
24  import org.apache.hadoop.hbase.KeyValue;
25  import org.apache.hadoop.hbase.KeyValue.KVComparator;
26  import org.apache.hadoop.hbase.KeyValue.Type;
27  import org.apache.hadoop.hbase.KeyValueUtil;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.filter.Filter;
30  import org.apache.hadoop.hbase.regionserver.DeleteTracker;
31  import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult;
32  import org.apache.hadoop.hbase.regionserver.HStore;
33  import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
34  import org.apache.hadoop.hbase.regionserver.ScanInfo;
35  
36  /**
37   * A query matcher that is specifically designed for the scan case.
38   */
39  @InterfaceAudience.Private
40  public abstract class ScanQueryMatcher {
41  
42    /**
43     * {@link #match} return codes. These instruct the scanner moving through memstores and StoreFiles
44     * what to do with the current KeyValue.
45     * <p>
46     * Additionally, this contains "early-out" language to tell the scanner to move on to the next
47     * File (memstore or Storefile), or to return immediately.
48     */
49    public static enum MatchCode {
50      /**
51       * Include KeyValue in the returned result
52       */
53      INCLUDE,
54  
55      /**
56       * Do not include KeyValue in the returned result
57       */
58      SKIP,
59  
60      /**
61       * Do not include, jump to next StoreFile or memstore (in time order)
62       */
63      NEXT,
64  
65      /**
66       * Do not include, return current result
67       */
68      DONE,
69  
70      /**
71       * These codes are used by the ScanQueryMatcher
72       */
73  
74      /**
75       * Done with the row, seek there.
76       */
77      SEEK_NEXT_ROW,
78  
79      /**
80       * Done with column, seek to next.
81       */
82      SEEK_NEXT_COL,
83  
84      /**
85       * Done with scan, thanks to the row filter.
86       */
87      DONE_SCAN,
88  
89      /**
90       * Seek to next key which is given as hint.
91       */
92      SEEK_NEXT_USING_HINT,
93  
94      /**
95       * Include KeyValue and done with column, seek to next.
96       */
97      INCLUDE_AND_SEEK_NEXT_COL,
98  
99      /**
100      * Include KeyValue and done with row, seek to next.
101      */
102     INCLUDE_AND_SEEK_NEXT_ROW,
103   }
104 
105   /** Row comparator for the region this query is for */
106   protected final KVComparator rowComparator;
107 
108   /** Key to seek to in memstore and StoreFiles */
109   protected final Cell startKey;
110 
111   /** Keeps track of columns and versions */
112   protected final ColumnTracker columns;
113 
114   /** The oldest timestamp we are interested in, based on TTL */
115   protected final long oldestUnexpiredTS;
116 
117   protected final long now;
118 
119   /** Row the query is on */
120   protected Cell currentRow;
121 
122   protected ScanQueryMatcher(Cell startKey, ScanInfo scanInfo, ColumnTracker columns,
123       long oldestUnexpiredTS, long now) {
124     this.rowComparator = scanInfo.getComparator();
125     this.startKey = startKey;
126     this.oldestUnexpiredTS = oldestUnexpiredTS;
127     this.now = now;
128     this.columns = columns;
129   }
130 
131   protected static Cell createStartKeyFromRow(byte[] startRow, ScanInfo scanInfo) {
132     return KeyValueUtil.createFirstDeleteFamilyOnRow(startRow, scanInfo.getFamily());
133   }
134 
135   /**
136    * Check before the delete logic.
137    * @return null means continue.
138    */
139   protected final MatchCode preCheck(Cell cell) {
140     if (currentRow == null) {
141       // Since the curCell is null it means we are already sure that we have moved over to the next
142       // row
143       return MatchCode.DONE;
144     }
145     // if row key is changed, then we know that we have moved over to the next row
146     if (rowComparator.compareRows(currentRow, cell) != 0) {
147       return MatchCode.DONE;
148     }
149 
150     if (this.columns.done()) {
151       return MatchCode.SEEK_NEXT_ROW;
152     }
153 
154     long timestamp = cell.getTimestamp();
155     // check if this is a fake cell. The fake cell is an optimization, we should make the scanner
156     // seek to next column or next row. See StoreFileScanner.requestSeek for more details.
157     // check for early out based on timestamp alone
158     if (timestamp == HConstants.OLDEST_TIMESTAMP || columns.isDone(timestamp)) {
159       return columns.getNextRowOrNextColumn(cell.getQualifierArray(), cell.getQualifierOffset(),
160         cell.getQualifierLength());
161     }
162     // check if the cell is expired by cell TTL
163     if (HStore.isCellTTLExpired(cell, this.oldestUnexpiredTS, this.now)) {
164       return MatchCode.SKIP;
165     }
166     return null;
167   }
168 
169   protected final MatchCode checkDeleted(DeleteTracker deletes, Cell cell) {
170     if (deletes.isEmpty()) {
171       return null;
172     }
173     DeleteResult deleteResult = deletes.isDeleted(cell);
174     switch (deleteResult) {
175       case FAMILY_DELETED:
176       case COLUMN_DELETED:
177         return columns.getNextRowOrNextColumn(cell.getQualifierArray(), cell.getQualifierOffset(),
178           cell.getQualifierLength());
179       case VERSION_DELETED:
180       case FAMILY_VERSION_DELETED:
181         return MatchCode.SKIP;
182       case NOT_DELETED:
183         return null;
184       default:
185         throw new RuntimeException("Unexpected delete result: " + deleteResult);
186     }
187   }
188 
189   /**
190    * Determines if the caller should do one of several things:
191    * <ul>
192    * <li>seek/skip to the next row (MatchCode.SEEK_NEXT_ROW)</li>
193    * <li>seek/skip to the next column (MatchCode.SEEK_NEXT_COL)</li>
194    * <li>include the current KeyValue (MatchCode.INCLUDE)</li>
195    * <li>ignore the current KeyValue (MatchCode.SKIP)</li>
196    * <li>got to the next row (MatchCode.DONE)</li>
197    * </ul>
198    * @param cell KeyValue to check
199    * @return The match code instance.
200    * @throws IOException in case there is an internal consistency problem caused by a data
201    *           corruption.
202    */
203   public abstract MatchCode match(Cell cell) throws IOException;
204 
205   /**
206    * @return the start key
207    */
208   public Cell getStartKey() {
209     return startKey;
210   }
211 
212   /**
213    * @return whether there is an null column in the query
214    */
215   public abstract boolean hasNullColumnInQuery();
216 
217   /**
218    * @return a cell represent the current row
219    */
220   public Cell currentRow() {
221     return currentRow;
222   }
223 
224   /**
225    * Make {@link #currentRow()} return null.
226    */
227   public void clearCurrentRow() {
228     currentRow = null;
229   }
230 
231   protected abstract void reset();
232 
233   /**
234    * Set the row when there is change in row
235    * @param currentRow
236    */
237   public void setToNewRow(Cell currentRow) {
238     this.currentRow = currentRow;
239     columns.reset();
240     reset();
241   }
242 
243   public abstract boolean isUserScan();
244 
245   /**
246    * @return Returns false if we know there are no more rows to be scanned (We've reached the
247    *         <code>stopRow</code> or we are scanning on row only because this Scan is for a Get,
248    *         etc.
249    */
250   public abstract boolean moreRowsMayExistAfter(Cell cell);
251 
252   public Cell getKeyForNextColumn(Cell cell) {
253     // We aren't sure whether any DeleteFamily cells exist, so we can't skip to next column.
254     // TODO: Current way disable us to seek to next column quickly. Is there any better solution?
255     // see HBASE-18471 for more details
256     // see TestFromClientSide3#testScanAfterDeletingSpecifiedRow
257     // see TestFromClientSide3#testScanAfterDeletingSpecifiedRowV2
258     if (cell.getQualifierLength() == 0) {
259       Cell nextKey = createNextOnRowCol(cell);
260       if (nextKey != cell) {
261         return nextKey;
262       }
263       // The cell is at the end of row/family/qualifier, so it is impossible to find any DeleteFamily cells.
264       // Let us seek to next column.
265     }
266     ColumnCount nextColumn = columns.getColumnHint();
267     if (nextColumn == null) {
268       return KeyValueUtil.createLastOnRow(cell.getRowArray(), cell.getRowOffset(),
269         cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
270         cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
271     } else {
272       return KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(),
273         cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
274         nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength());
275     }
276   }
277 
278   /**
279    * @param nextIndexed the key of the next entry in the block index (if any)
280    * @param currentCell The Cell we're using to calculate the seek key
281    * @return result of the compare between the indexed key and the key portion of the passed cell
282    */
283   public int compareKeyForNextRow(Cell nextIndexed, Cell currentCell) {
284     return rowComparator.compareKey(nextIndexed, currentCell.getRowArray(),
285       currentCell.getRowOffset(), currentCell.getRowLength(), null, 0, 0, null, 0, 0,
286       HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode());
287   }
288 
289   /**
290    * @param nextIndexed the key of the next entry in the block index (if any)
291    * @param currentCell The Cell we're using to calculate the seek key
292    * @return result of the compare between the indexed key and the key portion of the passed cell
293    */
294   public int compareKeyForNextColumn(Cell nextIndexed, Cell currentCell) {
295     ColumnCount nextColumn = columns.getColumnHint();
296     if (nextColumn == null) {
297       return rowComparator.compareKey(nextIndexed, currentCell.getRowArray(),
298         currentCell.getRowOffset(), currentCell.getRowLength(), currentCell.getFamilyArray(),
299         currentCell.getFamilyOffset(), currentCell.getFamilyLength(),
300         currentCell.getQualifierArray(), currentCell.getQualifierOffset(),
301         currentCell.getQualifierLength(), HConstants.OLDEST_TIMESTAMP, Type.Minimum.getCode());
302     } else {
303       return rowComparator.compareKey(nextIndexed, currentCell.getRowArray(),
304         currentCell.getRowOffset(), currentCell.getRowLength(), currentCell.getFamilyArray(),
305         currentCell.getFamilyOffset(), currentCell.getFamilyLength(), nextColumn.getBuffer(),
306         nextColumn.getOffset(), nextColumn.getLength(), HConstants.LATEST_TIMESTAMP,
307         Type.Maximum.getCode());
308     }
309   }
310 
311   /**
312    * @return the Filter
313    */
314   public abstract Filter getFilter();
315 
316   /**
317    * Delegate to {@link Filter#getNextCellHint(Cell)}. If no filter, return {@code null}.
318    */
319   public abstract Cell getNextKeyHint(Cell cell) throws IOException;
320 
321   protected static DeleteTracker instantiateDeleteTracker(RegionCoprocessorHost host)
322       throws IOException {
323     DeleteTracker tracker = new ScanDeleteTracker();
324     if (host != null) {
325       tracker = host.postInstantiateDeleteTracker(tracker);
326     }
327     return tracker;
328   }
329 
330   // Used only for testing purposes
331   static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset, int length,
332       long ttl, byte type, boolean ignoreCount) throws IOException {
333     MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
334     if (matchCode == MatchCode.INCLUDE) {
335       return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
336     }
337     return matchCode;
338   }
339 
340   /**
341    * @return An new cell is located following input cell. If both of type and timestamp are
342    *         minimum, the input cell will be returned directly.
343    */
344   private static Cell createNextOnRowCol(Cell cell) {
345     long ts = cell.getTimestamp();
346     byte type = cell.getTypeByte();
347     if (type != Type.Minimum.getCode()) {
348       type = KeyValue.Type.values()[KeyValue.Type.codeToType(type).ordinal() - 1].getCode();
349     } else if (ts != HConstants.OLDEST_TIMESTAMP) {
350       ts = ts - 1;
351       type = Type.Maximum.getCode();
352     } else {
353       return cell;
354     }
355     return createNextOnRowCol(cell, ts, type);
356   }
357 
358   private static Cell createNextOnRowCol(final Cell cell, final long ts, final byte type) {
359     return new Cell() {
360       @Override
361       public byte[] getRowArray() { return cell.getRowArray(); }
362 
363       @Override
364       public int getRowOffset() { return cell.getRowOffset(); }
365 
366       @Override
367       public short getRowLength() { return cell.getRowLength(); }
368 
369       @Override
370       public byte[] getFamilyArray() { return cell.getFamilyArray(); }
371 
372       @Override
373       public int getFamilyOffset() { return cell.getFamilyOffset(); }
374 
375       @Override
376       public byte getFamilyLength() { return cell.getFamilyLength(); }
377 
378       @Override
379       public byte[] getQualifierArray() { return cell.getQualifierArray(); }
380 
381       @Override
382       public int getQualifierOffset() { return cell.getQualifierOffset(); }
383 
384       @Override
385       public int getQualifierLength() { return cell.getQualifierLength(); }
386 
387       @Override
388       public long getTimestamp() { return ts; }
389 
390       @Override
391       public byte getTypeByte() {return type; }
392 
393       @Override
394       public long getMvccVersion() { return cell.getMvccVersion(); }
395 
396       @Override
397       public long getSequenceId() { return cell.getSequenceId(); }
398 
399       @Override
400       public byte[] getValueArray() { return cell.getValueArray(); }
401 
402       @Override
403       public int getValueOffset() { return cell.getValueOffset(); }
404 
405       @Override
406       public int getValueLength() { return cell.getValueLength(); }
407 
408       @Override
409       public byte[] getTagsArray() { return cell.getTagsArray(); }
410 
411       @Override
412       public int getTagsOffset() { return cell.getTagsOffset(); }
413 
414       @Override
415       public int getTagsLength() { return cell.getTagsLength(); }
416 
417       @Override
418       public byte[] getValue() { return cell.getValue(); }
419 
420       @Override
421       public byte[] getFamily() { return cell.getFamily(); }
422 
423       @Override
424       public byte[] getQualifier() { return cell.getQualifier(); }
425 
426       @Override
427       public byte[] getRow() {return cell.getRow(); }
428     };
429   }
430 }