View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.concurrent.Executors;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HRegionLocation;
34  import org.apache.hadoop.hbase.RegionLocations;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
38  import org.apache.hadoop.hbase.testclassification.ClientTests;
39  import org.apache.hadoop.hbase.testclassification.SmallTests;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  import org.mockito.Mockito;
45  
46  /**
47   * The purpose of this test is to make sure the region exception won't corrupt the results
48   * of batch. The prescription is shown below.
49   * 1) honor the action result rather than region exception. If the action have both of true result
50   * and region exception, the action is fine as the exception is caused by other actions
51   * which are in the same region.
52   * 2) honor the action exception rather than region exception. If the action have both of action
53   * exception and region exception, we deal with the action exception only. If we also
54   * handle the region exception for the same action, it will introduce the negative count of
55   * actions in progress. The AsyncRequestFuture#waitUntilDone will block forever.
56   *
57   * This bug can be reproduced by real use case. see TestMalformedCellFromClient(in branch-1.4+).
58   * It uses the batch of RowMutations to present the bug. Given that the batch of RowMutations is
59   * only supported by branch-1.4+, perhaps the branch-1.3 and branch-1.2 won't encounter this issue.
60   * We still backport the fix to branch-1.3 and branch-1.2 in case we ignore some write paths.
61   */
62  @Category({ ClientTests.class, SmallTests.class })
63  public class TestAsyncProcessWithRegionException {
64  
65    private static final Result EMPTY_RESULT = Result.create(null, true);
66    private static final IOException IOE = new IOException("YOU CAN'T PASS");
67    private static final Configuration CONF = new Configuration();
68    private static final TableName DUMMY_TABLE = TableName.valueOf("DUMMY_TABLE");
69    private static final byte[] GOOD_ROW = Bytes.toBytes("GOOD_ROW");
70    private static final byte[] BAD_ROW = Bytes.toBytes("BAD_ROW");
71    private static final byte[] BAD_ROW_WITHOUT_ACTION_EXCEPTION =
72      Bytes.toBytes("BAD_ROW_WITHOUT_ACTION_EXCEPTION");
73    private static final byte[] FAMILY = Bytes.toBytes("FAMILY");
74    private static final ServerName SERVER_NAME = ServerName.valueOf("s1,1,1");
75    private static final HRegionInfo REGION_INFO
76      = new HRegionInfo(DUMMY_TABLE, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
77  
78    private static final HRegionLocation REGION_LOCATION =
79      new HRegionLocation(REGION_INFO, SERVER_NAME);
80  
81    @BeforeClass
82    public static void setUpBeforeClass() {
83      // disable the retry
84      CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
85    }
86  
87    @Test(timeout=20000)
88    public void testSuccessivePut() throws Exception {
89      MyAsyncProcess ap = new MyAsyncProcess(createHConnection(), CONF);
90  
91      List<Put> puts = new ArrayList<>(1);
92      puts.add(new Put(GOOD_ROW).addColumn(FAMILY, FAMILY, FAMILY));
93      final int expectedSize = puts.size();
94      AsyncProcess.AsyncRequestFuture arf = ap.submit(DUMMY_TABLE, puts);
95      arf.waitUntilDone();
96      Object[] result = arf.getResults();
97      assertEquals(expectedSize, result.length);
98      for (Object r : result) {
99        assertEquals(Result.class, r.getClass());
100     }
101     assertTrue(puts.isEmpty());
102     assertActionsInProgress(arf);
103   }
104 
105   @Test(timeout=20000)
106   public void testFailedPut() throws Exception {
107     MyAsyncProcess ap = new MyAsyncProcess(createHConnection(), CONF);
108 
109     List<Put> puts = new ArrayList<>(2);
110     puts.add(new Put(GOOD_ROW).addColumn(FAMILY, FAMILY, FAMILY));
111     // this put should fail
112     puts.add(new Put(BAD_ROW).addColumn(FAMILY, FAMILY, FAMILY));
113     final int expectedSize = puts.size();
114 
115     AsyncProcess.AsyncRequestFuture arf = ap.submit(DUMMY_TABLE, puts);
116     arf.waitUntilDone();
117     // There is a failed puts
118     assertError(arf, 1);
119     Object[] result = arf.getResults();
120     assertEquals(expectedSize, result.length);
121     assertEquals(Result.class, result[0].getClass());
122     assertTrue(result[1] instanceof IOException);
123     assertTrue(puts.isEmpty());
124     assertActionsInProgress(arf);
125   }
126 
127   @Test(timeout=20000)
128   public void testFailedPutWithoutActionException() throws Exception {
129     MyAsyncProcess ap = new MyAsyncProcess(createHConnection(), CONF);
130 
131     List<Put> puts = new ArrayList<>(3);
132     puts.add(new Put(GOOD_ROW).addColumn(FAMILY, FAMILY, FAMILY));
133     // this put should fail
134     puts.add(new Put(BAD_ROW).addColumn(FAMILY, FAMILY, FAMILY));
135     // this put should fail, and it won't have action exception
136     puts.add(new Put(BAD_ROW_WITHOUT_ACTION_EXCEPTION).addColumn(FAMILY, FAMILY, FAMILY));
137     final int expectedSize = puts.size();
138 
139     AsyncProcess.AsyncRequestFuture arf = ap.submit(DUMMY_TABLE, puts);
140     arf.waitUntilDone();
141     // There are two failed puts
142     assertError(arf, 2);
143     Object[] result = arf.getResults();
144     assertEquals(expectedSize, result.length);
145     assertEquals(Result.class, result[0].getClass());
146     assertTrue(result[1] instanceof IOException);
147     assertTrue(result[2] instanceof IOException);
148     assertTrue(puts.isEmpty());
149     assertActionsInProgress(arf);
150   }
151 
152   private static void assertError(AsyncProcess.AsyncRequestFuture arf, int expectedCountOfFailure) {
153     assertTrue(arf.hasError());
154     RetriesExhaustedWithDetailsException e = arf.getErrors();
155     List<Throwable> errors = e.getCauses();
156     assertEquals(expectedCountOfFailure, errors.size());
157     for (Throwable t : errors) {
158       assertTrue(t instanceof IOException);
159     }
160   }
161 
162   private static void assertActionsInProgress(AsyncProcess.AsyncRequestFuture arf) {
163     if (arf instanceof AsyncProcess.AsyncRequestFutureImpl) {
164       assertEquals(0, ((AsyncProcess.AsyncRequestFutureImpl) arf).getActionsInProgress());
165     }
166   }
167 
168   private static ClusterConnection createHConnection() throws IOException {
169     ClusterConnection hc = Mockito.mock(ClusterConnection.class);
170     NonceGenerator ng = Mockito.mock(NonceGenerator.class);
171     Mockito.when(ng.getNonceGroup()).thenReturn(HConstants.NO_NONCE);
172     Mockito.when(hc.getNonceGenerator()).thenReturn(ng);
173     Mockito.when(hc.getConfiguration()).thenReturn(CONF);
174     Mockito.when(hc.getConnectionConfiguration()).thenReturn(new ConnectionConfiguration(CONF));
175     setMockLocation(hc, GOOD_ROW, new RegionLocations(REGION_LOCATION));
176     setMockLocation(hc, BAD_ROW, new RegionLocations(REGION_LOCATION));
177     Mockito
178       .when(hc.locateRegions(Mockito.eq(DUMMY_TABLE), Mockito.anyBoolean(), Mockito.anyBoolean()))
179       .thenReturn(Collections.singletonList(REGION_LOCATION));
180     return hc;
181   }
182 
183   private static void setMockLocation(ClusterConnection hc, byte[] row, RegionLocations result)
184     throws IOException {
185     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE), Mockito.eq(row), Mockito.anyBoolean(),
186       Mockito.anyBoolean(), Mockito.anyInt())).thenReturn(result);
187     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE), Mockito.eq(row), Mockito.anyBoolean(),
188       Mockito.anyBoolean())).thenReturn(result);
189   }
190 
191   private static class MyAsyncProcess extends AsyncProcess {
192 
193     MyAsyncProcess(ClusterConnection hc, Configuration conf) {
194       super(hc, conf, Executors.newFixedThreadPool(5),
195         new RpcRetryingCallerFactory(conf), false, new RpcControllerFactory(conf),
196         HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
197     }
198 
199     public AsyncRequestFuture submit(TableName tableName, List<? extends Row> rows)
200       throws InterruptedIOException {
201       return super.submit(tableName, rows, true, null, true);
202 
203     }
204 
205     @Override
206     protected RpcRetryingCaller<MultiResponse> createCaller(PayloadCarryingServerCallable callable,
207       int rpcTimeout) {
208       MultiServerCallable<Row> callable1 = (MultiServerCallable<Row>) callable;
209       final MultiResponse mr = new MultiResponse();
210       for (Map.Entry<byte[], List<Action<Row>>> entry : callable1.getMulti().actions.entrySet()) {
211         byte[] regionName = entry.getKey();
212         for (Action<Row> action : entry.getValue()) {
213           if (Bytes.equals(action.getAction().getRow(), GOOD_ROW)) {
214             mr.add(regionName, action.getOriginalIndex(), EMPTY_RESULT);
215           } else if (Bytes.equals(action.getAction().getRow(), BAD_ROW)) {
216             mr.add(regionName, action.getOriginalIndex(), IOE);
217           }
218         }
219       }
220       mr.addException(REGION_INFO.getRegionName(), IOE);
221 
222       return new RpcRetryingCaller<MultiResponse>(100, 500, 0, 0) {
223         @Override
224         public MultiResponse callWithoutRetries(RetryingCallable<MultiResponse> callable,
225           int callTimeout) {
226           try {
227             // sleep one second in order for threadpool to start another thread instead of reusing
228             // existing one.
229             Thread.sleep(1000);
230           } catch (InterruptedException e) {
231             // pass
232           }
233           return mr;
234         }
235       };
236     }
237   }
238 }