View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.ipc;
19  
20  import com.google.protobuf.Message;
21  import com.google.protobuf.Message.Builder;
22  import com.google.protobuf.TextFormat;
23  
24  import io.netty.buffer.ByteBuf;
25  import io.netty.buffer.ByteBufInputStream;
26  import io.netty.buffer.ByteBufOutputStream;
27  import io.netty.channel.ChannelDuplexHandler;
28  import io.netty.channel.ChannelHandlerContext;
29  import io.netty.channel.ChannelPromise;
30  import io.netty.handler.timeout.IdleStateEvent;
31  import io.netty.util.concurrent.PromiseCombiner;
32  
33  import java.io.IOException;
34  import java.util.HashMap;
35  import java.util.Map;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.hbase.CellScanner;
40  import org.apache.hadoop.hbase.classification.InterfaceAudience;
41  import org.apache.hadoop.hbase.codec.Codec;
42  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.CellBlockMeta;
43  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ExceptionResponse;
44  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
45  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ResponseHeader;
46  import org.apache.hadoop.io.compress.CompressionCodec;
47  import org.apache.hadoop.ipc.RemoteException;
48  
49  /**
50   * The netty rpc handler.
51   */
52  @InterfaceAudience.Private
53  class NettyRpcDuplexHandler extends ChannelDuplexHandler {
54  
55    private static final Log LOG = LogFactory.getLog(NettyRpcDuplexHandler.class);
56  
57    private final NettyRpcConnection conn;
58  
59    private final CellBlockBuilder cellBlockBuilder;
60  
61    private final Codec codec;
62  
63    private final CompressionCodec compressor;
64  
65    private final Map<Integer, Call> id2Call = new HashMap<Integer, Call>();
66  
67    public NettyRpcDuplexHandler(NettyRpcConnection conn, CellBlockBuilder cellBlockBuilder,
68        Codec codec, CompressionCodec compressor) {
69      this.conn = conn;
70      this.cellBlockBuilder = cellBlockBuilder;
71      this.codec = codec;
72      this.compressor = compressor;
73  
74    }
75  
76    private void writeRequest(ChannelHandlerContext ctx, Call call, ChannelPromise promise)
77        throws IOException {
78      id2Call.put(call.id, call);
79      ByteBuf cellBlock = cellBlockBuilder.buildCellBlock(codec, compressor, call.cells, ctx.alloc());
80      CellBlockMeta cellBlockMeta;
81      if (cellBlock != null) {
82        CellBlockMeta.Builder cellBlockMetaBuilder = CellBlockMeta.newBuilder();
83        cellBlockMetaBuilder.setLength(cellBlock.writerIndex());
84        cellBlockMeta = cellBlockMetaBuilder.build();
85      } else {
86        cellBlockMeta = null;
87      }
88      RequestHeader requestHeader = IPCUtil.buildRequestHeader(call, cellBlockMeta);
89      int sizeWithoutCellBlock = IPCUtil.getTotalSizeWhenWrittenDelimited(requestHeader, call.param);
90      int totalSize = cellBlock != null ? sizeWithoutCellBlock + cellBlock.writerIndex()
91          : sizeWithoutCellBlock;
92      ByteBuf buf = ctx.alloc().buffer(sizeWithoutCellBlock + 4);
93      buf.writeInt(totalSize);
94      ByteBufOutputStream bbos = new ByteBufOutputStream(buf);
95      requestHeader.writeDelimitedTo(bbos);
96      if (call.param != null) {
97        call.param.writeDelimitedTo(bbos);
98      }
99      if (cellBlock != null) {
100       ChannelPromise withoutCellBlockPromise = ctx.newPromise();
101       ctx.write(buf, withoutCellBlockPromise);
102       ChannelPromise cellBlockPromise = ctx.newPromise();
103       ctx.write(cellBlock, cellBlockPromise);
104       PromiseCombiner combiner = new PromiseCombiner();
105       combiner.addAll(withoutCellBlockPromise, cellBlockPromise);
106       combiner.finish(promise);
107     } else {
108       ctx.write(buf, promise);
109     }
110   }
111 
112   @Override
113   public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
114       throws Exception {
115     if (msg instanceof Call) {
116       writeRequest(ctx, (Call) msg, promise);
117     } else {
118       ctx.write(msg, promise);
119     }
120   }
121 
122   private void readResponse(ChannelHandlerContext ctx, ByteBuf buf) throws IOException {
123     int totalSize = buf.readInt();
124     ByteBufInputStream in = new ByteBufInputStream(buf);
125     ResponseHeader responseHeader = ResponseHeader.parseDelimitedFrom(in);
126     int id = responseHeader.getCallId();
127     if (LOG.isTraceEnabled()) {
128       LOG.trace("got response header " + TextFormat.shortDebugString(responseHeader)
129           + ", totalSize: " + totalSize + " bytes");
130     }
131     RemoteException remoteExc;
132     if (responseHeader.hasException()) {
133       ExceptionResponse exceptionResponse = responseHeader.getException();
134       remoteExc = IPCUtil.createRemoteException(exceptionResponse);
135       if (IPCUtil.isFatalConnectionException(exceptionResponse)) {
136         // Here we will cleanup all calls so do not need to fall back, just return.
137         exceptionCaught(ctx, remoteExc);
138         return;
139       }
140     } else {
141       remoteExc = null;
142     }
143     Call call = id2Call.remove(id);
144     if (call == null) {
145       // So we got a response for which we have no corresponding 'call' here on the client-side.
146       // We probably timed out waiting, cleaned up all references, and now the server decides
147       // to return a response. There is nothing we can do w/ the response at this stage. Clean
148       // out the wire of the response so its out of the way and we can get other responses on
149       // this connection.
150       int readSoFar = IPCUtil.getTotalSizeWhenWrittenDelimited(responseHeader);
151       int whatIsLeftToRead = totalSize - readSoFar;
152       if (LOG.isDebugEnabled()) {
153         LOG.debug("Unknown callId: " + id + ", skipping over this response of " + whatIsLeftToRead
154             + " bytes");
155       }
156       return;
157     }
158     if (remoteExc != null) {
159       call.setException(remoteExc);
160       return;
161     }
162     Message value;
163     if (call.responseDefaultType != null) {
164       Builder builder = call.responseDefaultType.newBuilderForType();
165       builder.mergeDelimitedFrom(in);
166       value = builder.build();
167     } else {
168       value = null;
169     }
170     CellScanner cellBlockScanner;
171     if (responseHeader.hasCellBlockMeta()) {
172       int size = responseHeader.getCellBlockMeta().getLength();
173       // Maybe we could read directly from the ByteBuf.
174       // The problem here is that we do not know when to release it.
175       byte[] cellBlock = new byte[size];
176       buf.readBytes(cellBlock);
177       cellBlockScanner = cellBlockBuilder.createCellScanner(this.codec, this.compressor, cellBlock);
178     } else {
179       cellBlockScanner = null;
180     }
181     call.setResponse(value, cellBlockScanner);
182   }
183 
184   @Override
185   public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
186     if (msg instanceof ByteBuf) {
187       ByteBuf buf = (ByteBuf) msg;
188       try {
189         readResponse(ctx, buf);
190       } finally {
191         buf.release();
192       }
193     } else {
194       super.channelRead(ctx, msg);
195     }
196   }
197 
198   private void cleanupCalls(ChannelHandlerContext ctx, IOException error) {
199     for (Call call : id2Call.values()) {
200       call.setException(error);
201     }
202     id2Call.clear();
203   }
204 
205   @Override
206   public void channelInactive(ChannelHandlerContext ctx) throws Exception {
207     if (!id2Call.isEmpty()) {
208       cleanupCalls(ctx, new IOException("Connection closed"));
209     }
210     conn.shutdown();
211     ctx.fireChannelInactive();
212   }
213 
214   @Override
215   public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
216     if (!id2Call.isEmpty()) {
217       cleanupCalls(ctx, IPCUtil.toIOE(cause));
218     }
219     conn.shutdown();
220   }
221 
222   @Override
223   public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
224     if (evt instanceof IdleStateEvent) {
225       IdleStateEvent idleEvt = (IdleStateEvent) evt;
226       switch (idleEvt.state()) {
227         case WRITER_IDLE:
228           if (id2Call.isEmpty()) {
229             if (LOG.isDebugEnabled()) {
230               LOG.debug("shutdown connection to " + conn.remoteId().address
231                   + " because idle for a long time");
232             }
233             // It may happen that there are still some pending calls in the event loop queue and
234             // they will get a closed channel exception. But this is not a big deal as it rarely
235             // rarely happens and the upper layer could retry immediately.
236             conn.shutdown();
237           }
238           break;
239         default:
240           LOG.warn("Unrecognized idle state " + idleEvt.state());
241           break;
242       }
243     } else if (evt instanceof CallEvent) {
244       // just remove the call for now until we add other call event other than timeout and cancel.
245       id2Call.remove(((CallEvent) evt).call.id);
246     } else {
247       ctx.fireUserEventTriggered(evt);
248     }
249   }
250 }