1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.fs;
19
20 import java.lang.reflect.InvocationTargetException;
21 import java.lang.reflect.Method;
22 import java.net.BindException;
23 import java.net.ServerSocket;
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.BlockLocation;
28 import org.apache.hadoop.fs.FSDataInputStream;
29 import org.apache.hadoop.fs.FSDataOutputStream;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.testclassification.LargeTests;
35 import org.apache.hadoop.hdfs.DistributedFileSystem;
36 import org.apache.hadoop.hdfs.MiniDFSCluster;
37 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
38 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
39 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
40 import org.apache.hadoop.hdfs.server.datanode.DataNode;
41 import org.junit.After;
42 import org.junit.Assert;
43 import org.junit.Before;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50
51
52
53
54
55 @Category(LargeTests.class)
56 public class TestBlockReorder {
57 private static final Log LOG = LogFactory.getLog(TestBlockReorder.class);
58
59 private Configuration conf;
60 private MiniDFSCluster cluster;
61 private HBaseTestingUtility htu;
62 private DistributedFileSystem dfs;
63 private static final String host1 = "host1";
64 private static final String host2 = "host2";
65 private static final String host3 = "host3";
66 private static Path rootDir;
67 private static Path walRootDir;
68
69 @Before
70 public void setUp() throws Exception {
71 htu = new HBaseTestingUtility();
72 htu.getConfiguration().setInt("dfs.blocksize", 1024);
73 htu.getConfiguration().setBoolean("dfs.support.append", true);
74 htu.getConfiguration().setInt("dfs.replication", 3);
75 htu.startMiniDFSCluster(3,
76 new String[]{"/r1", "/r2", "/r3"}, new String[]{host1, host2, host3});
77
78 conf = htu.getConfiguration();
79 cluster = htu.getDFSCluster();
80 dfs = (DistributedFileSystem) FileSystem.get(conf);
81 rootDir = htu.createRootDir();
82 walRootDir = htu.createWALRootDir();
83 }
84
85 @After
86 public void tearDownAfterClass() throws Exception {
87 dfs.delete(rootDir, true);
88 dfs.delete(walRootDir, true);
89 htu.shutdownMiniCluster();
90 }
91
92
93
94
95 @Test
96 public void testBlockLocationReorder() throws Exception {
97 Path p = new Path("hello");
98
99 Assert.assertTrue((short) cluster.getDataNodes().size() > 1);
100 final int repCount = 2;
101
102
103 FSDataOutputStream fop = dfs.create(p, (short) repCount);
104 final double toWrite = 875.5613;
105 fop.writeDouble(toWrite);
106 fop.close();
107
108
109 long start = System.currentTimeMillis();
110 FSDataInputStream fin = dfs.open(p);
111 Assert.assertTrue(toWrite == fin.readDouble());
112 long end = System.currentTimeMillis();
113 LOG.info("readtime= " + (end - start));
114 fin.close();
115 Assert.assertTrue((end - start) < 30 * 1000);
116
117
118
119 FileStatus f = dfs.getFileStatus(p);
120 BlockLocation[] lbs;
121 do {
122 lbs = dfs.getFileBlockLocations(f, 0, 1);
123 } while (lbs.length != 1 && lbs[0].getLength() != repCount);
124 final String name = lbs[0].getNames()[0];
125 Assert.assertTrue(name.indexOf(':') > 0);
126 String portS = name.substring(name.indexOf(':') + 1);
127 final int port = Integer.parseInt(portS);
128 LOG.info("port= " + port);
129 int ipcPort = -1;
130
131
132
133 boolean ok = false;
134 final String lookup = lbs[0].getHosts()[0];
135 StringBuilder sb = new StringBuilder();
136 for (DataNode dn : cluster.getDataNodes()) {
137 final String dnName = getHostName(dn);
138 sb.append(dnName).append(' ');
139 if (lookup.equals(dnName)) {
140 ok = true;
141 LOG.info("killing datanode " + name + " / " + lookup);
142 ipcPort = dn.ipcServer.getListenerAddress().getPort();
143 dn.shutdown();
144 LOG.info("killed datanode " + name + " / " + lookup);
145 break;
146 }
147 }
148 Assert.assertTrue(
149 "didn't find the server to kill, was looking for " + lookup + " found " + sb, ok);
150 LOG.info("ipc port= " + ipcPort);
151
152
153 Assert.assertTrue(HFileSystem.addLocationsOrderInterceptor(conf,
154 new HFileSystem.ReorderBlocks() {
155 @Override
156 public void reorderBlocks(Configuration c, LocatedBlocks lbs, String src) {
157 for (LocatedBlock lb : lbs.getLocatedBlocks()) {
158 if (lb.getLocations().length > 1) {
159 DatanodeInfo[] infos = lb.getLocations();
160 if (infos[0].getHostName().equals(lookup)) {
161 LOG.info("HFileSystem bad host, inverting");
162 DatanodeInfo tmp = infos[0];
163 infos[0] = infos[1];
164 infos[1] = tmp;
165 }
166 }
167 }
168 }
169 }));
170
171
172 final int retries = 10;
173 ServerSocket ss = null;
174 ServerSocket ssI;
175 try {
176 ss = new ServerSocket(port);
177 ssI = new ServerSocket(ipcPort);
178 } catch (BindException be) {
179 LOG.warn("Got bind exception trying to set up socket on " + port + " or " + ipcPort +
180 ", this means that the datanode has not closed the socket or" +
181 " someone else took it. It may happen, skipping this test for this time.", be);
182 if (ss != null) {
183 ss.close();
184 }
185 return;
186 }
187
188
189
190 for (int i = 0; i < retries; i++) {
191 start = System.currentTimeMillis();
192
193 fin = dfs.open(p);
194 Assert.assertTrue(toWrite == fin.readDouble());
195 fin.close();
196 end = System.currentTimeMillis();
197 LOG.info("HFileSystem readtime= " + (end - start));
198 Assert.assertFalse("We took too much time to read", (end - start) > 60000);
199 }
200
201 ss.close();
202 ssI.close();
203 }
204
205
206
207
208 private String getHostName(DataNode dn) throws InvocationTargetException, IllegalAccessException {
209 Method m;
210 try {
211 m = DataNode.class.getMethod("getDisplayName");
212 } catch (NoSuchMethodException e) {
213 try {
214 m = DataNode.class.getMethod("getHostName");
215 } catch (NoSuchMethodException e1) {
216 throw new RuntimeException(e1);
217 }
218 }
219
220 String res = (String) m.invoke(dn);
221 if (res.contains(":")) {
222 return res.split(":")[0];
223 } else {
224 return res;
225 }
226 }
227 }