Skip to content

Commit e5fdef5

Browse files
caroliney14xcangCRM
caroliney14
authored and
xcangCRM
committed
Provide an API to get list of successful regions and total expected regions in Canary (#612)
Signed-off-by: Xu Cang <[email protected]>
1 parent 3f84591 commit e5fdef5

File tree

2 files changed

+206
-12
lines changed

2 files changed

+206
-12
lines changed

‎hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,13 +120,19 @@ public interface Sink {
120120
public long incWriteFailureCount();
121121
public Map<String,String> getWriteFailures();
122122
public void updateWriteFailures(String regionName, String serverName);
123+
public long getReadSuccessCount();
124+
public long incReadSuccessCount();
125+
public long getWriteSuccessCount();
126+
public long incWriteSuccessCount();
123127
}
124128

125129
// Simple implementation of canary sink that allows to plot on
126130
// file or standard output timings or failures.
127131
public static class StdOutSink implements Sink {
128132
private AtomicLong readFailureCount = new AtomicLong(0),
129-
writeFailureCount = new AtomicLong(0);
133+
writeFailureCount = new AtomicLong(0),
134+
readSuccessCount = new AtomicLong(0),
135+
writeSuccessCount = new AtomicLong(0);
130136

131137
private Map<String, String> readFailures = new ConcurrentHashMap<String, String>();
132138
private Map<String, String> writeFailures = new ConcurrentHashMap<String, String>();
@@ -170,6 +176,26 @@ public Map<String, String> getWriteFailures() {
170176
public void updateWriteFailures(String regionName, String serverName) {
171177
writeFailures.put(regionName, serverName);
172178
}
179+
180+
@Override
181+
public long getReadSuccessCount() {
182+
return readSuccessCount.get();
183+
}
184+
185+
@Override
186+
public long incReadSuccessCount() {
187+
return readSuccessCount.incrementAndGet();
188+
}
189+
190+
@Override
191+
public long getWriteSuccessCount() {
192+
return writeSuccessCount.get();
193+
}
194+
195+
@Override
196+
public long incWriteSuccessCount() {
197+
return writeSuccessCount.incrementAndGet();
198+
}
173199
}
174200

175201
public static class RegionServerStdOutSink extends StdOutSink {
@@ -202,6 +228,7 @@ public static class RegionStdOutSink extends StdOutSink {
202228

203229
private Map<String, AtomicLong> perTableReadLatency = new HashMap<>();
204230
private AtomicLong writeLatency = new AtomicLong();
231+
private Map<String, RegionTaskResult> regionMap = new ConcurrentHashMap<>();
205232

206233
public void publishReadFailure(ServerName serverName, HRegionInfo region, Exception e) {
207234
incReadFailureCount();
@@ -215,6 +242,10 @@ public void publishReadFailure(ServerName serverName, HRegionInfo region, HColum
215242
}
216243

217244
public void publishReadTiming(ServerName serverName, HRegionInfo region, HColumnDescriptor column, long msTime) {
245+
incReadSuccessCount();
246+
RegionTaskResult res = this.regionMap.get(region.getRegionNameAsString());
247+
res.setReadSuccess();
248+
res.setReadLatency(msTime);
218249
LOG.info(String.format("read from region %s on regionserver %s column family %s in %dms",
219250
region.getRegionNameAsString(), serverName, column.getNameAsString(), msTime));
220251
}
@@ -231,6 +262,10 @@ public void publishWriteFailure(ServerName serverName, HRegionInfo region, HColu
231262
}
232263

233264
public void publishWriteTiming(ServerName serverName, HRegionInfo region, HColumnDescriptor column, long msTime) {
265+
incWriteSuccessCount();
266+
RegionTaskResult res = this.regionMap.get(region.getRegionNameAsString());
267+
res.setWriteSuccess();
268+
res.setWriteLatency(msTime);
234269
LOG.info(String.format("write to region %s on regionserver %s column family %s in %dms",
235270
region.getRegionNameAsString(), serverName, column.getNameAsString(), msTime));
236271
}
@@ -252,6 +287,14 @@ public void initializeWriteLatency() {
252287
public AtomicLong getWriteLatency() {
253288
return this.writeLatency;
254289
}
290+
291+
public Map<String, RegionTaskResult> getRegionMap() {
292+
return this.regionMap;
293+
}
294+
295+
public int getTotalExpectedRegions() {
296+
return this.regionMap.size();
297+
}
255298
}
256299

257300
static class ZookeeperTask implements Callable<Void> {
@@ -883,6 +926,96 @@ private void printUsageAndExit() {
883926
System.exit(USAGE_EXIT_CODE);
884927
}
885928

929+
/**
930+
* Canary region mode-specific data structure which stores information about each region
931+
* to be scanned
932+
*/
933+
public static class RegionTaskResult {
934+
private HRegionInfo region;
935+
private TableName tableName;
936+
private ServerName serverName;
937+
private AtomicLong readLatency = null;
938+
private AtomicLong writeLatency = null;
939+
private boolean readSuccess = false;
940+
private boolean writeSuccess = false;
941+
942+
public RegionTaskResult(HRegionInfo region, TableName tableName, ServerName serverName) {
943+
this.region = region;
944+
this.tableName = tableName;
945+
this.serverName = serverName;
946+
}
947+
948+
public HRegionInfo getRegionInfo() {
949+
return this.region;
950+
}
951+
952+
public String getRegionNameAsString() {
953+
return this.region.getRegionNameAsString();
954+
}
955+
956+
public TableName getTableName() {
957+
return this.tableName;
958+
}
959+
960+
public String getTableNameAsString() {
961+
return this.tableName.getNameAsString();
962+
}
963+
964+
public ServerName getServerName() {
965+
return this.serverName;
966+
}
967+
968+
public String getServerNameAsString() {
969+
return this.serverName.getServerName();
970+
}
971+
972+
public long getReadLatency() {
973+
if (this.readLatency == null) {
974+
return -1;
975+
}
976+
return this.readLatency.get();
977+
}
978+
979+
public void setReadLatency(long readLatency) {
980+
if (this.readLatency != null) {
981+
this.readLatency.set(readLatency);
982+
} else {
983+
this.readLatency = new AtomicLong(readLatency);
984+
}
985+
}
986+
987+
public long getWriteLatency() {
988+
if (this.writeLatency == null) {
989+
return -1;
990+
}
991+
return this.writeLatency.get();
992+
}
993+
994+
public void setWriteLatency(long writeLatency) {
995+
if (this.writeLatency != null) {
996+
this.writeLatency.set(writeLatency);
997+
} else {
998+
this.writeLatency = new AtomicLong(writeLatency);
999+
}
1000+
}
1001+
1002+
public boolean isReadSuccess() {
1003+
return this.readSuccess;
1004+
}
1005+
1006+
public void setReadSuccess() {
1007+
this.readSuccess = true;
1008+
}
1009+
1010+
public boolean isWriteSuccess() {
1011+
return this.writeSuccess;
1012+
}
1013+
1014+
public void setWriteSuccess() {
1015+
this.writeSuccess = true;
1016+
}
1017+
}
1018+
8861019
/**
8871020
* A Factory method for {@link Monitor}.
8881021
* Can be overridden by user.
@@ -1295,6 +1428,9 @@ private static List<Future<Void>> sniff(final Admin admin, final Sink sink,
12951428
HRegionInfo region = location.getRegionInfo();
12961429
tasks.add(new RegionTask(admin.getConnection(), region, rs, (RegionStdOutSink) sink, taskType, rawScanEnabled,
12971430
rwLatency));
1431+
Map<String, RegionTaskResult> regionMap = ((RegionStdOutSink) sink).getRegionMap();
1432+
regionMap.put(region.getRegionNameAsString(), new RegionTaskResult(region,
1433+
region.getTable(), rs));
12981434
}
12991435
} finally {
13001436
if (regionLocator != null) {

‎hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java‎

Lines changed: 69 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,25 @@
1919

2020
package org.apache.hadoop.hbase.tool;
2121

22+
import static org.junit.Assert.assertEquals;
23+
import static org.junit.Assert.assertFalse;
24+
import static org.junit.Assert.assertNotEquals;
25+
import static org.junit.Assert.assertNotNull;
26+
import static org.junit.Assert.assertTrue;
27+
import static org.mockito.Matchers.anyLong;
28+
import static org.mockito.Matchers.argThat;
29+
import static org.mockito.Matchers.eq;
30+
import static org.mockito.Matchers.isA;
31+
import static org.mockito.Mockito.atLeastOnce;
32+
import static org.mockito.Mockito.never;
33+
import static org.mockito.Mockito.spy;
34+
import static org.mockito.Mockito.times;
35+
import static org.mockito.Mockito.verify;
36+
37+
import java.util.Map;
38+
import java.util.concurrent.ExecutorService;
39+
import java.util.concurrent.ScheduledThreadPoolExecutor;
40+
2241
import org.apache.hadoop.hbase.HBaseTestingUtility;
2342
import org.apache.hadoop.hbase.HColumnDescriptor;
2443
import org.apache.hadoop.hbase.HConstants;
@@ -34,7 +53,6 @@
3453
import org.apache.log4j.Appender;
3554
import org.apache.log4j.LogManager;
3655
import org.apache.log4j.spi.LoggingEvent;
37-
import com.google.common.collect.Iterables;
3856
import org.junit.After;
3957
import org.junit.Before;
4058
import org.junit.Ignore;
@@ -45,16 +63,7 @@
4563
import org.mockito.Mock;
4664
import org.mockito.runners.MockitoJUnitRunner;
4765

48-
import java.util.concurrent.ExecutorService;
49-
import java.util.concurrent.ScheduledThreadPoolExecutor;
50-
51-
import static org.junit.Assert.assertNotEquals;
52-
import static org.mockito.Matchers.anyLong;
53-
import static org.mockito.Matchers.eq;
54-
import static org.mockito.Matchers.isA;
55-
import static org.mockito.Matchers.argThat;
56-
import static org.junit.Assert.assertEquals;
57-
import static org.mockito.Mockito.*;
66+
import com.google.common.collect.Iterables;
5867

5968
@RunWith(MockitoJUnitRunner.class)
6069
@Category({MediumTests.class})
@@ -113,6 +122,55 @@ public void testBasicCanaryWorks() throws Exception {
113122
verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
114123
}
115124

125+
@Test
126+
public void testCanaryRegionTaskResult() throws Exception {
127+
TableName tableName = TableName.valueOf("testCanaryRegionTaskResult");
128+
HTable table = testingUtility.createTable(tableName, new byte[][]{FAMILY});
129+
// insert some test rows
130+
for (int i = 0; i < 1000; i++) {
131+
byte[] iBytes = Bytes.toBytes(i);
132+
Put p = new Put(iBytes);
133+
p.addColumn(FAMILY, COLUMN, iBytes);
134+
table.put(p);
135+
}
136+
ExecutorService executor = new ScheduledThreadPoolExecutor(1);
137+
Canary.RegionStdOutSink sink = spy(new Canary.RegionStdOutSink());
138+
Canary canary = new Canary(executor, sink);
139+
String[] args = {"-writeSniffing", "-t", "10000", "testCanaryRegionTaskResult"};
140+
assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
141+
142+
assertTrue("verify read success count > 0", sink.getReadSuccessCount() > 0);
143+
assertTrue("verify write success count > 0", sink.getWriteSuccessCount() > 0);
144+
verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(HRegionInfo.class),
145+
isA(HColumnDescriptor.class), anyLong());
146+
verify(sink, atLeastOnce()).publishWriteTiming(isA(ServerName.class), isA(HRegionInfo.class),
147+
isA(HColumnDescriptor.class), anyLong());
148+
149+
assertTrue("canary should expect to scan at least 1 region",
150+
sink.getTotalExpectedRegions() > 0);
151+
Map<String, Canary.RegionTaskResult> regionMap = sink.getRegionMap();
152+
assertFalse("verify region map has size > 0", regionMap.isEmpty());
153+
154+
for (String regionName : regionMap.keySet()) {
155+
Canary.RegionTaskResult res = regionMap.get(regionName);
156+
assertNotNull("verify each expected region has a RegionTaskResult object in the map", res);
157+
assertNotNull("verify getRegionNameAsString()", regionName);
158+
assertNotNull("verify getRegionInfo()", res.getRegionInfo());
159+
assertNotNull("verify getTableName()", res.getTableName());
160+
assertNotNull("verify getTableNameAsString()", res.getTableNameAsString());
161+
assertNotNull("verify getServerName()", res.getServerName());
162+
assertNotNull("verify getServerNameAsString()", res.getServerNameAsString());
163+
164+
if (regionName.contains(Canary.DEFAULT_WRITE_TABLE_NAME.getNameAsString())) {
165+
assertTrue("write to region " + regionName + " succeeded", res.isWriteSuccess());
166+
assertTrue("write took some time", res.getWriteLatency() > -1);
167+
} else {
168+
assertTrue("read from region " + regionName + " succeeded", res.isReadSuccess());
169+
assertTrue("read took some time", res.getReadLatency() > -1);
170+
}
171+
}
172+
}
173+
116174
@Test
117175
@Ignore("Intermittent argument matching failures, see HBASE-18813")
118176
public void testReadTableTimeouts() throws Exception {

0 commit comments

Comments
 (0)