diff --git a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java index 5ff02fc8cb5..5f03d03f362 100644 --- a/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java +++ b/framework/src/test/java/org/tron/common/backup/BackupManagerTest.java @@ -1,5 +1,6 @@ package org.tron.common.backup; +import io.netty.channel.Channel; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.InetAddress; @@ -11,7 +12,9 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; +import java.util.function.BooleanSupplier; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -35,6 +38,7 @@ public class BackupManagerTest { private BackupManager manager; private BackupServer backupServer; private BiFunction savedLookup; + private boolean backupServerClosed; @Before public void setUp() throws Exception { @@ -47,9 +51,34 @@ public void setUp() throws Exception { } @After - public void tearDown() { - InetUtil.dnsLookup = savedLookup; - Args.clearParam(); + public void tearDown() throws Exception { + Throwable failure = null; + try { + if (!backupServerClosed && backupServer != null) { + backupServer.close(); + } + } catch (Throwable t) { + failure = t; + } finally { + try { + if (manager != null) { + manager.stop(); + } + assertExecutorsTerminated(); + } catch (Throwable t) { + if (failure == null) { + failure = t; + } else { + failure.addSuppressed(t); + } + } finally { + InetUtil.dnsLookup = savedLookup; + Args.clearParam(); + } + } + if (failure != null) { + throw new AssertionError("backup test cleanup failed", failure); + } } @Test @@ -121,7 +150,7 @@ public void test() throws Exception { } @Test - public void testSendKeepAliveMessage() throws Exception { + public void testBackupServerLifecycleDuringKeepAliveInterval() throws Exception { CommonParameter parameter = CommonParameter.getInstance(); parameter.setBackupPriority(8); List members = new ArrayList<>(); @@ -134,21 +163,21 @@ public void testSendKeepAliveMessage() throws Exception { Assert.assertEquals(manager.getStatus(), BackupManager.BackupStatusEnum.MASTER); backupServer.initServer(); + awaitCondition("backup channel to become active", () -> getChannel(backupServer) != null + && getChannel(backupServer).isActive()); + awaitCondition("backup message handler assignment", () -> getFieldValue(manager, + "messageHandler") != null); manager.init(); - - Thread.sleep(parameter.getKeepAliveInterval() + 1000);//test send KeepAliveMessage - - field = manager.getClass().getDeclaredField("executorService"); - field.setAccessible(true); - ScheduledExecutorService executorService = (ScheduledExecutorService) field.get(manager); - executorService.shutdown(); - - Field field2 = backupServer.getClass().getDeclaredField("executor"); - field2.setAccessible(true); - ExecutorService executorService2 = (ExecutorService) field2.get(backupServer); - executorService2.shutdown(); + long keepAliveDeadline = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(parameter.getKeepAliveInterval() + 1000L); + awaitCondition("keep-alive interval", () -> System.nanoTime() >= keepAliveDeadline); Assert.assertEquals(BackupManager.BackupStatusEnum.INIT, manager.getStatus()); + Channel channel = getChannel(backupServer); + backupServer.close(); + backupServerClosed = true; + Assert.assertFalse("backup channel must close", channel.isOpen()); + assertExecutorsTerminated(); } // ===== domain-handling tests for init() ===== @@ -254,4 +283,44 @@ private void invokeRefreshMemberIps(BackupManager mgr) throws Exception { m.setAccessible(true); m.invoke(mgr); } + + private Channel getChannel(BackupServer server) { + try { + return getField(server, "channel"); + } catch (Exception e) { + throw new AssertionError("cannot inspect backup channel", e); + } + } + + private Object getFieldValue(Object target, String name) { + try { + return getField(target, name); + } catch (Exception e) { + throw new AssertionError("cannot inspect " + name, e); + } + } + + private void assertExecutorsTerminated() throws Exception { + if (manager == null || backupServer == null) { + return; + } + ScheduledExecutorService managerExecutor = getField(manager, "executorService"); + Assert.assertTrue("backup manager executor must terminate", managerExecutor.isTerminated()); + ExecutorService serverExecutor = getField(backupServer, "executor"); + if (serverExecutor != null) { + Assert.assertTrue("backup server executor must terminate", serverExecutor.isTerminated()); + } + } + + private void awaitCondition(String description, BooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (condition.getAsBoolean()) { + return; + } + Thread.sleep(20); + } + Assert.fail("timed out waiting for " + description); + } + } diff --git a/framework/src/test/java/org/tron/common/backup/BackupServerTest.java b/framework/src/test/java/org/tron/common/backup/BackupServerTest.java index 50778970d87..cd300ba212a 100644 --- a/framework/src/test/java/org/tron/common/backup/BackupServerTest.java +++ b/framework/src/test/java/org/tron/common/backup/BackupServerTest.java @@ -1,8 +1,14 @@ package org.tron.common.backup; +import io.netty.channel.Channel; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -23,6 +29,8 @@ public class BackupServerTest { @Rule public Timeout globalTimeout = Timeout.seconds(60); private BackupServer backupServer; + private BackupManager backupManager; + private boolean backupServerClosed; @Before public void setUp() throws Exception { @@ -32,21 +40,91 @@ public void setUp() throws Exception { List members = new ArrayList<>(); members.add("127.0.0.2"); CommonParameter.getInstance().setBackupMembers(members); - BackupManager backupManager = new BackupManager(); + backupManager = new BackupManager(); backupManager.init(); backupServer = new BackupServer(backupManager); } @After - public void tearDown() { - backupServer.close(); - Args.clearParam(); + public void tearDown() throws Exception { + Throwable failure = null; + try { + if (!backupServerClosed && backupServer != null) { + backupServer.close(); + } + } catch (Throwable t) { + failure = t; + } finally { + try { + if (backupManager != null) { + backupManager.stop(); + } + assertExecutorsTerminated(); + } catch (Throwable t) { + if (failure == null) { + failure = t; + } else { + failure.addSuppressed(t); + } + } finally { + Args.clearParam(); + } + } + if (failure != null) { + throw new AssertionError("backup test cleanup failed", failure); + } } @Test(timeout = 60_000) - public void test() throws InterruptedException { + public void test() throws Exception { backupServer.initServer(); - // wait for the server to start so channel is assigned before close() is called - Thread.sleep(1000); + awaitCondition("backup channel to become active", () -> getChannel() != null + && getChannel().isActive()); + Channel channel = getChannel(); + Assert.assertTrue("backup channel must be active after startup", channel.isActive()); + + backupServer.close(); + backupServerClosed = true; + + Assert.assertFalse("backup channel must close", channel.isOpen()); + assertExecutorsTerminated(); + } + + private Channel getChannel() { + try { + return getField(backupServer, "channel"); + } catch (Exception e) { + throw new AssertionError("cannot inspect backup channel", e); + } + } + + private void assertExecutorsTerminated() throws Exception { + if (backupManager == null || backupServer == null) { + return; + } + ExecutorService managerExecutor = getField(backupManager, "executorService"); + Assert.assertTrue("backup manager executor must terminate", managerExecutor.isTerminated()); + ExecutorService serverExecutor = getField(backupServer, "executor"); + if (serverExecutor != null) { + Assert.assertTrue("backup server executor must terminate", serverExecutor.isTerminated()); + } + } + + private void awaitCondition(String description, BooleanSupplier condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (condition.getAsBoolean()) { + return; + } + Thread.sleep(20); + } + Assert.fail("timed out waiting for " + description); + } + + @SuppressWarnings("unchecked") + private T getField(Object target, String name) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + return (T) field.get(target); } } diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/TransactionsMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/TransactionsMsgHandlerTest.java index ed2121d360f..d0ebe15290f 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/TransactionsMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/TransactionsMsgHandlerTest.java @@ -9,11 +9,13 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; -import lombok.Getter; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.BeforeClass; @@ -23,7 +25,6 @@ import org.tron.common.TestConstants; import org.tron.common.runtime.TvmTestUtils; import org.tron.common.utils.ByteArray; -import org.tron.common.utils.ReflectUtils; import org.tron.core.ChainBaseManager; import org.tron.core.config.args.Args; import org.tron.core.exception.P2pException; @@ -48,6 +49,7 @@ public static void init() { @Test public void testProcessMessage() { TransactionsMsgHandler transactionsMsgHandler = new TransactionsMsgHandler(); + ExecutorService originalPool = null; try { transactionsMsgHandler.init(); @@ -86,11 +88,16 @@ public void testProcessMessage() { transactionList.add(trx); transactionsMsgHandler.processMessage(peer, new TransactionsMessage(transactionList)); Assert.assertNull(advInvRequest.get(item)); - //Thread.sleep(10); - BlockingQueue smartContractQueue = - new LinkedBlockingQueue(2); - smartContractQueue.offer(new TrxEvent(null, null)); - smartContractQueue.offer(new TrxEvent(null, null)); + + CountDownLatch smartContractSubmitted = new CountDownLatch(1); + ExecutorService mockPool = Mockito.mock(ExecutorService.class); + Future submittedTask = Mockito.mock(Future.class); + Mockito.when(mockPool.submit(Mockito.any(Runnable.class))).thenAnswer(invocation -> { + smartContractSubmitted.countDown(); + return submittedTask; + }); + originalPool = replaceTrxHandlePool(transactionsMsgHandler, mockPool); + BlockingQueue smartContractQueue = new LinkedBlockingQueue<>(1); Field field1 = TransactionsMsgHandler.class.getDeclaredField("smartContractQueue"); field1.setAccessible(true); field1.set(transactionsMsgHandler, smartContractQueue); @@ -99,15 +106,27 @@ public void testProcessMessage() { ByteArray.fromHexString("121212a9cf"), ByteArray.fromHexString("123456"), 100, 100000000, 0, 0); + Protocol.Transaction trx3 = TvmTestUtils.generateTriggerSmartContractAndGetTransaction( + ByteArray.fromHexString("121212a9cf"), + ByteArray.fromHexString("121212a9cf"), + ByteArray.fromHexString("123457"), + 100, 100000000, 0, 0); Map advInvRequest1 = new ConcurrentHashMap<>(); Item item1 = new Item(new TransactionMessage(trx1).getMessageId(), Protocol.Inventory.InventoryType.TRX); advInvRequest1.put(item1, 0L); + Item item3 = new Item(new TransactionMessage(trx3).getMessageId(), + Protocol.Inventory.InventoryType.TRX); + advInvRequest1.put(item3, 0L); Mockito.when(peer.getAdvInvRequest()).thenReturn(advInvRequest1); List transactionList1 = new ArrayList<>(); transactionList1.add(trx1); + transactionList1.add(trx3); transactionsMsgHandler.processMessage(peer, new TransactionsMessage(transactionList1)); - Assert.assertNull(advInvRequest.get(item1)); + Assert.assertNull(advInvRequest1.get(item1)); + Assert.assertNull(advInvRequest1.get(item3)); + Assert.assertTrue("smart-contract scheduler did not submit work", + smartContractSubmitted.await(3, TimeUnit.SECONDS)); // test 0 contract Protocol.Transaction trx2 = Protocol.Transaction.newBuilder().setRawData( @@ -132,37 +151,40 @@ public void testProcessMessage() { Assert.assertTrue(true); } } catch (Exception e) { - Assert.fail(); + Assert.fail(e.getMessage()); } finally { - transactionsMsgHandler.close(); + closeHandlerAndOriginalPool(transactionsMsgHandler, originalPool); } } @Test public void testProcessMessageAfterClose() throws Exception { TransactionsMsgHandler handler = new TransactionsMsgHandler(); - handler.init(); - handler.close(); + try { + handler.init(); + handler.close(); - PeerConnection peer = Mockito.mock(PeerConnection.class); - TransactionsMessage msg = Mockito.mock(TransactionsMessage.class); + PeerConnection peer = Mockito.mock(PeerConnection.class); + TransactionsMessage msg = Mockito.mock(TransactionsMessage.class); - handler.processMessage(peer, msg); + handler.processMessage(peer, msg); - Mockito.verify(msg, Mockito.never()).getTransactions(); - Mockito.verifyNoInteractions(peer); + Mockito.verify(msg, Mockito.never()).getTransactions(); + Mockito.verifyNoInteractions(peer); + } finally { + handler.close(); + } } @Test public void testRejectedExecution() throws Exception { TransactionsMsgHandler handler = new TransactionsMsgHandler(); + ExecutorService originalPool = null; try { ExecutorService mockPool = Mockito.mock(ExecutorService.class); Mockito.when(mockPool.submit(Mockito.any(Runnable.class))) .thenThrow(new RejectedExecutionException("pool closed")); - Field poolField = TransactionsMsgHandler.class.getDeclaredField("trxHandlePool"); - poolField.setAccessible(true); - poolField.set(handler, mockPool); + originalPool = replaceTrxHandlePool(handler, mockPool); PeerConnection peer = Mockito.mock(PeerConnection.class); TransactionsMessage msg = buildTransferMessage(2); @@ -172,26 +194,26 @@ public void testRejectedExecution() throws Exception { Mockito.verify(mockPool, Mockito.times(1)).submit(Mockito.any(Runnable.class)); } finally { - handler.close(); + closeHandlerAndOriginalPool(handler, originalPool); } } @Test public void testCloseDuringProcessing() throws Exception { TransactionsMsgHandler handler = new TransactionsMsgHandler(); + ExecutorService originalPool = null; try { Field closedField = TransactionsMsgHandler.class.getDeclaredField("isClosed"); closedField.setAccessible(true); ExecutorService mockPool = Mockito.mock(ExecutorService.class); + Future submittedTask = Mockito.mock(Future.class); // on the first submit, flip isClosed to true so the second iteration breaks Mockito.when(mockPool.submit(Mockito.any(Runnable.class))).thenAnswer(inv -> { closedField.set(handler, true); - return null; + return submittedTask; }); - Field poolField = TransactionsMsgHandler.class.getDeclaredField("trxHandlePool"); - poolField.setAccessible(true); - poolField.set(handler, mockPool); + originalPool = replaceTrxHandlePool(handler, mockPool); PeerConnection peer = Mockito.mock(PeerConnection.class); TransactionsMessage msg = buildTransferMessage(2); @@ -200,7 +222,7 @@ public void testCloseDuringProcessing() throws Exception { Mockito.verify(mockPool, Mockito.times(1)).submit(Mockito.any(Runnable.class)); } finally { - handler.close(); + closeHandlerAndOriginalPool(handler, originalPool); } } @@ -234,6 +256,34 @@ private void stubAdvInvRequest(PeerConnection peer, TransactionsMessage msg) { Mockito.when(peer.getAdvInvRequest()).thenReturn(advInvRequest); } + private ExecutorService replaceTrxHandlePool(TransactionsMsgHandler handler, ExecutorService pool) + throws Exception { + Field poolField = TransactionsMsgHandler.class.getDeclaredField("trxHandlePool"); + poolField.setAccessible(true); + ExecutorService originalPool = (ExecutorService) poolField.get(handler); + poolField.set(handler, pool); + return originalPool; + } + + private void closeHandlerAndOriginalPool(TransactionsMsgHandler handler, + ExecutorService originalPool) { + try { + handler.close(); + } finally { + if (originalPool != null) { + originalPool.shutdown(); + try { + if (!originalPool.awaitTermination(5, TimeUnit.SECONDS)) { + originalPool.shutdownNow(); + } + } catch (InterruptedException e) { + originalPool.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + } + } + @Test public void testHandleTransaction() throws Exception { TransactionsMsgHandler handler = new TransactionsMsgHandler(); @@ -425,38 +475,25 @@ public void testInvalidSigLength() throws Exception { @Test public void testIsBusyWithCachedTransactions() throws Exception { TransactionsMsgHandler handler = new TransactionsMsgHandler(); + try { + int threshold = Args.getInstance().getMaxTrxCacheSize(); + TronNetDelegate tronNetDelegateMock = Mockito.mock(TronNetDelegate.class); + Field field = TransactionsMsgHandler.class.getDeclaredField("tronNetDelegate"); + field.setAccessible(true); + field.set(handler, tronNetDelegateMock); - int threshold = Args.getInstance().getMaxTrxCacheSize(); - TronNetDelegate tronNetDelegateMock = Mockito.mock(TronNetDelegate.class); - Field field = TransactionsMsgHandler.class.getDeclaredField("tronNetDelegate"); - field.setAccessible(true); - field.set(handler, tronNetDelegateMock); - - // queue and smartContractQueue are empty, but cached size > threshold - Mockito.when(tronNetDelegateMock.getCachedTransactionSize()).thenReturn(threshold + 1); - Assert.assertTrue(handler.isBusy()); - - // boundary: cached size == threshold, isBusy() uses strict >, so not busy - Mockito.when(tronNetDelegateMock.getCachedTransactionSize()).thenReturn(threshold); - Assert.assertFalse(handler.isBusy()); - - Mockito.when(tronNetDelegateMock.getCachedTransactionSize()).thenReturn(0); - Assert.assertFalse(handler.isBusy()); - } - - class TrxEvent { + // queue and smartContractQueue are empty, but cached size > threshold + Mockito.when(tronNetDelegateMock.getCachedTransactionSize()).thenReturn(threshold + 1); + Assert.assertTrue(handler.isBusy()); - @Getter - private PeerConnection peer; - @Getter - private TransactionMessage msg; - @Getter - private long time; + // boundary: cached size == threshold, isBusy() uses strict >, so not busy + Mockito.when(tronNetDelegateMock.getCachedTransactionSize()).thenReturn(threshold); + Assert.assertFalse(handler.isBusy()); - public TrxEvent(PeerConnection peer, TransactionMessage msg) { - this.peer = peer; - this.msg = msg; - this.time = System.currentTimeMillis(); + Mockito.when(tronNetDelegateMock.getCachedTransactionSize()).thenReturn(0); + Assert.assertFalse(handler.isBusy()); + } finally { + handler.close(); } } }