Introduce test rule to work around intermittent mockk failures

3 years ago · 687209fee8
parent ff0befdd1d
commit 687209fee8
7 changed files with 90 additions and 12 deletions
--- a/app/src/test/java/org/mozilla/fenix/browser/FenixSnackbarDelegateTest.kt
+++ b/app/src/test/java/org/mozilla/fenix/browser/FenixSnackbarDelegateTest.kt
@ -14,11 +14,12 @@ import io.mockk.unmockkObject
 import io.mockk.verify
 import org.junit.After
 import org.junit.Before
-import org.junit.Ignore
+import org.junit.Rule
 import org.junit.Test
 import org.mozilla.fenix.R
 import org.mozilla.fenix.components.FenixSnackbar
 import org.mozilla.fenix.components.FenixSnackbar.Companion.LENGTH_SHORT
+import org.mozilla.fenix.helpers.MockkRetryTestRule

 class FenixSnackbarDelegateTest {

@ -26,6 +27,9 @@ class FenixSnackbarDelegateTest {
    @MockK(relaxed = true) private lateinit var snackbar: FenixSnackbar
    private lateinit var delegate: FenixSnackbarDelegate

+    @get:Rule
+    val mockkRule = MockkRetryTestRule()
+
    @Before
    fun setup() {
        MockKAnnotations.init(this)
@ -62,7 +66,6 @@ class FenixSnackbarDelegateTest {
    }

    @Test
-    @Ignore("Intermittent failing test on JDK11: https://github.com/mockk/mockk/issues/598")
    fun `show with listener but no action`() {
        delegate.show(
            snackBarParentView = mockk(),
@ -93,7 +96,6 @@ class FenixSnackbarDelegateTest {
    }

    @Test
-    @Ignore("Intermittent failing test on JDK11: https://github.com/mockk/mockk/issues/598")
    fun `show with listener and action`() {
        val listener = mockk<(View) -> Unit>(relaxed = true)
        delegate.show(
--- a/app/src/test/java/org/mozilla/fenix/browser/browsingmode/DefaultBrowsingModeManagerTest.kt
+++ b/app/src/test/java/org/mozilla/fenix/browser/browsingmode/DefaultBrowsingModeManagerTest.kt
@ -12,8 +12,9 @@ import io.mockk.just
 import io.mockk.verify
 import org.junit.Assert.assertEquals
 import org.junit.Before
-import org.junit.Ignore
+import org.junit.Rule
 import org.junit.Test
+import org.mozilla.fenix.helpers.MockkRetryTestRule
 import org.mozilla.fenix.utils.Settings

 class DefaultBrowsingModeManagerTest {
@ -24,6 +25,9 @@ class DefaultBrowsingModeManagerTest {

    private val initMode = BrowsingMode.Normal

+    @get:Rule
+    val mockkRule = MockkRetryTestRule()
+
    @Before
    fun before() {
        MockKAnnotations.init(this)
@ -33,7 +37,6 @@ class DefaultBrowsingModeManagerTest {
    }

    @Test
-    @Ignore("Intermittent failing test on JDK11: https://github.com/mockk/mockk/issues/598")
    fun `WHEN mode is updated THEN callback is invoked`() {
        verify(exactly = 0) { callback.invoke(any()) }

@ -50,7 +53,6 @@ class DefaultBrowsingModeManagerTest {
    }

    @Test
-    @Ignore("Intermittent failing test on JDK11: https://github.com/mockk/mockk/issues/598")
    fun `WHEN mode is updated THEN it should be returned from get`() {
        assertEquals(BrowsingMode.Normal, manager.mode)

--- a/app/src/test/java/org/mozilla/fenix/exceptions/viewholders/ExceptionsListItemViewHolderTest.kt
+++ b/app/src/test/java/org/mozilla/fenix/exceptions/viewholders/ExceptionsListItemViewHolderTest.kt
@ -17,9 +17,10 @@ import mozilla.components.browser.icons.IconRequest
 import mozilla.components.ui.widgets.WidgetSiteItemView
 import org.junit.Assert.assertNotNull
 import org.junit.Before
-import org.junit.Ignore
+import org.junit.Rule
 import org.junit.Test
 import org.mozilla.fenix.exceptions.ExceptionsInteractor
+import org.mozilla.fenix.helpers.MockkRetryTestRule

 class ExceptionsListItemViewHolderTest {

@ -27,6 +28,9 @@ class ExceptionsListItemViewHolderTest {
    @MockK private lateinit var icons: BrowserIcons
    @MockK private lateinit var interactor: ExceptionsInteractor<Exception>

+    @get:Rule
+    val mockkRule = MockkRetryTestRule()
+
    @Before
    fun setup() {
        MockKAnnotations.init(this)
@ -51,7 +55,6 @@ class ExceptionsListItemViewHolderTest {
    }

    @Test
-    @Ignore("Intermittent failing test on JDK11: https://github.com/mockk/mockk/issues/598")
    fun `delete button calls interactor`() {
        var clickListener: ((View) -> Unit)? = null
        val exception = Exception()
--- a/app/src/test/java/org/mozilla/fenix/helpers/MockkRetryTestRule.kt
+++ b/app/src/test/java/org/mozilla/fenix/helpers/MockkRetryTestRule.kt
@ -0,0 +1,60 @@
+package org.mozilla.fenix.helpers
+
+import io.mockk.MockKException
+import io.mockk.unmockkAll
+import mozilla.components.support.base.log.logger.Logger
+import org.junit.rules.TestRule
+import org.junit.runner.Description
+import org.junit.runners.model.Statement
+
+/**
+ * [TestRule] to work around mockk problem that causes intermittent failures
+ * of tests with mocked lambdas. This rule will call `unmockAll` and retry
+ * running the failing test until [maxTries] is reached.
+ *
+ * See:
+ * https://github.com/mockk/mockk/issues/598
+ * https://github.com/mozilla-mobile/fenix/issues/21952
+ * https://github.com/mozilla-mobile/fenix/issues/22240
+ */
+class MockkRetryTestRule(val maxTries: Int = 3) : TestRule {
+
+    private val logger = Logger("MockkRetryTestRule")
+
+    @Suppress("TooGenericExceptionCaught", "NestedBlockDepth")
+    override fun apply(base: Statement, description: Description): Statement {
+        return object : Statement() {
+            override fun evaluate() {
+                var failure: Throwable? = null
+
+                for (i in 0 until maxTries) {
+                    try {
+                        base.evaluate()
+                        return
+                    } catch (throwable: Throwable) {
+                        when (throwable) {
+                            // Work around intermittently failing tests with mocked lambdas
+                            // on JDK 11: https://github.com/mockk/mockk/issues/598
+                            is InstantiationError,
+                            is MockKException -> {
+                                failure = throwable
+                                val message = if (i < maxTries - 1) {
+                                    "Retrying test \"${description.displayName}\""
+                                } else {
+                                    "Giving up on test \"${description.displayName}\" after $maxTries tries"
+                                }
+                                logger.error(message, throwable)
+                                unmockkAll()
+                            }
+                            else -> {
+                                throw throwable
+                            }
+                        }
+                    }
+                }
+
+                throw failure!!
+            }
+        }
+    }
+}
--- a/app/src/test/java/org/mozilla/fenix/push/WebPushEngineIntegrationTest.kt
+++ b/app/src/test/java/org/mozilla/fenix/push/WebPushEngineIntegrationTest.kt
@ -32,8 +32,9 @@ import org.junit.Assert.assertNotNull
 import org.junit.Assert.assertNull
 import org.junit.Assert.assertTrue
 import org.junit.Before
-import org.junit.Ignore
+import org.junit.Rule
 import org.junit.Test
+import org.mozilla.fenix.helpers.MockkRetryTestRule

 class WebPushEngineIntegrationTest {

@ -44,6 +45,9 @@ class WebPushEngineIntegrationTest {
    private lateinit var delegate: CapturingSlot<WebPushDelegate>
    private lateinit var integration: WebPushEngineIntegration

+    @get:Rule
+    val mockkRule = MockkRetryTestRule()
+
    @Before
    fun setup() {
        MockKAnnotations.init(this)
@ -89,7 +93,6 @@ class WebPushEngineIntegrationTest {
    }

    @Test
-    @Ignore("Intermittent failing test on JDK11: https://github.com/mockk/mockk/issues/598")
    fun `delegate calls getSubscription`() {
        integration.start()
        var subscribeFn: ((AutoPushSubscription?) -> Unit)? = null
--- a/app/src/test/java/org/mozilla/fenix/tabstray/TabsTrayFragmentTest.kt
+++ b/app/src/test/java/org/mozilla/fenix/tabstray/TabsTrayFragmentTest.kt
@ -36,6 +36,7 @@ import org.junit.Assert.assertEquals
 import org.junit.Assert.assertFalse
 import org.junit.Assert.assertSame
 import org.junit.Before
+import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.mozilla.fenix.NavGraphDirections
@ -48,6 +49,7 @@ import org.mozilla.fenix.databinding.FragmentTabTrayDialogBinding
 import org.mozilla.fenix.ext.components
 import org.mozilla.fenix.ext.settings
 import org.mozilla.fenix.helpers.FenixRobolectricTestRunner
+import org.mozilla.fenix.helpers.MockkRetryTestRule
 import org.mozilla.fenix.home.HomeScreenViewModel
 import org.mozilla.fenix.tabstray.browser.BrowserTrayInteractor
 import org.mozilla.fenix.tabstray.ext.showWithTheme
@ -62,6 +64,9 @@ class TabsTrayFragmentTest {
    private lateinit var tabsTrayDialogBinding: FragmentTabTrayDialogBinding
    private lateinit var fabButtonBinding: ComponentTabstrayFabBinding

+    @get:Rule
+    val mockkRule = MockkRetryTestRule()
+
    @Before
    fun setup() {
        context = mockk(relaxed = true)
--- a/app/src/test/java/org/mozilla/fenix/tabstray/ext/FenixSnackbarKtTest.kt
+++ b/app/src/test/java/org/mozilla/fenix/tabstray/ext/FenixSnackbarKtTest.kt
@ -9,14 +9,18 @@ import android.view.View
 import io.mockk.every
 import io.mockk.mockk
 import io.mockk.verifyOrder
-import org.junit.Ignore
+import org.junit.Rule
 import org.junit.Test
 import org.mozilla.fenix.R
 import org.mozilla.fenix.components.FenixSnackbar
+import org.mozilla.fenix.helpers.MockkRetryTestRule
 import org.mozilla.fenix.tabstray.TabsTrayFragment.Companion.ELEVATION

 class FenixSnackbarKtTest {

+    @get:Rule
+    val mockkRule = MockkRetryTestRule()
+
    @Test
    fun `WHEN collectionMessage is called with different parameters THEN correct text will be set`() {
        val mockContext: Context = mockk {
@ -66,7 +70,6 @@ class FenixSnackbarKtTest {
    }

    @Test
-    @Ignore("Intermittent failing test on JDK11: https://github.com/mockk/mockk/issues/598")
    fun `WHEN anchorWithAction is called THEN correct text will be set`() {
        val mockContext: Context = mockk {
            every { getString(R.string.create_collection_view) }