Possibility to ignore tests for particular browsers

Closes #10213

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java

@@ -27,6 +27,7 @@ import org.jboss.arquillian.graphene.location.CustomizableURLResourceProvider;
 import org.jboss.arquillian.test.spi.TestEnricher;
 import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
 import org.jboss.arquillian.test.spi.execution.TestExecutionDecider;
+import org.keycloak.testsuite.arquillian.decider.BrowserDriverIgnoreDecider;
 import org.keycloak.testsuite.arquillian.h2.H2TestEnricher;
 import org.keycloak.testsuite.arquillian.jmx.JmxConnectorRegistryCreator;
 import org.keycloak.testsuite.arquillian.decider.AdapterTestExecutionDecider;
@@ -73,7 +74,8 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .service(TestExecutionDecider.class, MigrationTestExecutionDecider.class)
                 .service(TestExecutionDecider.class, AdapterTestExecutionDecider.class)
                 .service(TestExecutionDecider.class, VaultTestExecutionDecider.class)
-                .service(TestExecutionDecider.class, AuthServerExcludeExecutionDecider.class);
+                .service(TestExecutionDecider.class, AuthServerExcludeExecutionDecider.class)
+                .service(TestExecutionDecider.class, BrowserDriverIgnoreDecider.class);
 
         builder
                 .override(ResourceProvider.class, URLResourceProvider.class, URLProvider.class)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/IgnoreBrowserDriver.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.annotation;
+
+import org.openqa.selenium.WebDriver;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Repeatable;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Annotation for marking test method/class which should be ignored
+ *
+ * @author <a href="mailto:mabartos@redhat.com">Martin Bartos</a>
+ */
+@Documented
+@Retention(RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+@Repeatable(IgnoreBrowserDrivers.class)
+public @interface IgnoreBrowserDriver {
+
+    /**
+     * Define for which WebDriver the test method/class should be ignored
+     */
+    Class<? extends WebDriver> value();
+
+    /**
+     * Define whether the value should be negated
+     *
+     * Usable in cases when we want to execute test method/class with all WebDrivers except the one specified in value()
+     */
+    boolean negate() default false;
+
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/IgnoreBrowserDrivers.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Define more WebDrivers that should be ignored for a particular test method/class
+ * <p>
+ * See {@link IgnoreBrowserDriver }
+ *
+ * @author <a href="mailto:mabartos@redhat.com">Martin Bartos</a>
+ */
+@Documented
+@Retention(RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+public @interface IgnoreBrowserDrivers {
+
+    /**
+     * Several WebDrivers for which the test method/class should be ignored
+     */
+    IgnoreBrowserDriver[] value() default {};
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/decider/BrowserDriverIgnoreDecider.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.decider;
+
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.test.spi.execution.ExecutionDecision;
+import org.jboss.arquillian.test.spi.execution.TestExecutionDecider;
+import org.keycloak.testsuite.arquillian.TestContext;
+import org.keycloak.testsuite.arquillian.annotation.IgnoreBrowserDriver;
+import org.keycloak.testsuite.arquillian.annotation.IgnoreBrowserDrivers;
+import org.openqa.selenium.WebDriver;
+
+import java.lang.reflect.AnnotatedElement;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.function.Predicate;
+
+import static org.keycloak.testsuite.util.BrowserDriverUtil.isDriverInstanceOf;
+
+/**
+ * Decider for ignoring tests for particular browsers (WebDrivers)
+ *
+ * @author <a href="mailto:mabartos@redhat.com">Martin Bartos</a>
+ */
+public class BrowserDriverIgnoreDecider implements TestExecutionDecider {
+
+    @Inject
+    private Instance<WebDriver> driver;
+
+    @Inject
+    private Instance<TestContext> testContextInstance;
+
+    @Override
+    public ExecutionDecision decide(Method method) {
+        if (isAnnotationPresent(method)) {
+            return decideIgnoring(method);
+        } else { //class
+            final TestContext testContext = testContextInstance.get();
+
+            if (isAnnotationPresent(testContext.getTestClass())) {
+                return decideIgnoring(testContext.getTestClass());
+            }
+        }
+        return ExecutionDecision.execute();
+    }
+
+    private boolean isAnnotationPresent(AnnotatedElement element) {
+        return element.isAnnotationPresent(IgnoreBrowserDriver.class) || element.isAnnotationPresent(IgnoreBrowserDrivers.class);
+    }
+
+    private ExecutionDecision decideIgnoring(AnnotatedElement element) {
+        final WebDriver webDriver = driver.get();
+
+        Predicate<IgnoreBrowserDriver> shouldBeIgnored = (item) -> {
+            return webDriver != null && (isDriverInstanceOf(webDriver, item.value()) ^ item.negate());
+        };
+
+        return Arrays.stream(element.getAnnotationsByType(IgnoreBrowserDriver.class))
+                .filter(shouldBeIgnored)
+                .findAny()
+                .map(f -> ExecutionDecision.dontExecute("This test should not be executed with this browser."))
+                .orElse(ExecutionDecision.execute());
+    }
+
+    @Override
+    public int precedence() {
+        return Integer.MIN_VALUE;
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/drone/KeycloakDronePostSetup.java

@@ -20,6 +20,8 @@ package org.keycloak.testsuite.drone;
 import java.util.concurrent.TimeUnit;
 
 import io.appium.java_client.AppiumDriver;
+import org.jboss.arquillian.core.api.InstanceProducer;
+import org.jboss.arquillian.core.api.annotation.Inject;
 import org.jboss.arquillian.core.api.annotation.Observes;
 import org.jboss.arquillian.drone.spi.DroneContext;
 import org.jboss.arquillian.drone.spi.DronePoint;
@@ -27,6 +29,7 @@ import org.jboss.arquillian.drone.spi.event.AfterDroneEnhanced;
 import org.jboss.arquillian.graphene.proxy.GrapheneProxyInstance;
 import org.jboss.arquillian.graphene.proxy.Interceptor;
 import org.jboss.arquillian.graphene.proxy.InvocationContext;
+import org.jboss.arquillian.test.spi.annotation.ClassScoped;
 import org.jboss.logging.Logger;
 import org.keycloak.testsuite.util.WaitUtils;
 import org.openqa.selenium.WebDriver;
@@ -38,8 +41,11 @@ import org.openqa.selenium.remote.RemoteWebDriver;
  */
 public class KeycloakDronePostSetup {
 
-    protected static final Logger log = org.jboss.logging.Logger.getLogger(KeycloakDronePostSetup.class);
+    @Inject
+    @ClassScoped // needed in BrowserDriverIgnoreDecider
+    private InstanceProducer<WebDriver> webDriverProducer;
 
+    protected static final Logger log = org.jboss.logging.Logger.getLogger(KeycloakDronePostSetup.class);
 
     public void configureWebDriver(@Observes AfterDroneEnhanced event, DroneContext droneContext) {
         DronePoint<?> dronePoint = event.getDronePoint();
@@ -48,12 +54,13 @@ public class KeycloakDronePostSetup {
         if (drone instanceof RemoteWebDriver) {
             RemoteWebDriver remoteWebDriver = (RemoteWebDriver) drone;
             log.infof("Detected browser: %s %s", remoteWebDriver.getCapabilities().getBrowserName(), remoteWebDriver.getCapabilities().getVersion());
+            webDriverProducer.set(remoteWebDriver);
         }
 
-
         if (drone instanceof WebDriver && !(drone instanceof AppiumDriver)) {
             WebDriver webDriver = (WebDriver) drone;
             configureDriverSettings(webDriver);
+            webDriverProducer.set(webDriver);
         } else {
             log.warn("Drone is not instanceof WebDriver for a desktop browser! Drone is " + drone);
         }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/BrowserDriverUtil.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util;
+
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.edge.EdgeDriver;
+import org.openqa.selenium.firefox.FirefoxDriver;
+import org.openqa.selenium.opera.OperaDriver;
+import org.openqa.selenium.safari.SafariDriver;
+
+/**
+ * Determine which WebDriver is used
+ *
+ * @author <a href="mailto:mabartos@redhat.com">Martin Bartos</a>
+ */
+public class BrowserDriverUtil {
+
+    public static boolean isDriverInstanceOf(WebDriver driver, Class<? extends WebDriver> clazz) {
+        return clazz.isAssignableFrom(driver.getClass());
+    }
+
+    public static boolean isDriverChrome(WebDriver driver) {
+        return isDriverInstanceOf(driver, ChromeDriver.class);
+    }
+
+    public static boolean isDriverFirefox(WebDriver driver) {
+        return isDriverInstanceOf(driver, FirefoxDriver.class);
+    }
+
+    public static boolean isDriverOpera(WebDriver driver) {
+        return isDriverInstanceOf(driver, OperaDriver.class);
+    }
+
+    public static boolean isDriverEdge(WebDriver driver) {
+        return isDriverInstanceOf(driver, EdgeDriver.class);
+    }
+
+    public static boolean isDriverSafari(WebDriver driver) {
+        return isDriverInstanceOf(driver, SafariDriver.class);
+    }
+}

testsuite/integration-arquillian/tests/other/console/src/test/java/org/keycloak/testsuite/console/other/IgnoreBrowserDriverClassTest.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.console.other;
+
+import org.junit.Test;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.annotation.IgnoreBrowserDriver;
+import org.keycloak.testsuite.arquillian.annotation.IgnoreBrowserDrivers;
+import org.keycloak.testsuite.console.AbstractConsoleTest;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.firefox.FirefoxDriver;
+
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.keycloak.testsuite.util.BrowserDriverUtil.isDriverChrome;
+import static org.keycloak.testsuite.util.BrowserDriverUtil.isDriverFirefox;
+
+/**
+ * @author <a href="mailto:mabartos@redhat.com">Martin Bartos</a>
+ */
+@IgnoreBrowserDrivers({
+        @IgnoreBrowserDriver(FirefoxDriver.class),
+        @IgnoreBrowserDriver(value = ChromeDriver.class, negate = true)})
+public class IgnoreBrowserDriverClassTest extends AbstractConsoleTest {
+
+    @Test
+    public void ignoreFirefoxAndNotChrome() {
+        assertThat(isDriverChrome(driver), is(true));
+        assertThat(isDriverFirefox(driver), is(false));
+    }
+
+    @Test
+    @IgnoreBrowserDriver(ChromeDriver.class)
+    public void ignoreChrome() {
+        assertThat(isDriverChrome(driver), is(false));
+    }
+}

testsuite/integration-arquillian/tests/other/console/src/test/java/org/keycloak/testsuite/console/other/IgnoreBrowserDriverMethodTest.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.console.other;
+
+import org.junit.Test;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.annotation.IgnoreBrowserDriver;
+import org.keycloak.testsuite.arquillian.annotation.IgnoreBrowserDrivers;
+import org.keycloak.testsuite.console.AbstractConsoleTest;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.firefox.FirefoxDriver;
+
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.keycloak.testsuite.util.BrowserDriverUtil.isDriverChrome;
+import static org.keycloak.testsuite.util.BrowserDriverUtil.isDriverFirefox;
+
+/**
+ * @author <a href="mailto:mabartos@redhat.com">Martin Bartos</a>
+ */
+public class IgnoreBrowserDriverMethodTest extends AbstractConsoleTest {
+
+    @Test
+    @IgnoreBrowserDriver(FirefoxDriver.class)
+    public void allExceptFirefox() {
+        assertThat(isDriverFirefox(driver), is(false));
+    }
+
+    @Test
+    @IgnoreBrowserDriver(value = FirefoxDriver.class, negate = true)
+    public void onlyFirefox() {
+        assertThat(isDriverFirefox(driver), is(true));
+    }
+
+    @Test
+    @IgnoreBrowserDriver(ChromeDriver.class)
+    public void allExceptChrome() {
+        assertThat(isDriverChrome(driver), is(false));
+    }
+
+    @Test
+    @IgnoreBrowserDriver(value = ChromeDriver.class, negate = true)
+    public void onlyChrome() {
+        assertThat(isDriverChrome(driver), is(true));
+    }
+
+    @Test
+    @IgnoreBrowserDrivers({
+            @IgnoreBrowserDriver(FirefoxDriver.class),
+            @IgnoreBrowserDriver(ChromeDriver.class)
+    })
+    public void ignoreChromeAndFirefox() {
+        assertThat(isDriverChrome(driver), is(false));
+        assertThat(isDriverFirefox(driver), is(false));
+    }
+
+    @Test
+    public void executeWithEachDriver() {
+        assertThat(driver, notNullValue());
+    }
+}

testsuite/integration-arquillian/tests/other/webauthn/src/main/java/org/keycloak/testsuite/webauthn/utils/WebAuthnBrowserDriverUtil.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.webauthn.utils;
+
+import org.keycloak.testsuite.util.BrowserDriverUtil;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chromium.ChromiumDriver;
+
+/**
+ * @author <a href="mailto:mabartos@redhat.com">Martin Bartos</a>
+ */
+public class WebAuthnBrowserDriverUtil extends BrowserDriverUtil {
+
+    public static boolean isDriverChromiumBased(WebDriver driver) {
+        return BrowserDriverUtil.isDriverInstanceOf(driver, ChromiumDriver.class);
+    }
+}