fix: switching the raw distribution to a weak readiness check (#26097)

also adding a thread dump if the server doesn't seem to stop properly

closes: #23786

Signed-off-by: Steve Hawkins <shawkins@redhat.com>
This commit is contained in:
Steven Hawkins 2024-01-12 13:30:58 -05:00 committed by GitHub
parent c2b132171d
commit c78c2de653
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 60 additions and 5 deletions

View file

@ -78,6 +78,7 @@ public class FipsDistTest {
dist.copyOrReplaceFileFromClasspath("/server.keystore", Path.of("conf", "server.keystore"));
CLIResult cliResult = dist.run("start", "--fips-mode=strict");
cliResult.assertMessage("ERROR: java.lang.IllegalArgumentException: malformed sequence");
dist.assertStopped();
});
}
@ -125,6 +126,7 @@ public class FipsDistTest {
dist.copyOrReplaceFileFromClasspath("/server.keystore.pkcs12", Path.of("conf", "server.keystore"));
CLIResult cliResult = dist.run("start", "--fips-mode=strict", "--https-key-store-password=passwordpassword");
cliResult.assertMessage("ERROR: java.lang.IllegalArgumentException: malformed sequence");
dist.assertStopped();
});
}

View file

@ -121,6 +121,11 @@ public class KeycloakDistributionDecorator implements KeycloakDistribution {
delegate.copyOrReplaceFile(file, targetFile);
}
@Override
public void assertStopped() {
delegate.assertStopped();
}
@Override
public <D extends KeycloakDistribution> D unwrap(Class<D> type) {
if (!KeycloakDistribution.class.isAssignableFrom(type)) {

View file

@ -258,4 +258,9 @@ public final class DockerKeycloakDistribution implements KeycloakDistribution {
throw new IllegalArgumentException("Not a " + type + " type");
}
@Override
public void assertStopped() {
// not implemented
}
}

View file

@ -27,6 +27,8 @@ public interface KeycloakDistribution {
boolean isManualStop();
void assertStopped();
default String[] getCliArgs(List<String> arguments) {
throw new RuntimeException("Not implemented");
}

View file

@ -36,14 +36,17 @@ import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.LockSupport;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -58,6 +61,8 @@ import javax.net.ssl.X509TrustManager;
import io.quarkus.deployment.util.FileUtil;
import io.quarkus.fs.util.ZipUtils;
import org.awaitility.Awaitility;
import org.jboss.logging.Logger;
import org.jboss.shrinkwrap.api.ShrinkWrap;
import org.jboss.shrinkwrap.api.asset.EmptyAsset;
import org.jboss.shrinkwrap.api.exporter.ZipExporter;
@ -65,6 +70,7 @@ import org.jboss.shrinkwrap.api.spec.JavaArchive;
import org.keycloak.common.Version;
import org.keycloak.it.TestProvider;
import org.keycloak.it.junit5.extension.CLIResult;
import org.keycloak.quarkus.runtime.Environment;
import org.keycloak.quarkus.runtime.cli.command.Build;
import org.keycloak.quarkus.runtime.configuration.mappers.PropertyMapper;
import org.keycloak.quarkus.runtime.configuration.mappers.PropertyMappers;
@ -76,11 +82,12 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
private static final int DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 10;
private static final Logger LOG = Logger.getLogger(RawKeycloakDistribution.class);
private Process keycloak;
private int exitCode = -1;
private final Path distPath;
private final List<String> outputStream = new ArrayList<>();
private final List<String> errorStream = new ArrayList<>();
private final List<String> outputStream = Collections.synchronizedList(new ArrayList<>());
private final List<String> errorStream = Collections.synchronizedList(new ArrayList<>());
private boolean manualStop;
private String relativePath;
private int httpPort;
@ -89,7 +96,6 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
private boolean enableTls;
private boolean reCreate;
private boolean removeBuildOptionsAfterBuild;
private boolean createAdminUser;
private ExecutorService outputExecutor;
private boolean inited = false;
private Map<String, String> envVars = new HashMap<>();
@ -253,6 +259,24 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
return allArgs.toArray(String[]::new);
}
@Override
public void assertStopped() {
try {
if (keycloak != null) {
keycloak.onExit().get(1, TimeUnit.MINUTES);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
} catch (TimeoutException e) {
LOG.warn("Process did not exit within a minute as expected, will attempt a thread dump");
threadDump();
LOG.warn("TODO: this should be a hard error / re-diagnosed after https://issues.redhat.com/browse/JBTM-3830 is pulled into Keycloak");
}
}
private void waitForReadiness() throws MalformedURLException {
waitForReadiness("http", httpPort);
@ -269,8 +293,9 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
while (true) {
if (System.currentTimeMillis() - startTime > getStartTimeout()) {
throw new IllegalStateException(
"Timeout [" + getStartTimeout() + "] while waiting for Quarkus server", ex);
threadDump();
LOG.warn("Timeout [" + getStartTimeout() + "] while waiting for Quarkus server", ex);
LOG.warn("TODO: this should be a hard error / re-diagnosed after https://issues.redhat.com/browse/JBTM-3830 is pulled into Keycloak");
}
if (!keycloak.isAlive()) {
@ -308,6 +333,22 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
}
}
private void threadDump() {
if (Environment.isWindows()) {
return;
}
try {
ProcessBuilder builder = new ProcessBuilder("kill", "-3", String.valueOf(keycloak.pid()));
Process p = builder.start();
p.onExit().get(getStartTimeout(), TimeUnit.MILLISECONDS);
} catch (Exception e) {
LOG.warn("A thread dump may not have been successfully triggered", e);
return;
}
Awaitility.await().atMost(1, TimeUnit.MINUTES)
.until(() -> getOutputStream().stream().anyMatch(s -> s.contains("JNI global refs")));
}
private long getStartTimeout() {
return TimeUnit.SECONDS.toMillis(120);
}