From f994cc54d538aad352565ce107924cafe47ec003 Mon Sep 17 00:00:00 2001 From: Andy <41118244+andy-clapson@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:09:50 -0400 Subject: [PATCH] Remove robots.txt entirely * remove robots.txt entirely, as blocking page- crawling prevents the `X-Robots-Tag` headers (and similar meta tags) from working as intended. Closes #17433 Signed-off-by: Andy Signed-off-by: Alexander Schwartz Co-authored-by: Alexander Schwartz --- .../topics/changes/changes-26_1_0.adoc | 4 ++ docs/guides/server/reverseproxy.adoc | 5 --- .../services/resources/RobotsResource.java | 39 ------------------- .../resteasy/ResteasyKeycloakApplication.java | 2 - 4 files changed, 4 insertions(+), 46 deletions(-) delete mode 100755 services/src/main/java/org/keycloak/services/resources/RobotsResource.java diff --git a/docs/documentation/upgrading/topics/changes/changes-26_1_0.adoc b/docs/documentation/upgrading/topics/changes/changes-26_1_0.adoc index 5206ef4604..8edce7c898 100644 --- a/docs/documentation/upgrading/topics/changes/changes-26_1_0.adoc +++ b/docs/documentation/upgrading/topics/changes/changes-26_1_0.adoc @@ -45,3 +45,7 @@ The {project_name} Operator will continue to configure `kubernetes` as a transpo When developing extensions for {project_name}, developers can now specify dependencies between provider factories classes by implementing the method `dependsOn()` in the `ProviderFactory` interface. See the Javadoc for a detailed description. + += Removal of robots.txt file + +The `robots.txt` file, previously included by default, is now removed. The default `robots.txt` file blocked all crawling, which prevented the `noindex`/`nofollow` directives from being followed. The desired default behaviour is for {project_name} pages to not show up in search engine results and this is accomplished by the existing `X-Robots-Tag` header, which is set to `none` by default. The value of this header can be overidden per-realm if a different behaviour is needed. diff --git a/docs/guides/server/reverseproxy.adoc b/docs/guides/server/reverseproxy.adoc index cf84301766..74d8820420 100644 --- a/docs/guides/server/reverseproxy.adoc +++ b/docs/guides/server/reverseproxy.adoc @@ -121,11 +121,6 @@ The following table shows the recommended paths to expose. |Yes |This path is needed to serve assets correctly. It may be served from a CDN instead of the {project_name} path. -|/robots.txt -|/robots.txt -|Yes -|Search engine rules - |/metrics |- |No diff --git a/services/src/main/java/org/keycloak/services/resources/RobotsResource.java b/services/src/main/java/org/keycloak/services/resources/RobotsResource.java deleted file mode 100755 index 2ae831897d..0000000000 --- a/services/src/main/java/org/keycloak/services/resources/RobotsResource.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2016 Red Hat, Inc. and/or its affiliates - * and other contributors as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.keycloak.services.resources; - -import org.keycloak.utils.MediaType; - -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.ext.Provider; - -@Provider -@Path("/robots.txt") -public class RobotsResource { - - private static final String robots = "User-agent: *\n" + "Disallow: /"; - - @GET - @Produces(MediaType.TEXT_PLAIN_UTF_8) - public String getRobots() { - return robots; - } - -} diff --git a/services/src/test/java/org/keycloak/services/resteasy/ResteasyKeycloakApplication.java b/services/src/test/java/org/keycloak/services/resteasy/ResteasyKeycloakApplication.java index 994abdc3c7..e55e79a13a 100644 --- a/services/src/test/java/org/keycloak/services/resteasy/ResteasyKeycloakApplication.java +++ b/services/src/test/java/org/keycloak/services/resteasy/ResteasyKeycloakApplication.java @@ -28,7 +28,6 @@ import org.keycloak.services.filters.KeycloakSecurityHeadersFilter; import org.keycloak.services.resources.KeycloakApplication; import org.keycloak.services.resources.LoadBalancerResource; import org.keycloak.services.resources.RealmsResource; -import org.keycloak.services.resources.RobotsResource; import org.keycloak.services.resources.ThemeResource; import org.keycloak.services.resources.WelcomeResource; import org.keycloak.services.resources.admin.AdminRoot; @@ -43,7 +42,6 @@ public class ResteasyKeycloakApplication extends KeycloakApplication { protected Set> classes = new HashSet<>(); public ResteasyKeycloakApplication() { - classes.add(RobotsResource.class); classes.add(RealmsResource.class); if (Profile.isFeatureEnabled(Profile.Feature.ADMIN_API)) { classes.add(AdminRoot.class);