KEYCLOAK-4693

Improve blocking search indexing

server-spi-private/src/main/java/org/keycloak/migration/MigrationModelManager.java

@@ -33,6 +33,7 @@ import org.keycloak.migration.migrators.MigrateTo2_2_0;
 import org.keycloak.migration.migrators.MigrateTo2_3_0;
 import org.keycloak.migration.migrators.MigrateTo2_5_0;
 import org.keycloak.migration.migrators.MigrateTo3_0_0;
+import org.keycloak.migration.migrators.MigrateTo3_1_0;
 import org.keycloak.migration.migrators.Migration;
 import org.keycloak.models.KeycloakSession;
 
@@ -58,7 +59,8 @@ public class MigrationModelManager {
             new MigrateTo2_2_0(),
             new MigrateTo2_3_0(),
             new MigrateTo2_5_0(),
-            new MigrateTo3_0_0()
+            new MigrateTo3_0_0(),
+            new MigrateTo3_1_0()
     };
 
     public static void migrate(KeycloakSession session) {

server-spi-private/src/main/java/org/keycloak/migration/migrators/MigrateTo3_1_0.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.migration.migrators;
+
+
+import org.keycloak.migration.ModelVersion;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.RealmModel;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:bburke@redhat.com">Bill Burke</a>
+ */
+public class MigrateTo3_1_0 implements Migration {
+
+    public static final ModelVersion VERSION = new ModelVersion("3.1.0");
+
+    @Override
+    public void migrate(KeycloakSession session) {
+        for (RealmModel realm : session.realms().getRealms()) {
+            if (realm.getBrowserSecurityHeaders() != null) {
+
+                Map<String, String> browserSecurityHeaders = new HashMap<>(realm.getBrowserSecurityHeaders());
+                browserSecurityHeaders.put("xRobotsTag", "none");
+
+                realm.setBrowserSecurityHeaders(Collections.unmodifiableMap(browserSecurityHeaders));
+            }
+        }
+    }
+
+    @Override
+    public ModelVersion getVersion() {
+        return VERSION;
+    }
+
+}

server-spi-private/src/main/java/org/keycloak/models/BrowserSecurityHeaders.java

@@ -34,11 +34,13 @@ public class BrowserSecurityHeaders {
         headerMap.put("xFrameOptions", "X-Frame-Options");
         headerMap.put("contentSecurityPolicy", "Content-Security-Policy");
         headerMap.put("xContentTypeOptions", "X-Content-Type-Options");
+        headerMap.put("xRobotsTag", "X-Robots-Tag");
 
         Map<String, String> dh = new HashMap<>();
         dh.put("xFrameOptions", "SAMEORIGIN");
         dh.put("contentSecurityPolicy", "frame-src 'self'");
         dh.put("xContentTypeOptions", "nosniff");
+        dh.put("xRobotsTag", "none");
 
         defaultHeaders = Collections.unmodifiableMap(dh);
         headerAttributeMap = Collections.unmodifiableMap(headerMap);

themes/src/main/resources/theme/base/admin/messages/admin-messages_en.properties

@@ -111,6 +111,8 @@ content-sec-policy=Content-Security-Policy
 content-sec-policy-tooltip=Default value prevents pages from being included via non-origin iframes (click label for more information)
 content-type-options=X-Content-Type-Options
 content-type-options-tooltip=Default value prevents Internet Explorer and Google Chrome from MIME-sniffing a response away from the declared content-type (click label for more information)
+robots-tag=X-Robots-Tag
+robots-tag-tooltip=Prevent pages from appearing in search engines (click label for more information)
 max-login-failures=Max Login Failures
 max-login-failures.tooltip=How many failures before wait is triggered.
 wait-increment=Wait Increment

themes/src/main/resources/theme/base/admin/resources/partials/defense-headers.html

@@ -23,12 +23,19 @@
                 <kc-tooltip>{{:: 'content-sec-policy-tooltip' | translate}}</kc-tooltip>
             </div>
             <div class="form-group">
-                <label class="col-md-2 control-label" for="xContentTypeOptions"><a href="https://www.owasp.org/index.php/List_of_useful_HTTP_headers">{{:: 'content-type-options' | translate}}</a></label>
+                <label class="col-md-2 control-label" for="xContentTypeOptions"><a href="https://www.owasp.org/index.php/List_of_useful_HTTP_headers" target="_blank">{{:: 'content-type-options' | translate}}</a></label>
                 <div class="col-sm-6">
                     <input class="form-control" id="xContentTypeOptions" type="text" ng-model="realm.browserSecurityHeaders.xContentTypeOptions">
                 </div>
                 <kc-tooltip>{{:: 'content-type-options-tooltip' | translate}}</kc-tooltip>
             </div>
+            <div class="form-group">
+                <label class="col-md-2 control-label" for="xContentTypeOptions"><a href="https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag" target="_blank">{{:: 'robots-tag' | translate}}</a></label>
+                <div class="col-sm-6">
+                    <input class="form-control" id="xRobotsTag" type="text" ng-model="realm.browserSecurityHeaders.xRobotsTag">
+                </div>
+                <kc-tooltip>{{:: 'robots-tag-tooltip' | translate}}</kc-tooltip>
+            </div>
         </fieldset>
         <div class="form-group" data-ng-show="access.manageRealm">
             <div class="col-md-10 col-md-offset-2">