ABR12: AllNodesDisabledNamespacesUpdater (#5876)

leader-election-api/build.gradle

@@ -17,7 +17,6 @@ dependencies {
   implementation 'com.palantir.safe-logging:safe-logging'
   implementation project(':commons-annotations')
 
-
   annotationProcessor group: 'org.immutables', name: 'value'
   compileOnly 'org.immutables:value::annotations'
 }

leader-election-impl/src/main/java/com/palantir/paxos/PaxosQuorumChecker.java

@@ -65,6 +65,24 @@ private PaxosQuorumChecker() {
         // Private constructor. Disallow instantiation.
     }
 
+    public static <SERVICE, RESPONSE extends PaxosResponse>
+            PaxosResponsesWithRemote<SERVICE, RESPONSE> collectAllResponses(
+                    ImmutableList<SERVICE> remotes,
+                    Function<SERVICE, RESPONSE> request,
+                    Map<SERVICE, CheckedRejectionExecutorService> executors,
+                    Duration remoteRequestTimeout,
+                    boolean cancelRemainingCalls) {
+        Preconditions.checkState(executors.keySet().containsAll(remotes), "Each remote should have an executor.");
+        return collectResponses(
+                remotes,
+                request,
+                remotes.size(), // wait until all responses have been received
+                remoteRequestTimeout,
+                _unused -> false, // never abort early
+                cancelRemainingCalls,
+                MultiplexingCompletionService.createFromCheckedExecutors(executors));
+    }
+
     /**
      * Collects a list of responses from a quorum of remote services.
      * This method short-circuits if a quorum can no longer be obtained (if too many servers have sent nacks), and

timelock-api/build.gradle

@@ -9,10 +9,21 @@ conjure {
     }
 }
 
+dependencies {
+    implementation project(':leader-election-api')
+
+    compileOnly 'com.fasterxml.jackson.core:jackson-databind'
+    compileOnly 'org.immutables:value::annotations'
+    annotationProcessor group: 'org.immutables', name: 'value'
+}
+
 subprojects {
     apply from: "../../gradle/shared.gradle"
     dependencies {
         compile project(':lock-api-objects')
+        compile project(':timelock-api')
+        implementation project(':leader-election-api')
+
         compile 'com.palantir.conjure.java:conjure-lib'
 
         implementation 'com.fasterxml.jackson.core:jackson-annotations'

timelock-api/src/main/conjure/timelock-api.yml

@@ -4,6 +4,10 @@ types:
       base-type: any
       external:
         java: com.palantir.lock.v2.LeaderTime
+    Namespace:
+      base-type: string
+      external:
+        java: com.palantir.atlasdb.timelock.api.Namespace
     NanoTime:
       base-type: safelong
       external:
@@ -114,8 +118,6 @@ types:
       LockWatchRequest:
         fields:
           references: set<LockWatchReference>
-      Namespace:
-        alias: string
       LeaderTimes:
         fields:
           leaderTimes: map<Namespace, LeaderTime>

timelock-api/src/main/conjure/timelock-management-api.yml

@@ -1,6 +1,10 @@
 types:
   conjure-imports:
     api: timelock-api.yml
+  imports:
+    SingleNodeUpdateResponse:
+      external:
+        java: com.palantir.atlasdb.timelock.api.SingleNodeUpdateResponse
   definitions:
     default-package: com.palantir.atlasdb.timelock.api
     objects:
@@ -13,8 +17,14 @@ types:
           lockId: uuid
       UnsuccessfulDisableNamespacesResponse:
         fields:
-          # if this set is non-empty, action will be required
-          disabledNamespaces: set<api.Namespace>
+          consistentlyDisabledNamespaces:
+            type: set<api.Namespace>
+            docs: Namespaces where we can assume another restore is in progress
+          partiallyDisabledNamespaces:
+            type: set<api.Namespace>
+            docs: |
+              Either another restore is in progress for this namespace (and we've hit a race condition), or
+              the namespace is stuck and needs to be manually fixed.
       DisableNamespacesResponse:
         union:
           successful: SuccessfulDisableNamespacesResponse
@@ -23,10 +33,22 @@ types:
         fields:
           namespaces: set<api.Namespace>
           lockId: uuid
-      ReenableNamespacesResponse:
+      SuccessfulReenableNamespacesResponse:
+        alias: boolean
+      UnsuccessfulReenableNamespacesResponse:
         fields:
-          wasSuccessful: boolean
-          lockedNamespaces: set<api.Namespace>
+          consistentlyLockedNamespaces:
+            type: set<api.Namespace>
+            docs: We can assume another restore is in progress for this namespace (we lost our lock)
+          partiallyLockedNamespaces:
+            type: set<api.Namespace>
+            docs: |
+              Either another restore is in progress for this namespace (and we lost our lock), or
+              the namespace is stuck and needs to be manually fixed.
+      ReenableNamespacesResponse:
+        union:
+          successful: SuccessfulReenableNamespacesResponse
+          unsuccessful: UnsuccessfulReenableNamespacesResponse
 
 services:
   DisabledNamespacesUpdaterService:
@@ -42,12 +64,12 @@ services:
         http: POST /disable
         args:
           request: DisableNamespacesRequest
-        returns: DisableNamespacesResponse
+        returns: SingleNodeUpdateResponse
       reenable:
         http: POST /reenable
         args:
           request: ReenableNamespacesRequest
-        returns: ReenableNamespacesResponse
+        returns: SingleNodeUpdateResponse
   TimeLockManagementService:
     name: TimeLock Management Service
     default-auth: header

timelock-api/src/main/java/com/palantir/atlasdb/timelock/api/Namespace.java

@@ -0,0 +1,43 @@
+/*
+ * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.timelock.api;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import org.immutables.value.Value;
+
+@JsonDeserialize(as = ImmutableNamespace.class)
+@JsonSerialize(as = ImmutableNamespace.class)
+@Value.Immutable
+public interface Namespace {
+    @JsonValue
+    String value();
+
+    // For back-compatibility with conjure-generated Namespace
+    default String get() {
+        return value();
+    }
+
+    static Namespace valueOf(String value) {
+        return of(value);
+    }
+
+    static Namespace of(String value) {
+        return ImmutableNamespace.builder().value(value).build();
+    }
+}

timelock-api/src/main/java/com/palantir/atlasdb/timelock/api/SingleNodeUpdateResponse.java

@@ -0,0 +1,45 @@
+/*
+ * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.timelock.api;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.palantir.paxos.PaxosResponse;
+import java.util.Map;
+import java.util.UUID;
+import org.immutables.value.Value;
+
+@JsonDeserialize(as = ImmutableSingleNodeUpdateResponse.class)
+@JsonSerialize(as = ImmutableSingleNodeUpdateResponse.class)
+@Value.Immutable
+public interface SingleNodeUpdateResponse extends PaxosResponse {
+    /**
+     * other namespaces will not have been disabled/re-enabled (the transaction will not complete).
+     */
+    Map<Namespace, UUID> lockedNamespaces();
+
+    static SingleNodeUpdateResponse successful() {
+        return ImmutableSingleNodeUpdateResponse.builder().isSuccessful(true).build();
+    }
+
+    static SingleNodeUpdateResponse failed(Map<Namespace, UUID> lockedNamespaces) {
+        return ImmutableSingleNodeUpdateResponse.builder()
+                .isSuccessful(false)
+                .lockedNamespaces(lockedNamespaces)
+                .build();
+    }
+}

timelock-impl/build.gradle

@@ -8,6 +8,7 @@ dependencies {
     compile project(":lock-conjure-api:lock-conjure-api-undertow")
     compile project(":lock-conjure-api:lock-conjure-api-jersey")
     compile project(":lock-impl")
+    compile project(':timelock-api')
     compile project(':timelock-api:timelock-api-undertow')
     compile project(':timelock-api:timelock-api-jersey')
     compile project(":leader-election-impl")

timelock-impl/src/main/java/com/palantir/atlasdb/timelock/TimelockNamespaces.java

@@ -21,12 +21,9 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import com.palantir.atlasdb.timelock.api.DisableNamespacesRequest;
-import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse;
 import com.palantir.atlasdb.timelock.api.Namespace;
 import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest;
-import com.palantir.atlasdb.timelock.api.ReenableNamespacesResponse;
-import com.palantir.atlasdb.timelock.api.SuccessfulDisableNamespacesResponse;
-import com.palantir.atlasdb.timelock.api.UnsuccessfulDisableNamespacesResponse;
+import com.palantir.atlasdb.timelock.api.SingleNodeUpdateResponse;
 import com.palantir.atlasdb.timelock.management.DisabledNamespaces;
 import com.palantir.atlasdb.timelock.paxos.PaxosTimeLockConstants;
 import com.palantir.atlasdb.util.MetricsManager;
@@ -36,7 +33,9 @@
 import com.palantir.logsafe.logger.SafeLogger;
 import com.palantir.logsafe.logger.SafeLoggerFactory;
 import com.palantir.paxos.Client;
+import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Function;
@@ -130,34 +129,24 @@ public void invalidateResourcesForClient(String namespace) {
         }
     }
 
-    public DisableNamespacesResponse disable(DisableNamespacesRequest request) {
-        DisableNamespacesResponse response = disabledNamespaces.disable(request);
-        response.accept(new DisableNamespacesResponse.Visitor<Void>() {
-            @Override
-            public Void visitSuccessful(SuccessfulDisableNamespacesResponse _unused) {
-                request.getNamespaces().stream().map(Namespace::get).forEach(ns -> invalidateResourcesForClient(ns));
-                return null;
-            }
-
-            @Override
-            public Void visitUnsuccessful(UnsuccessfulDisableNamespacesResponse response) {
-                log.info(
-                        "Not invalidating resources, as the request to disable namespaces was unsuccessful",
-                        SafeArg.of("response", response));
-                return null;
-            }
-
-            @Override
-            public Void visitUnknown(String unknownTypeThatShouldNeverHappen) {
-                throw new SafeIllegalStateException(
-                        "Unknown response when disabling namespaces",
-                        SafeArg.of("response", unknownTypeThatShouldNeverHappen));
-            }
-        });
+    public Map<Namespace, UUID> getNamespacesLockedWithDifferentLockId(Set<Namespace> namespaces, UUID expectedLockId) {
+        return disabledNamespaces.getNamespacesLockedWithDifferentLockId(namespaces, expectedLockId);
+    }
+
+    public SingleNodeUpdateResponse disable(DisableNamespacesRequest request) {
+        SingleNodeUpdateResponse response = disabledNamespaces.disable(request);
+        if (response.isSuccessful()) {
+            request.getNamespaces().stream().map(Namespace::get).forEach(this::invalidateResourcesForClient);
+        } else {
+            log.info(
+                    "Not invalidating resources, as the request to disable namespaces was unsuccessful",
+                    SafeArg.of("response", response));
+        }
+
         return response;
     }
 
-    public ReenableNamespacesResponse reEnable(ReenableNamespacesRequest request) {
+    public SingleNodeUpdateResponse reEnable(ReenableNamespacesRequest request) {
         return disabledNamespaces.reEnable(request);
     }