feat: multi-address DNS resolution for contact points and connections (DRIVER-201)

core/src/main/java/com/datastax/dse/driver/api/core/auth/DseGssApiAuthProviderBase.java

@@ -27,6 +27,7 @@
 import com.datastax.oss.protocol.internal.util.Bytes;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.Nullable;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedActionException;
@@ -319,7 +320,7 @@ protected GssApiAuthenticator(
                 SUPPORTED_MECHANISMS,
                 options.getAuthorizationId(),
                 protocol,
-                ((InetSocketAddress) endPoint.resolve()).getAddress().getCanonicalHostName(),
+                serverName(endPoint),
                 options.getSaslProperties(),
                 null);
       } catch (LoginException | SaslException e) {
@@ -328,6 +329,23 @@ protected GssApiAuthenticator(
       this.endPoint = endPoint;
     }
 
+    /**
+     * The host name to build the Kerberos service principal from.
+     *
+     * <p>Prefers the canonical name of the resolved address, which is what Kerberos expects. The
+     * driver's own endpoints always hand this a resolved address — the channel carries an endpoint
+     * bound to the address it connected to (see {@code PinnableEndPoint}) — but a custom {@link
+     * EndPoint} implementation may still yield an unresolved one, in which case {@code
+     * getAddress()} is null. Fall back to the host string rather than throwing a {@link
+     * NullPointerException}: the hostname is usually the right service name anyway, and a failed
+     * reverse lookup should not take authentication down.
+     */
+    private static String serverName(EndPoint endPoint) {
+      InetSocketAddress address = (InetSocketAddress) endPoint.resolve();
+      InetAddress inetAddress = address.getAddress();
+      return inetAddress != null ? inetAddress.getCanonicalHostName() : address.getHostString();
+    }
+
     @NonNull
     @Override
     protected ByteBuffer getMechanism() {

core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java

@@ -701,8 +701,14 @@ public enum DefaultDriverOption implements DriverOption {
   CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"),
 
   /**
-   * Whether to forcibly add original contact points held by MetadataManager to the reconnection
-   * plan, in case there is no live nodes available according to LBP. Experimental.
+   * Whether to append the original contact points held by MetadataManager to the reconnection plan,
+   * after the live nodes reported by the load balancing policy. Defaults to {@code true}.
+   *
+   * <p>This is also the driver's DNS re-resolution path. Contact points are appended as-is, still
+   * unresolved hostnames, and each is expanded to its current DNS IPs at connection time through
+   * Netty's configured resolver. Metadata nodes, in contrast, hold an already-resolved endpoint
+   * that is never re-resolved. Keeping this enabled lets control-connection reconnects re-resolve
+   * the original hostnames and pick up new IPs once the live-node plan is exhausted.
    *
    * <p>Value-type: boolean
    */
@@ -837,7 +843,11 @@ public enum DefaultDriverOption implements DriverOption {
    * Whether to resolve the addresses passed to `basic.contact-points`.
    *
    * <p>Value-type: boolean
+   *
+   * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all
+   *     their DNS-mapped IPs lazily at connection time. Setting this option has no effect.
    */
+  @Deprecated
   RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"),
 
   /**

core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java

@@ -245,6 +245,9 @@ private void readObject(ObjectInputStream stream) throws InvalidObjectException
     throw new InvalidObjectException("Proxy required");
   }
 
+  // RESOLVE_CONTACT_POINTS is deprecated and has no effect, but it is still a driver option, so the
+  // defaults map stays complete by carrying its reference.conf value.
+  @SuppressWarnings("deprecation")
   protected static void fillWithDriverDefaults(OptionsMap map) {
     Duration initQueryTimeout = Duration.ofSeconds(5);
     Duration requestTimeout = Duration.ofSeconds(2);
@@ -369,7 +372,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true);
-    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false);
+    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true);
     map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true);
     map.put(TypedDriverOption.REPREPARE_ENABLED, true);
     map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false);

core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java

@@ -600,7 +600,15 @@ public String toString() {
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_AGREEMENT_WARN =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN);
-  /** Whether to forcibly try original contacts if no live nodes are available */
+  /**
+   * Whether to append the original contact points to the control-connection reconnection plan,
+   * after the live nodes reported by the load balancing policy (defaults to {@code true}).
+   *
+   * <p>Contact points are appended as-is (unresolved hostnames); each is expanded to all of its
+   * current DNS IPs at connection time, which is also the driver's DNS re-resolution mechanism. The
+   * append is skipped for topology monitors that re-resolve node addresses themselves (such as the
+   * cloud/proxy monitors).
+   */
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN);
@@ -664,7 +672,13 @@ public String toString() {
   /** The coalescer reschedule interval. */
   public static final TypedDriverOption<Duration> COALESCER_INTERVAL =
       new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION);
-  /** Whether to resolve the addresses passed to `basic.contact-points`. */
+  /**
+   * Whether to resolve the addresses passed to `basic.contact-points`.
+   *
+   * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all
+   *     their DNS-mapped IPs lazily at connection time. Setting this option has no effect.
+   */
+  @Deprecated
   public static final TypedDriverOption<Boolean> RESOLVE_CONTACT_POINTS =
       new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN);
   /**

core/src/main/java/com/datastax/oss/driver/api/core/metadata/EndPoint.java

@@ -18,24 +18,53 @@
 package com.datastax.oss.driver.api.core.metadata;
 
 import edu.umd.cs.findbugs.annotations.NonNull;
-import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 
 /**
  * Encapsulates the information needed to open connections to a node.
  *
  * <p>By default, the driver assumes plain TCP connections, and this is just a wrapper around an
- * {@link InetSocketAddress}. However, more complex deployment scenarios might use a custom
+ * {@link java.net.InetSocketAddress}. However, more complex deployment scenarios might use a custom
  * implementation that contains additional information; for example, if the nodes are accessed
  * through a proxy with SNI routing, an SNI server name is needed in addition to the proxy address.
  */
 public interface EndPoint {
 
   /**
-   * Resolves this instance to a socket address.
+   * Resolves this instance to the socket address connections should be opened to.
    *
    * <p>This will be called each time the driver opens a new connection to the node. The returned
    * address cannot be null.
+   *
+   * <p><b>Returning a hostname is fine, and is how multi-address support works.</b> The returned
+   * address need not be resolved: an {@linkplain java.net.InetSocketAddress#isUnresolved()
+   * unresolved} {@link java.net.InetSocketAddress} is expanded by the driver to <b>every</b>
+   * address the name maps to, and each one is tried in turn until a connection succeeds. That is
+   * what {@link com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint} does for contact
+   * points backed by a hostname, so a single unreachable IP behind a multi-record name no longer
+   * fails the connection.
+   *
+   * <p><b>Implementations must not resolve names themselves, and must not block.</b> The driver
+   * calls this from its admin event loop, and it performs the expansion through Netty's configured
+   * {@code AddressResolverGroup} — the same resolver an unresolved address reaches when it is
+   * handed to {@code Bootstrap.connect()}. Looking the name up here instead (for example with
+   * {@link java.net.InetAddress#getAllByName(String)}) would both block that loop and bypass a
+   * custom resolver installed via {@code NettyOptions#afterBootstrapInitialized(Bootstrap)}.
+   *
+   * <p><b>Callers must not assume the returned address is resolved.</b> It is for a node discovered
+   * from {@code system.peers} (built from that node's physical broadcast RPC address) and for the
+   * node the control connection is on (bound to the address that connection reached). It is
+   * <b>not</b> for a node reached through the Cloud SNI proxy, or through a cloud private-endpoint
+   * client route: there the address is the configured hostname, and {@link
+   * java.net.InetSocketAddress#getAddress()} returns {@code null}. Read the host with {@link
+   * java.net.InetSocketAddress#getHostString()}, which yields whichever of the two the address
+   * carries and never triggers a reverse lookup.
+   *
+   * @apiNote <b>Timeout note:</b> when a name expands to several addresses they are tried in
+   *     sequence, so if every attempt times out the worst-case time before the node is declared
+   *     unreachable is {@code N × advanced.connection.connect-timeout}. In practice DNS round-robin
+   *     entries have only a small number of records, so this is rarely a concern, but it is worth
+   *     bearing in mind when configuring connect timeouts.
    */
   @NonNull
   SocketAddress resolve();

core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java

@@ -166,11 +166,15 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad
    * <p>Contact points can also be provided statically in the configuration. If both are specified,
    * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042.
    *
-   * <p>Contrary to the configuration, DNS names with multiple A-records will not be handled here.
-   * If you need that, extract them manually with {@link java.net.InetAddress#getAllByName(String)}
-   * before calling this method. Similarly, if you need connect addresses to stay unresolved, make
-   * sure you pass unresolved instances here (see {@code advanced.resolve-contact-points} in the
-   * configuration for more explanations).
+   * <p>The driver automatically expands any contact point backed by an unresolved hostname to all
+   * its DNS-mapped IPs at connection time (through Netty's configured resolver, so a custom {@code
+   * AddressResolverGroup} still applies), so passing a single hostname is sufficient to try all its
+   * IPs on initial connect. This applies equally to hostnames provided here programmatically (build
+   * an unresolved {@link InetSocketAddress} with {@link InetSocketAddress#createUnresolved(String,
+   * int)} to opt in) and to hostnames specified in the configuration. An already-resolved address
+   * passed here (the common case when constructing an {@code InetSocketAddress} directly from a
+   * hostname, which resolves eagerly) is used as provided, with no further expansion. The {@code
+   * advanced.resolve-contact-points} option is deprecated and has no effect.
    */
   @NonNull
   public SelfT addContactPoints(@NonNull Collection<InetSocketAddress> contactPoints) {
@@ -741,6 +745,12 @@ public SelfT withCloudSecureConnectBundle(@NonNull InputStream cloudConfigInputS
    *
    * <p>For more information, please refer to the DataStax Astra documentation.
    *
+   * <p>A proxy given as a hostname is resolved at connection time, to <b>all</b> of its addresses,
+   * and each is tried in turn. That holds however the {@link InetSocketAddress} was built: the
+   * driver keeps a proxy hostname unresolved internally, so passing one that the ordinary {@code
+   * InetSocketAddress(String, int)} constructor already resolved does not bind the session to that
+   * single address.
+   *
    * @param cloudProxyAddress The address of the Cloud proxy to use.
    * @see <a href="https://en.wikipedia.org/wiki/Server_Name_Indication">Server Name Indication</a>
    */
@@ -957,11 +967,10 @@ protected final CompletionStage<CqlSession> buildDefaultSessionAsync() {
         programmaticArguments = programmaticArgumentsBuilder.build();
       }
 
-      boolean resolveAddresses =
-          defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false);
-
+      // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved
+      // hostnames, and expanded to all their DNS IPs at connection time by ChannelFactory.
       Set<EndPoint> contactPoints =
-          ContactPoints.merge(programmaticContactPoints, configContactPoints, resolveAddresses);
+          ContactPoints.merge(programmaticContactPoints, configContactPoints, false);
 
       if (keyspace == null && defaultConfig.isDefined(DefaultDriverOption.SESSION_KEYSPACE)) {
         keyspace =