Support external tablet files outside Accumulo volumes

core/src/main/java/org/apache/accumulo/core/gc/Reference.java

@@ -36,6 +36,11 @@ public interface Reference {
    */
   boolean isScan();
 
+  /**
+   * Returns true when this reference points to a file outside Accumulo's defined Volumes.
+   */
+  boolean isExternal();
+
   /**
    * Get the {@link TableId} of the reference.
    */

core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java

@@ -18,9 +18,10 @@
  */
 package org.apache.accumulo.core.gc;
 
-import java.util.Objects;
+import static java.util.Objects.requireNonNull;
 
 import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
 import org.apache.accumulo.core.metadata.ScanServerRefTabletFile;
 import org.apache.accumulo.core.metadata.StoredTabletFile;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema;
@@ -35,32 +36,52 @@ public class ReferenceFile implements Reference, Comparable<ReferenceFile> {
   public final TableId tableId; // 2a
   public final boolean isScan;
   public final boolean isDirectory;
+  public final boolean isExternal;
 
   // the exact path from the file reference string that is stored in the metadata
   protected final String metadataPath;
 
   protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan,
       boolean isDirectory) {
-    this.tableId = Objects.requireNonNull(tableId);
-    this.metadataPath = Objects.requireNonNull(metadataPath);
+    this(tableId, metadataPath, isScan, isDirectory, false);
+  }
+
+  protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan, boolean isDirectory,
+      boolean isExternal) {
+    this.tableId = requireNonNull(tableId);
+    this.metadataPath = requireNonNull(metadataPath);
     this.isScan = isScan;
     this.isDirectory = isDirectory;
+    this.isExternal = isExternal;
   }
 
   public static ReferenceFile forFile(TableId tableId, StoredTabletFile tabletFile) {
-    return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false, false);
+    return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false, false,
+        tabletFile.isExternal());
+  }
+
+  public static ReferenceFile forFile(TableId tableId, ReferencedTabletFile tabletFile) {
+    return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), false, false,
+        tabletFile.isExternal());
   }
 
   public static ReferenceFile forFile(TableId tableId, Path metadataPathPath) {
     return new ReferenceFile(tableId, metadataPathPath.toString(), false, false);
   }
 
   public static ReferenceFile forScan(TableId tableId, ScanServerRefTabletFile tabletFile) {
-    return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false);
+    return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false,
+        tabletFile.isExternal());
   }
 
   public static ReferenceFile forScan(TableId tableId, StoredTabletFile tabletFile) {
-    return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true, false);
+    return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true, false,
+        tabletFile.isExternal());
+  }
+
+  public static ReferenceFile forScan(TableId tableId, ReferencedTabletFile tabletFile) {
+    return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false,
+        tabletFile.isExternal());
   }
 
   public static ReferenceFile forScan(TableId tableId, Path metadataPathPath) {
@@ -82,6 +103,11 @@ public boolean isScan() {
     return isScan;
   }
 
+  @Override
+  public boolean isExternal() {
+    return isExternal;
+  }
+
   @Override
   public TableId getTableId() {
     return tableId;

core/src/main/java/org/apache/accumulo/core/metadata/ExternalTabletFile.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import static java.util.Objects.requireNonNull;
+
+import org.apache.accumulo.core.data.Range;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A reference to a tablet file whose path is not in Accumulo's conventional tables directory. An
+ * external file is still owned by Accumulo and is eligible for garbage collection when no longer
+ * referenced.
+ */
+public class ExternalTabletFile extends ReferencedTabletFile {
+
+  public ExternalTabletFile(Path path) {
+    this(path, new Range());
+  }
+
+  public ExternalTabletFile(Path path, Range range) {
+    super(requireNonNull(path), range, true);
+  }
+
+  public static ExternalTabletFile of(Path path) {
+    return new ExternalTabletFile(path);
+  }
+
+  public static ExternalTabletFile of(Path path, Range range) {
+    return new ExternalTabletFile(path, range);
+  }
+}

core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java

@@ -18,6 +18,7 @@
  */
 package org.apache.accumulo.core.metadata;
 
+import static java.util.Objects.requireNonNull;
 import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR;
 
 import java.net.URI;
@@ -154,6 +155,7 @@ public static FileParts parsePath(Path filePath) {
   }
 
   private final FileParts parts;
+  private final boolean external;
 
   private static final Logger log = LoggerFactory.getLogger(ReferencedTabletFile.class);
   private static final String HDFS_TABLES_DIR_NAME = HDFS_TABLES_DIR.substring(1);
@@ -171,41 +173,64 @@ public ReferencedTabletFile(Path metaPath) {
    * qualify an absolute path or create a new file.
    */
   public ReferencedTabletFile(Path metaPath, Range range) {
-    super(Objects.requireNonNull(metaPath), range);
-    log.trace("Parsing TabletFile from {}", metaPath);
-    parts = parsePath(metaPath);
+    this(metaPath, range, false);
+  }
+
+  protected ReferencedTabletFile(Path metaPath, Range range, boolean external) {
+    super(requireNonNull(metaPath), range);
+    this.external = external;
+    if (external) {
+      Preconditions.checkArgument(metaPath.toUri().getScheme() != null,
+          constructErrorMsg(metaPath));
+      ValidationUtil.validateFileName(metaPath.getName());
+      Preconditions.checkArgument(!isVolumePath(metaPath),
+          "External tablet file path must not use the Accumulo table layout: %s", metaPath);
+      parts = null;
+    } else {
+      log.trace("Parsing TabletFile from {}", metaPath);
+      parts = parsePath(metaPath);
+    }
   }
 
   public String getVolume() {
+    Preconditions.checkState(!external, "External tablet files do not have an Accumulo volume");
     return parts.getVolume();
   }
 
   public TableId getTableId() {
+    Preconditions.checkState(!external,
+        "External tablet files do not have a path-derived table ID");
     return parts.getTableId();
   }
 
   public String getTabletDir() {
+    Preconditions.checkState(!external, "External tablet files do not have a tablet directory");
     return parts.getTabletDir();
   }
 
   @Override
   public String getFileName() {
-    return parts.getFileName();
+    return external ? path.getName() : parts.getFileName();
+  }
+
+  public boolean isExternal() {
+    return external;
   }
 
   /**
    * Return a string for opening and reading the tablet file. Doesn't have to be exact string in
    * metadata.
    */
   public String getNormalizedPathStr() {
-    return parts.getNormalizedPath();
+    return external ? path.toString() : parts.getNormalizedPath();
   }
 
   /**
    * New file was written to metadata so return a StoredTabletFile
    */
   public StoredTabletFile insert() {
-    return StoredTabletFile.of(getPath(), getRange());
+    return external ? StoredTabletFile.ofExternal(getPath(), getRange())
+        : StoredTabletFile.of(getPath(), getRange());
   }
 
   @Override
@@ -216,20 +241,56 @@ public int compareTo(ReferencedTabletFile o) {
   @Override
   public boolean equals(Object obj) {
     if (obj instanceof ReferencedTabletFile that) {
-      return parts.getNormalizedPath().equals(that.parts.getNormalizedPath())
+      return getNormalizedPathStr().equals(that.getNormalizedPathStr()) && external == that.external
           && range.equals(that.range);
     }
     return false;
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(parts.getNormalizedPath(), range);
+    return Objects.hash(getNormalizedPathStr(), external, range);
   }
 
   @Override
   public String toString() {
-    return parts.getNormalizedPath();
+    return getNormalizedPathStr();
+  }
+
+  /**
+   * Determines whether a path uses Accumulo's conventional
+   * {@code tables/<tableId>/<tabletDir>/<fileName>} layout.
+   */
+  public static boolean isVolumePath(Path path) {
+    try {
+      parsePath(path);
+      return true;
+    } catch (RuntimeException e) {
+      return false;
+    }
+  }
+
+  /**
+   * Determines whether a path is a conventional Accumulo tablet directory.
+   */
+  public static boolean isTabletDirPath(Path path) {
+    if (path == null || path.toUri().getScheme() == null) {
+      return false;
+    }
+    Path tableIdPath = path.getParent();
+    if (tableIdPath == null) {
+      return false;
+    }
+    Path tablesPath = tableIdPath.getParent();
+    if (tablesPath == null || !HDFS_TABLES_DIR_NAME.equals(tablesPath.getName())) {
+      return false;
+    }
+    try {
+      TableId.of(tableIdPath.getName());
+      return true;
+    } catch (RuntimeException e) {
+      return false;
+    }
   }
 
   public static ReferencedTabletFile of(final Path path) {

core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java

@@ -18,12 +18,15 @@
  */
 package org.apache.accumulo.core.metadata;
 
+import static java.util.Objects.requireNonNull;
+
 import java.net.URI;
 import java.util.Objects;
 import java.util.UUID;
 
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
@@ -35,21 +38,23 @@ public class ScanServerRefTabletFile extends ReferencedTabletFile {
   private final Text uuid;
 
   public ScanServerRefTabletFile(UUID serverLockUUID, String serverAddress, String file) {
-    super(new Path(URI.create(file)));
-    this.serverAddress = new Text(serverAddress);
-    uuid = new Text(serverLockUUID.toString());
+    this(new Path(URI.create(file)), new Text(serverAddress),
+        new Text(requireNonNull(serverLockUUID).toString()));
   }
 
   public ScanServerRefTabletFile(String file, String serverAddress, UUID serverLockUUID) {
-    super(new Path(URI.create(file)));
-    this.serverAddress = new Text(serverAddress);
-    this.uuid = new Text(serverLockUUID.toString());
+    this(new Path(URI.create(file)), new Text(serverAddress),
+        new Text(requireNonNull(serverLockUUID).toString()));
   }
 
   public ScanServerRefTabletFile(Key k) {
-    super(new Path(URI.create(k.getColumnQualifier().toString())));
-    serverAddress = k.getColumnFamily();
-    uuid = k.getRow();
+    this(new Path(URI.create(k.getColumnQualifier().toString())), k.getColumnFamily(), k.getRow());
+  }
+
+  private ScanServerRefTabletFile(Path path, Text serverAddress, Text uuid) {
+    super(path, new Range(), !ReferencedTabletFile.isVolumePath(path));
+    this.serverAddress = serverAddress;
+    this.uuid = uuid;
   }
 
   public Mutation putMutation() {