Firestore clone support

CHANGELOG.md

@@ -1,2 +1,3 @@
+- Add a command `firebase firestore:databases:clone` to clone a Firestore database (#9262).
 - Add JSON format support for Cloud Functions secrets with `--format json` flag and auto-detection from file extensions (#1745)
 - `firebase dataconnect:sdk:generate` will run `init dataconnect:sdk` automatically if no SDKs are configured (#9325).

src/commands/firestore-databases-clone.ts

@@ -0,0 +1,139 @@
+import * as clc from "colorette";
+
+import { Command } from "../command";
+import * as fsi from "../firestore/api";
+import * as types from "../firestore/api-types";
+import { getCurrentMinuteAsIsoString, parseDatabaseName } from "../firestore/util";
+import { logger } from "../logger";
+import { requirePermissions } from "../requirePermissions";
+import { Emulators } from "../emulator/types";
+import { warnEmulatorNotSupported } from "../emulator/commandUtils";
+import { EncryptionType, FirestoreOptions } from "../firestore/options";
+import { PrettyPrint } from "../firestore/pretty-print";
+import { FirebaseError } from "../error";
+
+export const command = new Command("firestore:databases:clone <sourceDatabase> <targetDatabase>")
+  .description("clone one Firestore database to another")
+  .option(
+    "-e, --encryption-type <encryptionType>",
+    `encryption method of the cloned database; one of ${EncryptionType.USE_SOURCE_ENCRYPTION} (default), ` +
+      `${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}, ${EncryptionType.GOOGLE_DEFAULT_ENCRYPTION}`,
+  )
+  // TODO(b/356137854): Remove allowlist only message once feature is public GA.
+  .option(
+    "-k, --kms-key-name <kmsKeyName>",
+    "resource ID of the Cloud KMS key to encrypt the cloned database. This " +
+      "feature is allowlist only in initial launch",
+  )
+  .option(
+    "-s, --snapshot-time <snapshotTime>",
+    "snapshot time of the source database to use, in ISO 8601 format. Can be any minutely snapshot after the database's earliest version time. If unspecified, takes the most recent available snapshot",
+  )
+  .before(requirePermissions, ["datastore.databases.clone"])
+  .before(warnEmulatorNotSupported, Emulators.FIRESTORE)
+  .action(async (sourceDatabase: string, targetDatabase: string, options: FirestoreOptions) => {
+    const api = new fsi.FirestoreApi();
+    const printer = new PrettyPrint();
+    const helpCommandText = "See firebase firestore:databases:clone --help for more info.";
+
+    if (options.database) {
+      throw new FirebaseError(
+        `--database is not a supported flag for 'firestoree:databases:clone'. ${helpCommandText}`,
+      );
+    }
+
+    let snapshotTime: string;
+    if (options.snapshotTime) {
+      snapshotTime = options.snapshotTime;
+    } else {
+      snapshotTime = getCurrentMinuteAsIsoString();
+    }
+
+    let encryptionConfig: types.EncryptionConfig | undefined = undefined;
+    switch (options.encryptionType) {
+      case EncryptionType.GOOGLE_DEFAULT_ENCRYPTION:
+        throwIfKmsKeyNameIsSet(options.kmsKeyName);
+        encryptionConfig = { googleDefaultEncryption: {} };
+        break;
+      case EncryptionType.USE_SOURCE_ENCRYPTION:
+        throwIfKmsKeyNameIsSet(options.kmsKeyName);
+        encryptionConfig = { useSourceEncryption: {} };
+        break;
+      case EncryptionType.CUSTOMER_MANAGED_ENCRYPTION:
+        encryptionConfig = {
+          customerManagedEncryption: { kmsKeyName: getKmsKeyOrThrow(options.kmsKeyName) },
+        };
+        break;
+      case undefined:
+        throwIfKmsKeyNameIsSet(options.kmsKeyName);
+        break;
+      default:
+        throw new FirebaseError(`Invalid value for flag --encryption-type. ${helpCommandText}`);
+    }
+
+    // projects must be the same
+    const targetDatabaseName = parseDatabaseName(targetDatabase);
+    const parentProject = targetDatabaseName.projectId;
+    const targetDatabaseId = targetDatabaseName.databaseId;
+    const sourceProject = parseDatabaseName(sourceDatabase).projectId;
+    if (parentProject !== sourceProject) {
+      throw new FirebaseError(`Cloning across projects is not supported.`);
+    }
+    const lro: types.Operation = await api.cloneDatabase(
+      sourceProject,
+      {
+        database: sourceDatabase,
+        snapshotTime,
+      },
+      targetDatabaseId,
+      encryptionConfig,
+    );
+
+    if (lro.error) {
+      logger.error(
+        clc.bold(
+          `Clone to ${printer.prettyDatabaseString(targetDatabase)} failed. See below for details.`,
+        ),
+      );
+      printer.prettyPrintOperation(lro);
+    } else {
+      logger.info(
+        clc.bold(`Successfully initiated clone to ${printer.prettyDatabaseString(targetDatabase)}`),
+      );
+      logger.info(
+        "Please be sure to configure Firebase rules in your Firebase config file for\n" +
+          "the new database. By default, created databases will have closed rules that\n" +
+          "block any incoming third-party traffic.",
+      );
+      logger.info();
+      logger.info(`You can monitor the progress of this clone by executing this command:`);
+      logger.info();
+      logger.info(
+        `firebase firestore:operations:describe --database="${targetDatabaseId}" ${lro.name}`,
+      );
+      logger.info();
+      logger.info(
+        `Once the clone is complete, your database may be viewed at ${printer.firebaseConsoleDatabaseUrl(options.project, targetDatabaseId)}`,
+      );
+    }
+
+    return lro;
+
+    function throwIfKmsKeyNameIsSet(kmsKeyName: string | undefined): void {
+      if (kmsKeyName) {
+        throw new FirebaseError(
+          "--kms-key-name can only be set when specifying an --encryption-type " +
+            `of ${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}.`,
+        );
+      }
+    }
+
+    function getKmsKeyOrThrow(kmsKeyName: string | undefined): string {
+      if (kmsKeyName) return kmsKeyName;
+
+      throw new FirebaseError(
+        "--kms-key-name must be provided when specifying an --encryption-type " +
+          `of ${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}.`,
+      );
+    }
+  });

src/commands/index.ts

@@ -2,11 +2,11 @@
 /**
  * Loads all commands for our parser.
  */
 export function load(client: any): any {
   function loadCommand(name: string) {
     const t0 = process.hrtime.bigint();
     const { command: cmd } = require(`./${name}`);
     cmd.register(client);
     const t1 = process.hrtime.bigint();
     const diffMS = (t1 - t0) / BigInt(1e6);
     if (diffMS > 75) {
@@ -14,7 +14,7 @@
       // console.error(`Loading ${name} took ${diffMS}ms`);
     }
 
     return cmd.runner();
   }
 
   const t0 = process.hrtime.bigint();
@@ -115,6 +115,7 @@
   client.firestore.databases.update = loadCommand("firestore-databases-update");
   client.firestore.databases.delete = loadCommand("firestore-databases-delete");
   client.firestore.databases.restore = loadCommand("firestore-databases-restore");
+  client.firestore.databases.clone = loadCommand("firestore-databases-clone");
   client.firestore.backups = {};
   client.firestore.backups.schedules = {};
   client.firestore.backups.list = loadCommand("firestore-backups-list");

src/firestore/api-types.ts

@@ -219,6 +219,12 @@ export interface RestoreDatabaseReq {
   encryptionConfig?: EncryptionConfig;
 }
 
+export interface CloneDatabaseReq {
+  databaseId: string;
+  pitrSnapshot: PITRSnapshot;
+  encryptionConfig?: EncryptionConfig;
+}
+
 export enum RecurrenceType {
   DAILY = "DAILY",
   WEEKLY = "WEEKLY",
@@ -239,3 +245,8 @@ export type EncryptionConfig =
   | UseCustomerManagedEncryption
   | UseSourceEncryption
   | UseGoogleDefaultEncryption;
+
+export interface PITRSnapshot {
+  database: string;
+  snapshotTime: string;
+}

src/firestore/api.ts

@@ -902,6 +902,39 @@ export class FirestoreApi {
     return database;
   }
 
+  /**
+   * Clone one Firestore Database to another.
+   * @param project the source project ID
+   * @param pitrSnapshot Source database PITR snapshot specification
+   * @param databaseId ID of the target database
+   * @param encryptionConfig the encryption configuration of the new database
+   */
+  async cloneDatabase(
+    project: string,
+    pitrSnapshot: types.PITRSnapshot,
+    databaseId: string,
+    encryptionConfig?: types.EncryptionConfig,
+  ): Promise<types.Operation> {
+    const url = `/projects/${project}/databases:clone`;
+    const payload: types.CloneDatabaseReq = {
+      databaseId,
+      pitrSnapshot,
+      encryptionConfig,
+    };
+    const options = { queryParams: { databaseId: databaseId } };
+    const res = await this.apiClient.post<types.CloneDatabaseReq, types.Operation>(
+      url,
+      payload,
+      options,
+    );
+    const lro = res.body;
+    if (!lro) {
+      throw new FirebaseError("Not found");
+    }
+
+    return lro;
+  }
+
   /**
    * List the long-running Firestore operations.
    * @param project the Firebase project id.

src/firestore/options.ts

@@ -32,6 +32,9 @@ export interface FirestoreOptions extends Options {
   // CMEK
   encryptionType?: EncryptionType;
   kmsKeyName?: string;
+
+  // Clone
+  snapshotTime?: string;
 }
 
 export enum EncryptionType {

src/firestore/util.spec.ts

@@ -2,6 +2,32 @@ import { expect } from "chai";
 
 import * as util from "./util";
 
+describe("Database name parsing", () => {
+  it("should parse a database other than (default) correctly", () => {
+    const name = "projects/myproject/databases/named-db";
+    expect(util.parseDatabaseName(name)).to.eql({
+      projectId: "myproject",
+      databaseId: "named-db",
+    });
+  });
+
+  it("should parse the (default) database name correctly", () => {
+    const name = "projects/myproject/databases/(default)";
+    expect(util.parseDatabaseName(name)).to.eql({
+      projectId: "myproject",
+      databaseId: "(default)",
+    });
+  });
+
+  it("should work even if the name has a trailing slash", () => {
+    const name = "projects/myproject/databases/with-trailing-slash/";
+    expect(util.parseDatabaseName(name)).to.eql({
+      projectId: "myproject",
+      databaseId: "with-trailing-slash",
+    });
+  });
+});
+
 describe("IndexNameParsing", () => {
   it("should parse an index name correctly", () => {
     const name =
@@ -47,3 +73,13 @@ describe("IndexNameParsing", () => {
     });
   });
 });
+
+describe("Get current minute", () => {
+  it("should be a string in ISO 8601 format with no second or millisecond component", () => {
+    const currentMinuteString = util.getCurrentMinuteAsIsoString();
+    expect(currentMinuteString.endsWith("Z")).to.eql(true);
+    const asDate = new Date(Date.parse(currentMinuteString));
+    expect(asDate.getSeconds()).to.eql(0);
+    expect(asDate.getMilliseconds()).to.eql(0);
+  });
+});

src/firestore/util.ts

@@ -1,5 +1,10 @@
 import { FirebaseError } from "../error";
 
+interface DatabaseName {
+  projectId: string;
+  databaseId: string;
+}
+
 interface IndexName {
   projectId: string;
   databaseId: string;
@@ -14,6 +19,9 @@ interface FieldName {
   fieldPath: string;
 }
 
+// projects/$PROJECT_ID/databases/$DATABASE_ID
+const DATABASE_NAME_REGEX = /projects\/([^\/]+?)\/databases\/([^\/]+)/;
+
 // projects/$PROJECT_ID/databases/$DATABASE_ID/collectionGroups/$COLLECTION_GROUP_ID/indexes/$INDEX_ID
 const INDEX_NAME_REGEX =
   /projects\/([^\/]+?)\/databases\/([^\/]+?)\/collectionGroups\/([^\/]+?)\/indexes\/([^\/]*)/;
@@ -22,6 +30,25 @@ const INDEX_NAME_REGEX =
 const FIELD_NAME_REGEX =
   /projects\/([^\/]+?)\/databases\/([^\/]+?)\/collectionGroups\/([^\/]+?)\/fields\/([^\/]*)/;
 
+/**
+ * Parse a Database name into useful pieces.
+ */
+export function parseDatabaseName(name?: string): DatabaseName {
+  if (!name) {
+    throw new FirebaseError(`Cannot parse undefined database name.`);
+  }
+
+  const m = name.match(DATABASE_NAME_REGEX);
+  if (!m || m.length < 3) {
+    throw new FirebaseError(`Error parsing database name: ${name}`);
+  }
+
+  return {
+    projectId: m[1],
+    databaseId: m[2],
+  };
+}
+
 /**
  * Parse an Index name into useful pieces.
  */
@@ -66,3 +93,17 @@ export function parseFieldName(name: string): FieldName {
 export function booleanXOR(a: boolean, b: boolean): boolean {
   return !!(Number(a) - Number(b));
 }
+
+/**
+ * Get the current time truncated to minutes.
+ *
+ * For some tasks, eg. database clone, this will be the most recent available snapshot time.
+ *
+ * @returns A Protobuf-friendly ISO string of the current time in the UTC timezone
+ */
+export function getCurrentMinuteAsIsoString(): string {
+  // Note that JS Dates support millisecond precision at max and that toISOString forces the UTC timezone, which will be represented by the (Protobuf-friendly) "Z"
+  const mostRecentTimestamp = new Date(Date.now());
+  mostRecentTimestamp.setSeconds(0, 0);
+  return mostRecentTimestamp.toISOString();
+}