Skip to content

Commit ec112ac

Browse files
author
Lewis Hemens
authored
Require defaultLocation to be set in BigQuery (#1353)
* Enforce defaultLocation requirement in core and cli initialization * Add a test * Bump version
1 parent b06c329 commit ec112ac

File tree

11 files changed

+116
-50
lines changed

11 files changed

+116
-50
lines changed

‎cli/index.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ const timeoutOption: INamedOption<yargs.Options> = {
206206
};
207207

208208
const defaultDatabaseOptionName = "default-database";
209+
const defaultLocationOptionName = "default-location";
209210
const skipInstallOptionName = "skip-install";
210211
const includeSchedulesOptionName = "include-schedules";
211212
const includeEnvironmentsOptionName = "include-environments";
@@ -264,6 +265,28 @@ export function runCli() {
264265
}
265266
}
266267
},
268+
{
269+
name: defaultLocationOptionName,
270+
option: {
271+
describe:
272+
"The default BigQuery location to use. See https://cloud.google.com/bigquery/docs/locations for supported values."
273+
},
274+
check: (argv: yargs.Arguments<any>) => {
275+
if (
276+
argv[defaultLocationOptionName] &&
277+
!["bigquery"].includes(argv[warehouseOption.name])
278+
) {
279+
throw new Error(
280+
`The --${defaultLocationOptionName} flag is only used for BigQuery.`
281+
);
282+
}
283+
if (!argv[defaultLocationOptionName] && argv[warehouseOption.name] === "bigquery") {
284+
throw new Error(
285+
`The --${defaultLocationOptionName} flag is required for BigQuery projects. Please run 'dataform help init' for more information.`
286+
);
287+
}
288+
}
289+
},
267290
{
268291
name: skipInstallOptionName,
269292
option: {
@@ -293,6 +316,7 @@ export function runCli() {
293316
{
294317
warehouse: argv[warehouseOption.name],
295318
defaultDatabase: argv[defaultDatabaseOptionName],
319+
defaultLocation: argv[defaultLocationOptionName],
296320
useRunCache: false
297321
},
298322
{

‎core/session.ts‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,13 @@ export class Session {
375375

376376
public compile(): dataform.CompiledGraph {
377377
this.indexedActions = new ActionIndex(this.adapter(), this.actions);
378+
379+
if (this.config.warehouse === "bigquery" && !this.config.defaultLocation) {
380+
this.compileError(
381+
"A defaultLocation is required for BigQuery. This can be configured in dataform.json."
382+
);
383+
}
384+
378385
const compiledGraph = dataform.CompiledGraph.create({
379386
projectConfig: this.config,
380387
tables: this.compileGraphChunk(
@@ -615,8 +622,11 @@ export class Session {
615622
}
616623

617624
// materialized
618-
if(!!table.materialized){
619-
if (table.type !== "view" || (this.config.warehouse !== "snowflake" && this.config.warehouse !== "bigquery")) {
625+
if (!!table.materialized) {
626+
if (
627+
table.type !== "view" ||
628+
(this.config.warehouse !== "snowflake" && this.config.warehouse !== "bigquery")
629+
) {
620630
this.compileError(
621631
new Error(`The 'materialized' option is only valid for Snowflake and BigQuery views`),
622632
table.fileName,
@@ -735,35 +745,42 @@ export class Session {
735745
// BigQuery config
736746
if (!!table.bigquery) {
737747
if (
738-
(table.bigquery.partitionBy || table.bigquery.clusterBy?.length || table.bigquery.partitionExpirationDays
739-
|| table.bigquery.requirePartitionFilter) &&
748+
(table.bigquery.partitionBy ||
749+
table.bigquery.clusterBy?.length ||
750+
table.bigquery.partitionExpirationDays ||
751+
table.bigquery.requirePartitionFilter) &&
740752
table.type === "view"
741753
) {
742754
this.compileError(
743755
`partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views; they are only valid for tables`,
744756
table.fileName,
745757
table.target
746758
);
747-
}
748-
else if (
749-
(!table.bigquery.partitionBy && (table.bigquery.partitionExpirationDays || table.bigquery.requirePartitionFilter)) &&
759+
} else if (
760+
!table.bigquery.partitionBy &&
761+
(table.bigquery.partitionExpirationDays || table.bigquery.requirePartitionFilter) &&
750762
table.type === "table"
751763
) {
752764
this.compileError(
753765
`requirePartitionFilter/partitionExpirationDays are not valid for non partitioned BigQuery tables`,
754766
table.fileName,
755767
table.target
756768
);
757-
}
758-
else if(table.bigquery.additionalOptions) {
759-
if(table.bigquery.partitionExpirationDays && table.bigquery.additionalOptions.partition_expiration_days) {
769+
} else if (table.bigquery.additionalOptions) {
770+
if (
771+
table.bigquery.partitionExpirationDays &&
772+
table.bigquery.additionalOptions.partition_expiration_days
773+
) {
760774
this.compileError(
761775
`partitionExpirationDays has been declared twice`,
762776
table.fileName,
763777
table.target
764778
);
765779
}
766-
if (table.bigquery.requirePartitionFilter && table.bigquery.additionalOptions.require_partition_filter) {
780+
if (
781+
table.bigquery.requirePartitionFilter &&
782+
table.bigquery.additionalOptions.require_partition_filter
783+
) {
767784
this.compileError(
768785
`requirePartitionFilter has been declared twice`,
769786
table.fileName,

‎examples/backwards_compatibility/dataform.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@
22
"warehouse": "bigquery",
33
"defaultDatabase": "tada-analytics",
44
"defaultSchema": "dataform_example",
5-
"assertionSchema": "dataform_assertions"
5+
"assertionSchema": "dataform_assertions",
6+
"defaultLocation": "US"
67
}

‎examples/common_v1/dataform.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
22
"defaultSchema": "df_integration_test",
3-
"assertionSchema": "df_integration_test_assertions"
3+
"assertionSchema": "df_integration_test_assertions",
4+
"defaultLocation": "US"
45
}

‎examples/common_v2/dataform.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,6 @@
33
"defaultDatabase": "tada-analytics",
44
"defaultSchema": "df_integration_test",
55
"assertionSchema": "df_integration_test_assertions",
6-
"useRunCache": true
6+
"useRunCache": true,
7+
"defaultLocation": "US"
78
}

‎examples/never_finishes_compiling/dataform.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"warehouse": "redshift",
33
"defaultSchema": "df_integration_test",
4-
"assertionSchema": "df_integration_test_assertions"
4+
"assertionSchema": "df_integration_test_assertions",
5+
"defaultLocation": "US"
56
}

‎tests/api/api.spec.ts‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ suite("@dataform/api", () => {
224224
//
225225
// op_d +---> tab_a
226226
const TEST_GRAPH_WITH_TAGS: dataform.ICompiledGraph = dataform.CompiledGraph.create({
227-
projectConfig: { warehouse: "bigquery" },
227+
projectConfig: { warehouse: "bigquery", defaultLocation: "US" },
228228
operations: [
229229
{
230230
target: { schema: "schema", name: "op_a" },
@@ -260,7 +260,7 @@ suite("@dataform/api", () => {
260260

261261
test("prune removes inline tables", async () => {
262262
const graph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
263-
projectConfig: { warehouse: "bigquery" },
263+
projectConfig: { warehouse: "bigquery", defaultLocation: "US" },
264264
tables: [
265265
{ target: { schema: "schema", name: "a" }, type: "table" },
266266
{
@@ -379,7 +379,7 @@ suite("@dataform/api", () => {
379379
suite("sql_generating", () => {
380380
test("bigquery_incremental", () => {
381381
const graph = dataform.CompiledGraph.create({
382-
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb" },
382+
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
383383
tables: [
384384
{
385385
target: {
@@ -428,7 +428,7 @@ suite("@dataform/api", () => {
428428

429429
test("bigquery_materialized", () => {
430430
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
431-
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb" },
431+
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
432432
tables: [
433433
{
434434
target: {
@@ -492,7 +492,7 @@ suite("@dataform/api", () => {
492492

493493
test("snowflake_materialized", () => {
494494
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
495-
projectConfig: { warehouse: "snowflake"},
495+
projectConfig: { warehouse: "snowflake" },
496496
tables: [
497497
{
498498
target: {
@@ -524,8 +524,7 @@ suite("@dataform/api", () => {
524524
tasks: [
525525
{
526526
type: "statement",
527-
statement:
528-
`create or replace materialized view "schema"."materialized" as select 1 as test`
527+
statement: `create or replace materialized view "schema"."materialized" as select 1 as test`
529528
}
530529
],
531530
dependencyTargets: [],
@@ -716,7 +715,7 @@ suite("@dataform/api", () => {
716715

717716
test("bigquery_partitionby", () => {
718717
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
719-
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb" },
718+
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
720719
tables: [
721720
{
722721
target: {
@@ -783,7 +782,7 @@ suite("@dataform/api", () => {
783782

784783
test("bigquery_options", () => {
785784
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
786-
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb" },
785+
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
787786
tables: [
788787
{
789788
target: {
@@ -795,8 +794,8 @@ suite("@dataform/api", () => {
795794
bigquery: {
796795
partitionBy: "DATE(test)",
797796
clusterBy: [],
798-
partitionExpirationDays : 1,
799-
requirePartitionFilter : true
797+
partitionExpirationDays: 1,
798+
requirePartitionFilter: true
800799
}
801800
},
802801
{
@@ -852,7 +851,7 @@ suite("@dataform/api", () => {
852851

853852
test("bigquery_clusterby", () => {
854853
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
855-
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb" },
854+
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
856855
tables: [
857856
{
858857
target: {
@@ -919,7 +918,7 @@ suite("@dataform/api", () => {
919918

920919
test("bigquery_additional_options", () => {
921920
const testGraph: dataform.ICompiledGraph = dataform.CompiledGraph.create({
922-
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb" },
921+
projectConfig: { warehouse: "bigquery", defaultDatabase: "deeb", defaultLocation: "US" },
923922
tables: [
924923
{
925924
target: {
@@ -929,10 +928,10 @@ suite("@dataform/api", () => {
929928
type: "table",
930929
query: "select 1 as test",
931930
bigquery: {
932-
additionalOptions : {
933-
partition_expiration_days : "1",
934-
"require_partition_filter" : "true",
935-
friendly_name : '"friendlyName"',
931+
additionalOptions: {
932+
partition_expiration_days: "1",
933+
require_partition_filter: "true",
934+
friendly_name: '"friendlyName"'
936935
}
937936
}
938937
},
@@ -958,7 +957,7 @@ suite("@dataform/api", () => {
958957
{
959958
type: "statement",
960959
statement:
961-
"create or replace table `deeb.schema.additional_options` OPTIONS(partition_expiration_days=1,require_partition_filter=true,friendly_name=\"friendlyName\")as select 1 as test"
960+
'create or replace table `deeb.schema.additional_options` OPTIONS(partition_expiration_days=1,require_partition_filter=true,friendly_name="friendlyName")as select 1 as test'
962961
}
963962
],
964963
dependencyTargets: [],
@@ -1114,7 +1113,8 @@ suite("@dataform/api", () => {
11141113
warehouse: "bigquery",
11151114
defaultSchema: "foo",
11161115
assertionSchema: "bar",
1117-
defaultDatabase: "database"
1116+
defaultDatabase: "database",
1117+
defaultLocation: "US"
11181118
},
11191119
runConfig: {
11201120
fullRefresh: true
@@ -1469,7 +1469,8 @@ suite("@dataform/api", () => {
14691469
projectConfig: {
14701470
warehouse: "bigquery",
14711471
defaultSchema: "foo",
1472-
assertionSchema: "bar"
1472+
assertionSchema: "bar",
1473+
defaultLocation: "US"
14731474
},
14741475
warehouseState: {
14751476
tables: []
@@ -1530,7 +1531,8 @@ suite("@dataform/api", () => {
15301531
projectConfig: {
15311532
warehouse: "bigquery",
15321533
defaultSchema: "foo",
1533-
assertionSchema: "bar"
1534+
assertionSchema: "bar",
1535+
defaultLocation: "US"
15341536
},
15351537
warehouseState: {
15361538
tables: []

‎tests/cli/cli.spec.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ suite(__filename, () => {
2727
projectDir,
2828
"--skip-install",
2929
"--default-database",
30-
"dataform-integration-tests"
30+
"dataform-integration-tests",
31+
"--default-location",
32+
"US"
3133
])
3234
);
3335

@@ -91,6 +93,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2}
9193
defaultSchema: "dataform",
9294
assertionSchema: "dataform_assertions",
9395
defaultDatabase: "dataform-integration-tests",
96+
defaultLocation: "US",
9497
useRunCache: false,
9598
vars: {
9699
testVar1: "testValue1",
@@ -148,6 +151,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2}
148151
projectConfig: {
149152
assertionSchema: "dataform_assertions",
150153
defaultDatabase: "dataform-integration-tests",
154+
defaultLocation: "US",
151155
defaultSchema: "dataform",
152156
useRunCache: false,
153157
warehouse: "bigquery",

0 commit comments

Comments
 (0)