From 4b2043c0deeded6ccd8e6117e9e45bb9b1d57b16 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 10 Jul 2025 11:51:06 +0100 Subject: [PATCH 01/54] Store the FQN of the provider used in PSS in representations of the parsed config. This can only be done once modules have been parsed and the required providers data is available. There are multiple places where config is parsed, into either Config or Module structs, so this needs to be implemented in multiple places. --- internal/configs/state_store.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/configs/state_store.go b/internal/configs/state_store.go index aa12509095c3..16da71827daa 100644 --- a/internal/configs/state_store.go +++ b/internal/configs/state_store.go @@ -116,8 +116,9 @@ func resolveStateStoreProviderType(requiredProviders map[string]*RequiredProvide diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Missing entry in required_providers", - Detail: fmt.Sprintf("The provider used for state storage must have a matching entry in required_providers. Please add an entry for provider %q", - stateStore.Provider.Name), + Detail: fmt.Sprintf("The provider used for state storage must have a matching entry in required_providers. Please add an entry for provider %s (%q)", + stateStore.Provider.Name, + stateStore.ProviderAddr), Subject: &stateStore.DeclRange, }) return tfaddr.Provider{}, diags From a2566699d85dabd2d1f7e7a761594c28042309e0 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 11 Jul 2025 14:05:56 +0100 Subject: [PATCH 02/54] Add test that hits the code path for adding a state store to a new (or implied local) project --- internal/command/meta_backend_test.go | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 9821bfe9903c..868802447a53 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -316,6 +316,75 @@ func TestMetaBackend_configureNewBackend(t *testing.T) { } } +// Newly configured state store +// +// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch +// case for this scenario, and will need to be updated when that init feature is implemented. +func TestMetaBackend_configureNewStateStore(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-new"), td) + defer testChdir(t, td)() + + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true + + // Get the state store's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } + + // Get mock provider factory to be used during init + // + // This imagines a provider called foo that contains + // a pluggable state store implementation called bar. + mock := &testing_provider.MockProvider{ + GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ + Provider: providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "region": {Type: cty.String, Optional: true}, + }, + }, + }, + DataSources: map[string]providers.Schema{}, + ResourceTypes: map[string]providers.Schema{}, + ListResourceTypes: map[string]providers.Schema{}, + StateStores: map[string]providers.Schema{ + "foo_bar": { + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "bar": { + Type: cty.String, + Required: true, + }, + }, + }, + }, + }, + }, + } + factory := func() (providers.Interface, error) { + return mock, nil + } + + // Get the operations backend + _, beDiags := m.Backend(&BackendOpts{ + Init: true, + StateStoreConfig: mod.StateStore, + ProviderFactory: factory, + }) + if !beDiags.HasErrors() { + t.Fatal("expected an error to be returned during partial implementation of PSS") + } + wantErr := "Configuring a state store for the first time is not implemented yet" + if !strings.Contains(beDiags.Err().Error(), wantErr) { + t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) + } + +} + // Newly configured backend with prior local state and no remote state func TestMetaBackend_configureNewBackendWithState(t *testing.T) { // Create a temporary working directory that is empty From 03b734fc8cb20e483d45489f3f17ffda813e1341 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 11 Jul 2025 14:12:43 +0100 Subject: [PATCH 03/54] Add test for use of `-reconfigure` flag; show that it hits the code path for adding a state store for the first time --- internal/command/meta_backend_test.go | 78 +++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 868802447a53..c5bc77cb2542 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -869,6 +869,84 @@ func TestMetaBackend_reconfigureBackendChange(t *testing.T) { } } +// Reconfiguring with an already configured state store. +// This should ignore the existing state_store config, and configure the new +// state store is if this is the first time. +// +// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch +// case for this scenario, and will need to be updated when that init feature is implemented. +func TestMetaBackend_reconfigureStateStoreChange(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-reconfigure"), td) + defer testChdir(t, td)() + + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true + + // this should not ask for input + m.input = false + + // cli flag -reconfigure + m.reconfigure = true + + // Get the state store's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } + + // Get mock provider factory to be used during init + // + // This imagines a provider called foo that contains + // a pluggable state store implementation called bar. + mock := &testing_provider.MockProvider{ + GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ + Provider: providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "region": {Type: cty.String, Optional: true}, + }, + }, + }, + DataSources: map[string]providers.Schema{}, + ResourceTypes: map[string]providers.Schema{}, + ListResourceTypes: map[string]providers.Schema{}, + StateStores: map[string]providers.Schema{ + "foo_bar": { + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "bar": { + Type: cty.String, + Required: true, + }, + }, + }, + }, + }, + }, + } + factory := func() (providers.Interface, error) { + return mock, nil + } + + // Get the operations backend + _, beDiags := m.Backend(&BackendOpts{ + Init: true, + StateStoreConfig: mod.StateStore, + ProviderFactory: factory, + }) + + if !beDiags.HasErrors() { + t.Fatal("expected an error to be returned during partial implementation of PSS") + } + wantErr := "Configuring a state store for the first time is not implemented yet" + if !strings.Contains(beDiags.Err().Error(), wantErr) { + t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) + } + +} + // Initializing a backend which supports workspaces and does *not* have // the currently selected workspace should prompt the user with a list of // workspaces to choose from to select a valid one, if more than one workspace From 50807742843959bba4ee25f673597fa720ea47f5 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 11 Jul 2025 15:03:21 +0100 Subject: [PATCH 04/54] Add test that hits the code path for removing use of a state store, migrating to (implied) local backend --- internal/command/meta_backend_test.go | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index c5bc77cb2542..8a1d56c37ad2 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -1643,6 +1643,44 @@ func TestMetaBackend_configuredBackendUnset(t *testing.T) { } } +// Unsetting a saved state store +// +// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch +// case for this scenario, and will need to be updated when that init feature is implemented. +func TestMetaBackend_configuredStateStoreUnset(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-unset"), td) + defer testChdir(t, td)() + + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true + + // Get the state store's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } + + // No mock provider is used here - yet + // Logic will need to be implemented that lets the init have access to + // a factory for the 'old' provider used for PSS previously. This will be + // used when migrating away from PSS entirely, or to a new PSS configuration. + + // Get the operations backend + _, beDiags := m.Backend(&BackendOpts{ + Init: true, + StateStoreConfig: mod.StateStore, + }) + if !beDiags.HasErrors() { + t.Fatal("expected an error to be returned during partial implementation of PSS") + } + wantErr := "Unsetting a state store is not implemented yet" + if !strings.Contains(beDiags.Err().Error(), wantErr) { + t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) + } +} + // Unsetting a saved backend and copying the remote state func TestMetaBackend_configuredBackendUnsetCopy(t *testing.T) { // Create a temporary working directory that is empty From 9ce9ef75083ff6e3fc4dda0b2c89924074b0469d Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 11 Jul 2025 15:04:37 +0100 Subject: [PATCH 05/54] Add test that hits the code path for changing a state store's configuration --- internal/command/meta_backend_test.go | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 8a1d56c37ad2..8bc566f616b9 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -817,6 +817,75 @@ func TestMetaBackend_changeConfiguredBackend(t *testing.T) { } } +// Changing a configured state store +// +// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch +// case for this scenario, and will need to be updated when that init feature is implemented. +// ALSO, this test will need to be split into multiple scenarios in future. +func TestMetaBackend_changeConfiguredStateStore(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-changed"), td) + defer testChdir(t, td)() + + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true + + // Get the state store's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } + + // Get mock provider factory to be used during init + // + // This imagines a provider called foo that contains + // a pluggable state store implementation called bar. + mock := &testing_provider.MockProvider{ + GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ + Provider: providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "region": {Type: cty.String, Optional: true}, + }, + }, + }, + DataSources: map[string]providers.Schema{}, + ResourceTypes: map[string]providers.Schema{}, + ListResourceTypes: map[string]providers.Schema{}, + StateStores: map[string]providers.Schema{ + "foo_bar": { + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "bar": { + Type: cty.String, + Required: true, + }, + }, + }, + }, + }, + }, + } + factory := func() (providers.Interface, error) { + return mock, nil + } + + // Get the operations backend + _, beDiags := m.Backend(&BackendOpts{ + Init: true, + StateStoreConfig: mod.StateStore, + ProviderFactory: factory, + }) + if !beDiags.HasErrors() { + t.Fatal("expected an error to be returned during partial implementation of PSS") + } + wantErr := "Changing a state store configuration is not implemented yet" + if !strings.Contains(beDiags.Err().Error(), wantErr) { + t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) + } +} + // Reconfiguring with an already configured backend. // This should ignore the existing backend config, and configure the new // backend is if this is the first time. From 751fcbb359b02bea6f1121e081b812d2d846f89b Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 11 Jul 2025 15:10:23 +0100 Subject: [PATCH 06/54] Update existing test names to be backend-specific --- internal/command/meta_backend_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 8bc566f616b9..4e0f9ebcdf55 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -699,7 +699,7 @@ func TestMetaBackend_configureNewBackendWithStateExistingNoMigrate(t *testing.T) } // Saved backend state matching config -func TestMetaBackend_configuredUnchanged(t *testing.T) { +func TestMetaBackend_configuredBackendUnchanged(t *testing.T) { td := t.TempDir() testCopyDir(t, testFixturePath("backend-unchanged"), td) t.Chdir(td) From 7704d6aa054d851ab3b4811058e5103af936055c Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 11 Jul 2025 15:31:22 +0100 Subject: [PATCH 07/54] Add tests that hits the code path for migrating between PSS and backends --- internal/command/meta_backend_test.go | 98 +++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 4e0f9ebcdf55..e8f86f78345d 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -886,6 +886,104 @@ func TestMetaBackend_changeConfiguredStateStore(t *testing.T) { } } +func TestMetaBackend_configuredBackendToStateStore(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath("backend-to-state-store"), td) + defer testChdir(t, td)() + + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true + + // Get the state store's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } + + // Get mock provider factory to be used during init + // + // This imagines a provider called foo that contains + // a pluggable state store implementation called bar. + mock := &testing_provider.MockProvider{ + GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ + Provider: providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "region": {Type: cty.String, Optional: true}, + }, + }, + }, + DataSources: map[string]providers.Schema{}, + ResourceTypes: map[string]providers.Schema{}, + ListResourceTypes: map[string]providers.Schema{}, + StateStores: map[string]providers.Schema{ + "foo_bar": { + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "bar": { + Type: cty.String, + Required: true, + }, + }, + }, + }, + }, + }, + } + factory := func() (providers.Interface, error) { + return mock, nil + } + + // Get the operations backend + _, beDiags := m.Backend(&BackendOpts{ + Init: true, + StateStoreConfig: mod.StateStore, + ProviderFactory: factory, + }) + if !beDiags.HasErrors() { + t.Fatal("expected an error to be returned during partial implementation of PSS") + } + wantErr := "Migration from backend to state store is not implemented yet" + if !strings.Contains(beDiags.Err().Error(), wantErr) { + t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) + } +} + +func TestMetaBackend_configuredStateStoreToBackend(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-to-backend"), td) + defer testChdir(t, td)() + + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true + + // Get the backend's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } + + // No mock provider is used here - yet + // Logic will need to be implemented that lets the init have access to + // a factory for the 'old' provider used for PSS previously. This will be + // used when migrating away from PSS entirely, or to a new PSS configuration. + + // Get the operations backend + _, beDiags := m.Backend(&BackendOpts{ + Init: true, + BackendConfig: mod.Backend, + }) + if !beDiags.HasErrors() { + t.Fatal("expected an error to be returned during partial implementation of PSS") + } + wantErr := "Migration from state store to backend is not implemented yet" + if !strings.Contains(beDiags.Err().Error(), wantErr) { + t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) + } +} + // Reconfiguring with an already configured backend. // This should ignore the existing backend config, and configure the new // backend is if this is the first time. From 552b9b93fb4f87c7296a6b608e7787ed5809eead Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 11 Jul 2025 16:55:43 +0100 Subject: [PATCH 08/54] Consolidate PSS-related tests at end of the file --- internal/command/meta_backend_test.go | 352 -------------------------- 1 file changed, 352 deletions(-) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index e8f86f78345d..f9ff25184347 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -316,75 +316,6 @@ func TestMetaBackend_configureNewBackend(t *testing.T) { } } -// Newly configured state store -// -// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch -// case for this scenario, and will need to be updated when that init feature is implemented. -func TestMetaBackend_configureNewStateStore(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-new"), td) - defer testChdir(t, td)() - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true - - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } - - // Get mock provider factory to be used during init - // - // This imagines a provider called foo that contains - // a pluggable state store implementation called bar. - mock := &testing_provider.MockProvider{ - GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ - Provider: providers.Schema{ - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "region": {Type: cty.String, Optional: true}, - }, - }, - }, - DataSources: map[string]providers.Schema{}, - ResourceTypes: map[string]providers.Schema{}, - ListResourceTypes: map[string]providers.Schema{}, - StateStores: map[string]providers.Schema{ - "foo_bar": { - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "bar": { - Type: cty.String, - Required: true, - }, - }, - }, - }, - }, - }, - } - factory := func() (providers.Interface, error) { - return mock, nil - } - - // Get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - StateStoreConfig: mod.StateStore, - ProviderFactory: factory, - }) - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") - } - wantErr := "Configuring a state store for the first time is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) - } - -} - // Newly configured backend with prior local state and no remote state func TestMetaBackend_configureNewBackendWithState(t *testing.T) { // Create a temporary working directory that is empty @@ -817,173 +748,6 @@ func TestMetaBackend_changeConfiguredBackend(t *testing.T) { } } -// Changing a configured state store -// -// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch -// case for this scenario, and will need to be updated when that init feature is implemented. -// ALSO, this test will need to be split into multiple scenarios in future. -func TestMetaBackend_changeConfiguredStateStore(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-changed"), td) - defer testChdir(t, td)() - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true - - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } - - // Get mock provider factory to be used during init - // - // This imagines a provider called foo that contains - // a pluggable state store implementation called bar. - mock := &testing_provider.MockProvider{ - GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ - Provider: providers.Schema{ - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "region": {Type: cty.String, Optional: true}, - }, - }, - }, - DataSources: map[string]providers.Schema{}, - ResourceTypes: map[string]providers.Schema{}, - ListResourceTypes: map[string]providers.Schema{}, - StateStores: map[string]providers.Schema{ - "foo_bar": { - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "bar": { - Type: cty.String, - Required: true, - }, - }, - }, - }, - }, - }, - } - factory := func() (providers.Interface, error) { - return mock, nil - } - - // Get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - StateStoreConfig: mod.StateStore, - ProviderFactory: factory, - }) - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") - } - wantErr := "Changing a state store configuration is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) - } -} - -func TestMetaBackend_configuredBackendToStateStore(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("backend-to-state-store"), td) - defer testChdir(t, td)() - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true - - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } - - // Get mock provider factory to be used during init - // - // This imagines a provider called foo that contains - // a pluggable state store implementation called bar. - mock := &testing_provider.MockProvider{ - GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ - Provider: providers.Schema{ - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "region": {Type: cty.String, Optional: true}, - }, - }, - }, - DataSources: map[string]providers.Schema{}, - ResourceTypes: map[string]providers.Schema{}, - ListResourceTypes: map[string]providers.Schema{}, - StateStores: map[string]providers.Schema{ - "foo_bar": { - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "bar": { - Type: cty.String, - Required: true, - }, - }, - }, - }, - }, - }, - } - factory := func() (providers.Interface, error) { - return mock, nil - } - - // Get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - StateStoreConfig: mod.StateStore, - ProviderFactory: factory, - }) - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") - } - wantErr := "Migration from backend to state store is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) - } -} - -func TestMetaBackend_configuredStateStoreToBackend(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-to-backend"), td) - defer testChdir(t, td)() - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true - - // Get the backend's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } - - // No mock provider is used here - yet - // Logic will need to be implemented that lets the init have access to - // a factory for the 'old' provider used for PSS previously. This will be - // used when migrating away from PSS entirely, or to a new PSS configuration. - - // Get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - BackendConfig: mod.Backend, - }) - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") - } - wantErr := "Migration from state store to backend is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) - } -} - // Reconfiguring with an already configured backend. // This should ignore the existing backend config, and configure the new // backend is if this is the first time. @@ -1036,84 +800,6 @@ func TestMetaBackend_reconfigureBackendChange(t *testing.T) { } } -// Reconfiguring with an already configured state store. -// This should ignore the existing state_store config, and configure the new -// state store is if this is the first time. -// -// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch -// case for this scenario, and will need to be updated when that init feature is implemented. -func TestMetaBackend_reconfigureStateStoreChange(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-reconfigure"), td) - defer testChdir(t, td)() - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true - - // this should not ask for input - m.input = false - - // cli flag -reconfigure - m.reconfigure = true - - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } - - // Get mock provider factory to be used during init - // - // This imagines a provider called foo that contains - // a pluggable state store implementation called bar. - mock := &testing_provider.MockProvider{ - GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ - Provider: providers.Schema{ - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "region": {Type: cty.String, Optional: true}, - }, - }, - }, - DataSources: map[string]providers.Schema{}, - ResourceTypes: map[string]providers.Schema{}, - ListResourceTypes: map[string]providers.Schema{}, - StateStores: map[string]providers.Schema{ - "foo_bar": { - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "bar": { - Type: cty.String, - Required: true, - }, - }, - }, - }, - }, - }, - } - factory := func() (providers.Interface, error) { - return mock, nil - } - - // Get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - StateStoreConfig: mod.StateStore, - ProviderFactory: factory, - }) - - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") - } - wantErr := "Configuring a state store for the first time is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) - } - -} - // Initializing a backend which supports workspaces and does *not* have // the currently selected workspace should prompt the user with a list of // workspaces to choose from to select a valid one, if more than one workspace @@ -1810,44 +1496,6 @@ func TestMetaBackend_configuredBackendUnset(t *testing.T) { } } -// Unsetting a saved state store -// -// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch -// case for this scenario, and will need to be updated when that init feature is implemented. -func TestMetaBackend_configuredStateStoreUnset(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-unset"), td) - defer testChdir(t, td)() - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true - - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } - - // No mock provider is used here - yet - // Logic will need to be implemented that lets the init have access to - // a factory for the 'old' provider used for PSS previously. This will be - // used when migrating away from PSS entirely, or to a new PSS configuration. - - // Get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - StateStoreConfig: mod.StateStore, - }) - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") - } - wantErr := "Unsetting a state store is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) - } -} - // Unsetting a saved backend and copying the remote state func TestMetaBackend_configuredBackendUnsetCopy(t *testing.T) { // Create a temporary working directory that is empty From f1694870307e969daf80d1e458a3735e4238994e Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 14 Jul 2025 15:25:56 +0100 Subject: [PATCH 09/54] Make init command output report if a backend or state store is being initialized --- internal/command/views/init.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/command/views/init.go b/internal/command/views/init.go index 35d79e1ba815..c97d2f7ffcf5 100644 --- a/internal/command/views/init.go +++ b/internal/command/views/init.go @@ -182,6 +182,10 @@ var MessageRegistry map[InitMessageCode]InitMessage = map[InitMessageCode]InitMe HumanValue: "\n[reset][bold]Initializing the backend...", JSONValue: "Initializing the backend...", }, + "initializing_state_store_message": { + HumanValue: "\n[reset][bold]Initializing the state store...", + JSONValue: "Initializing the state store...", + }, "initializing_provider_plugin_message": { HumanValue: "\n[reset][bold]Initializing provider plugins...", JSONValue: "Initializing provider plugins...", @@ -194,10 +198,6 @@ var MessageRegistry map[InitMessageCode]InitMessage = map[InitMessageCode]InitMe HumanValue: "\n[reset][bold]Initializing provider plugins found in the state...", JSONValue: "Initializing provider plugins found in the state...", }, - "initializing_state_store_message": { - HumanValue: "\n[reset][bold]Initializing the state store...", - JSONValue: "Initializing the state store...", - }, "dependencies_lock_changes_info": { HumanValue: dependenciesLockChangesInfo, JSONValue: dependenciesLockChangesInfo, From 62f3192cb21870f1c04268fdcca602eedd720aaa Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 14 Jul 2025 15:39:04 +0100 Subject: [PATCH 10/54] Only process -backend-config flags for use with backend if the user has passed some through --- internal/command/init.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/command/init.go b/internal/command/init.go index 984c6be63f87..83bdccf577a1 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -286,15 +286,19 @@ func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, ext backendSchema := b.ConfigSchema() backendConfig := root.Backend - backendConfigOverride, overrideDiags := c.backendConfigOverrideBody(extraConfig, backendSchema) - diags = diags.Append(overrideDiags) - if overrideDiags.HasErrors() { - return nil, true, diags + var configOverride hcl.Body + if len(*extraConfig.Items) > 0 { + var overrideDiags tfdiags.Diagnostics + configOverride, overrideDiags = c.backendConfigOverrideBody(extraConfig, backendSchema) + diags = diags.Append(overrideDiags) + if overrideDiags.HasErrors() { + return nil, true, diags + } } opts = &BackendOpts{ BackendConfig: backendConfig, - ConfigOverride: backendConfigOverride, + ConfigOverride: configOverride, Init: true, ViewType: viewType, } From 76352816d8230b8c7f9a0809e324092427fbe411 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 14 Jul 2025 16:08:11 +0100 Subject: [PATCH 11/54] Implement initialising an uninitialised dir with PSS --- internal/backend/pluggable/pluggable.go | 9 + internal/command/init.go | 1 + internal/command/init_run_experiment.go | 3 + internal/command/meta_backend.go | 436 +++++++++++++++++- internal/getproviders/providerreqs/version.go | 6 + 5 files changed, 450 insertions(+), 5 deletions(-) diff --git a/internal/backend/pluggable/pluggable.go b/internal/backend/pluggable/pluggable.go index 23a412c6fd55..279d963becf9 100644 --- a/internal/backend/pluggable/pluggable.go +++ b/internal/backend/pluggable/pluggable.go @@ -68,6 +68,15 @@ func (p *Pluggable) ConfigSchema() *configschema.Block { return val.Body } +// ProviderSchema returns the schema for the provider implementing the state store. +// +// This isn't part of the backend.Backend interface but is needed in calling code. +// When it's used the backend.Backend will need to be cast to a Pluggable. +func (p *Pluggable) ProviderSchema() *configschema.Block { + schemaResp := p.provider.GetProviderSchema() + return schemaResp.Provider.Body +} + // PrepareConfig validates configuration for the state store in // the state storage provider. The configuration sent from Terraform core // will not include any values from environment variables; it is the diff --git a/internal/command/init.go b/internal/command/init.go index 83bdccf577a1..79775f7f4633 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -247,6 +247,7 @@ func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, ext opts = &BackendOpts{ StateStoreConfig: root.StateStore, + Locks: locks, ProviderFactory: factory, ConfigOverride: configOverride, Init: true, diff --git a/internal/command/init_run_experiment.go b/internal/command/init_run_experiment.go index f546f1444f5b..59b44e1ec3aa 100644 --- a/internal/command/init_run_experiment.go +++ b/internal/command/init_run_experiment.go @@ -205,6 +205,9 @@ func (c *InitCommand) runPssInit(initArgs *arguments.Init, view views.Init) int case initArgs.Cloud && rootModEarly.CloudConfig != nil: back, backendOutput, backDiags = c.initCloud(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, view) case initArgs.Backend: + // This handles case when config contains either backend or state_store blocks. + // This is valid as either can be implementations of backend.Backend, which is what we + // obtain here. back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, configLocks, view) default: // load the previously-stored backend config diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 059ef2993771..2782d9c88d27 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -19,14 +19,17 @@ import ( "strconv" "strings" + version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hcldec" "github.com/zclconf/go-cty/cty" + "github.com/hashicorp/terraform/internal/addrs" "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/backend/backendrun" backendInit "github.com/hashicorp/terraform/internal/backend/init" backendLocal "github.com/hashicorp/terraform/internal/backend/local" + backendPluggable "github.com/hashicorp/terraform/internal/backend/pluggable" "github.com/hashicorp/terraform/internal/cloud" "github.com/hashicorp/terraform/internal/command/arguments" "github.com/hashicorp/terraform/internal/command/clistate" @@ -35,8 +38,10 @@ import ( "github.com/hashicorp/terraform/internal/configs" "github.com/hashicorp/terraform/internal/depsfile" "github.com/hashicorp/terraform/internal/didyoumean" + "github.com/hashicorp/terraform/internal/getproviders/providerreqs" "github.com/hashicorp/terraform/internal/plans" "github.com/hashicorp/terraform/internal/providers" + "github.com/hashicorp/terraform/internal/states" "github.com/hashicorp/terraform/internal/states/statemgr" "github.com/hashicorp/terraform/internal/terraform" "github.com/hashicorp/terraform/internal/tfdiags" @@ -59,6 +64,13 @@ type BackendOpts struct { // This will only be set if the configuration contains a state_store block. ProviderFactory providers.Factory + // Locks allows state-migration logic to detect when the provider used for pluggable state storage + // during the last init (i.e. what's in the backend state file) is mismatched with the provider + // version in use currently. + // + // This will only be set if the configuration contains a state_store block. + Locks *depsfile.Locks + // ConfigOverride is an hcl.Body that, if non-nil, will be used with // configs.MergeBodies to override the type-specific backend configuration // arguments in Config. @@ -788,11 +800,22 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di stateStoreConfig.Provider.Name, stateStoreConfig.ProviderAddr, ) - return nil, diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Not implemented yet", - Detail: "Configuring a state store for the first time is not implemented yet", - }) + + if !opts.Init { + initReason := fmt.Sprintf("Initial configuration of the requested state_store %q in provider %s (%q)", + stateStoreConfig.Type, + stateStoreConfig.Provider.Name, + stateStoreConfig.ProviderAddr, + ) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "State store initialization required, please run \"terraform init\"", + fmt.Sprintf(strings.TrimSpace(errStateStoreInit), initReason), + )) + return nil, diags + } + + return m.stateStore_C_s(stateStoreConfig, cHash, sMgr, opts) // Migration from state store to backend case backendConfig != nil && s.Backend.Empty() && @@ -1452,6 +1475,211 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clista return b, diags } +//------------------------------------------------------------------- +// State Store Config Scenarios +// The functions below cover handling all the various scenarios that +// can exist when loading a state store. They are named in the format of +// "stateStore_C_S" where C and S may be upper or lowercase. Lowercase +// means it is false, uppercase means it is true. +// +// The fields are: +// +// * C - State store configuration is set and changed in TF files +// * S - State store configuration is set in the state +// +//------------------------------------------------------------------- + +// Configuring a state_store for the first time. +func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + + vt := arguments.ViewJSON + // Set default viewtype if none was set as the StateLocker needs to know exactly + // what viewType we want to have. + if opts == nil || opts.ViewType != vt { + vt = arguments.ViewHuman + } + + // Grab a purely local backend to get the local state if it exists + localB, localBDiags := m.Backend(&BackendOpts{ForceLocal: true, Init: true}) + if localBDiags.HasErrors() { + diags = diags.Append(localBDiags) + return nil, diags + } + + workspaces, err := localB.Workspaces() + if err != nil { + diags = diags.Append(fmt.Errorf(errBackendLocalRead, err)) + return nil, diags + } + + var localStates []statemgr.Full + for _, workspace := range workspaces { + localState, err := localB.StateMgr(workspace) + if err != nil { + diags = diags.Append(fmt.Errorf(errBackendLocalRead, err)) + return nil, diags + } + if err := localState.RefreshState(); err != nil { + diags = diags.Append(fmt.Errorf(errBackendLocalRead, err)) + return nil, diags + } + + // We only care about non-empty states. + if localS := localState.State(); !localS.Empty() { + log.Printf("[TRACE] Meta.Backend: will need to migrate workspace states because of existing %q workspace", workspace) + localStates = append(localStates, localState) + } else { + log.Printf("[TRACE] Meta.Backend: ignoring local %q workspace because its state is empty", workspace) + } + } + + // Get the state store as an instance of backend.Backend + b, storeConfigVal, providerConfigVal, moreDiags := m.stateStoreInitFromConfig(c, opts) + diags = diags.Append(moreDiags) + if diags.HasErrors() { + return nil, diags + } + + if len(localStates) > 0 { + // Migrate any local states into the new state store + err = m.backendMigrateState(&backendMigrateOpts{ + SourceType: "local", + DestinationType: c.Type, + Source: localB, + Destination: b, + ViewType: vt, + }) + if err != nil { + diags = diags.Append(err) + return nil, diags + } + + // We remove the local state after migration to prevent confusion + // As we're migrating to a state store we don't have insight into whether it stores + // files locally at all, and whether those local files conflict with the location of + // the old local state. + log.Printf("[TRACE] Meta.Backend: removing old state snapshots from old backend") + for _, localState := range localStates { + // We always delete the local state, unless that was our new state too. + if err := localState.WriteState(nil); err != nil { + diags = diags.Append(fmt.Errorf(errBackendMigrateLocalDelete, err)) + return nil, diags + } + if err := localState.PersistState(nil); err != nil { + diags = diags.Append(fmt.Errorf(errBackendMigrateLocalDelete, err)) + return nil, diags + } + } + } + + if m.stateLock { + view := views.NewStateLocker(vt, m.View) + stateLocker := clistate.NewLocker(m.stateLockTimeout, view) + if err := stateLocker.Lock(sMgr, "state_store from plan"); err != nil { + diags = diags.Append(fmt.Errorf("Error locking state: %s", err)) + return nil, diags + } + defer stateLocker.Unlock() + } + + // Store the state_store metadata in our saved state location + s := sMgr.State() + if s == nil { + s = workdir.NewBackendStateFile() + } + + var pVersion *version.Version + if c.ProviderAddr.Equals(addrs.NewBuiltInProvider("terraform")) { + // If we're handling the builtin "terraform" provider then there's no version information to store in the dependency lock file, so don't access it. + // We must record a value into the backend state file, and we cannot include a value that changes (e.g. the Terraform core binary version) as migration + // is impossible with builtin providers. + // So, we use a hardcoded version number of 42. + pVersion, err = version.NewVersion("0.42.0") + if err != nil { + diags = diags.Append(fmt.Errorf("Error when creating a backend state file containing a builtin provider. This is a bug in Terraform and should be reported: %w", + err)) + return nil, diags + } + } else { + pLock := opts.Locks.Provider(c.ProviderAddr) + if pLock == nil { + diags = diags.Append(fmt.Errorf("The provider %s (%q) is not present in the lockfile, despite being used for state store %q. This is a bug in Terraform and should be reported: %w", + c.Provider.Name, + c.ProviderAddr, + c.Type, + err)) + return nil, diags + } + var err error + pVersion, err = providerreqs.GoVersionFromVersion(pLock.Version()) + if err != nil { + diags = diags.Append(fmt.Errorf("Failed obtain the in-use version of provider %s (%q) when recording backend state for state store %q. This is a bug in Terraform and should be reported: %w", + c.Provider.Name, + c.ProviderAddr, + c.Type, + err)) + return nil, diags + } + } + s.StateStore = &workdir.StateStoreConfigState{ + Type: c.Type, + Hash: uint64(cHash), + Provider: &workdir.ProviderConfigState{ + Source: &c.ProviderAddr, + Version: pVersion, + }, + } + s.StateStore.SetConfig(storeConfigVal, b.ConfigSchema()) + if plug, ok := b.(*backendPluggable.Pluggable); ok { + // We need to convert away from backend.Backend interface to use the method + // for accessing the provider schema. + s.StateStore.Provider.SetConfig(providerConfigVal, plug.ProviderSchema()) + } + + // Verify that selected workspace exists in the state store. + // TODO (SarahFrench/radeksimko) - TF core should be responsible for creating the new workspace. + // > Is this the correct place to do so? + // > Should we prompt the user to approve creating a new workspace? + if opts.Init && b != nil { + err := m.selectWorkspace(b) + if strings.Contains(err.Error(), "No existing workspaces") { + // Make the default workspace. All other workspaces are user-created via the workspace commands. + bStateMgr, err := b.StateMgr(backend.DefaultStateName) + if err != nil { + diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug in Terraform and should be reported: %w", + c.Type, + c.Provider.Name, + c.ProviderAddr, + err)) + return nil, diags + } + emptyState := states.NewState() + if err := bStateMgr.WriteState(emptyState); err != nil { + diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) + return nil, diags + } + if err := sMgr.PersistState(); err != nil { + diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) + return nil, diags + } + } else if err != nil { + diags = diags.Append(err) + } + } + + if err := sMgr.WriteState(s); err != nil { + diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) + return nil, diags + } + if err := sMgr.PersistState(); err != nil { + diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) + return nil, diags + } + + return b, diags +} + // Initializing a saved backend from the cache file (legacy state file) // // TODO: This is extremely similar to Meta.backendFromState() but for legacy reasons this is the @@ -1649,6 +1877,181 @@ func (m *Meta) backendInitFromConfig(c *configs.Backend) (backend.Backend, cty.V return b, configVal, diags } +// stateStoreInitFromConfig returns an initialized and configured state store, using the backend.Backend interface. +// During this process: +// > Users are prompted for input if required attributes are missing. +// > The provider is configured, after validating provider config +// > The state store is configured, after validating state_store config +func (m *Meta) stateStoreInitFromConfig(c *configs.StateStore, opts *BackendOpts) (backend.Backend, cty.Value, cty.Value, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + + provider, err := opts.ProviderFactory() + if err != nil { + diags = diags.Append(fmt.Errorf("error when obtaining provider instance during state store initialization: %w", err)) + return nil, cty.NilVal, cty.NilVal, diags + } + // We purposefully don't have a deferred call to the provider's Close method here because the calling code needs a + // running provider instance inside the returned backend.Backend instance. + // Stopping the provider process is the responsibility of the calling code. + + resp := provider.GetProviderSchema() + + if len(resp.StateStores) == 0 { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Provider does not support pluggable state storage", + Detail: fmt.Sprintf("There are no state stores implemented by provider %s (%q)", + c.Provider.Name, + c.ProviderAddr), + Subject: &c.DeclRange, + }) + return nil, cty.NilVal, cty.NilVal, diags + } + + schema, exists := resp.StateStores[c.Type] + if !exists { + suggestions := slices.Sorted(maps.Keys(resp.StateStores)) + suggestion := didyoumean.NameSuggestion(c.Type, suggestions) + if suggestion != "" { + suggestion = fmt.Sprintf(" Did you mean %q?", suggestion) + } + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "State store not implemented by the provider", + Detail: fmt.Sprintf("State store %q is not implemented by provider %s (%q)%s", + c.Type, c.Provider.Name, + c.ProviderAddr, suggestion), + Subject: &c.DeclRange, + }) + return nil, cty.NilVal, cty.NilVal, diags + } + + // Handle the nested provider block. + pDecSpec := resp.Provider.Body.NoneRequired().DecoderSpec() + pConfig := c.Provider.Config + providerConfigVal, pDecDiags := hcldec.Decode(pConfig, pDecSpec, nil) + diags = diags.Append(pDecDiags) + + if !providerConfigVal.IsWhollyKnown() { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Unknown values within state_store's nested provider block", + "The `terraform` configuration block should contain only concrete and static values. Another diagnostic should contain more information about which part of the configuration is problematic.")) + return nil, cty.NilVal, cty.NilVal, diags + } + + if m.Input() { + var err error + // TODO (SarahFrench/radeksimko) - Should we allow input to the provider block? + providerConfigVal, err = m.inputForSchema(providerConfigVal, resp.Provider.Body) + if err != nil { + diags = diags.Append( + fmt.Errorf("Error asking for input to configure provider %s (%q) for state store %q: %s", + c.Provider.Name, + c.ProviderAddr, + c.Type, + err, + ), + ) + return nil, cty.NilVal, cty.NilVal, diags + } + + // We get an unknown here if the if the user aborted input, but we can't + // turn that into a config value, so set it to null and let the provider + // handle it in PrepareConfig. + if !providerConfigVal.IsKnown() { + providerConfigVal = cty.NullVal(providerConfigVal.Type()) + } + } + + // Handle the schema for the state store itself, excluding the provider block. + ssdecSpec := schema.Body.NoneRequired().DecoderSpec() + stateStoreConfigVal, ssDecDiags := hcldec.Decode(c.Config, ssdecSpec, nil) + diags = diags.Append(ssDecDiags) + if ssDecDiags.HasErrors() { + return nil, cty.NilVal, cty.NilVal, diags + } + + if !stateStoreConfigVal.IsWhollyKnown() { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Unknown values within state_store definition", + "The `terraform` configuration block should contain only concrete and static values. Another diagnostic should contain more information about which part of the configuration is problematic.")) + return nil, cty.NilVal, cty.NilVal, diags + } + + if m.Input() { + var err error + // TODO (SarahFrench/radeksimko) + // > Should we do this? + // > Could this accidentally prompt users to supply values in the nested provider block? + stateStoreConfigVal, err = m.inputForSchema(stateStoreConfigVal, schema.Body) + if err != nil { + diags = diags.Append(fmt.Errorf("Error asking for input to configure state store %q: %s", c.Type, err)) + return nil, cty.NilVal, cty.NilVal, diags + } + + // We get an unknown here if the if the user aborted input, but we can't + // turn that into a config value, so set it to null and let the provider + // handle it in PrepareConfig. + if !stateStoreConfigVal.IsKnown() { + stateStoreConfigVal = cty.NullVal(stateStoreConfigVal.Type()) + } + } + + // Validate and configure the provider + + // TODO (SarahFrench/radeksimko) : deal with marks + validateResp := provider.ValidateProviderConfig(providers.ValidateProviderConfigRequest{ + Config: providerConfigVal, + }) + diags = diags.Append(validateResp.Diagnostics) + if validateResp.Diagnostics.HasErrors() { + return nil, cty.NilVal, cty.NilVal, diags + } + + configureResp := provider.ConfigureProvider(providers.ConfigureProviderRequest{ + // TODO TerraformVersion: , + Config: validateResp.PreparedConfig, + // TODO ClientCapabilities: , + }) + diags = diags.Append(configureResp.Diagnostics) + if configureResp.Diagnostics.HasErrors() { + return nil, cty.NilVal, cty.NilVal, diags + } + + // Validate Store Config + // TODO (SarahFrench/radeksimko) : deal with marks + validateStoreResp := provider.ValidateStateStoreConfig(providers.ValidateStateStoreConfigRequest{ + TypeName: c.Type, + Config: stateStoreConfigVal, + }) + diags = diags.Append(validateStoreResp.Diagnostics) + if validateStoreResp.Diagnostics.HasErrors() { + return nil, cty.NilVal, cty.NilVal, diags + } + + // Configure State Store + cfgStoreResp := provider.ConfigureStateStore(providers.ConfigureStateStoreRequest{ + TypeName: c.Type, + Config: stateStoreConfigVal, + }) + diags = diags.Append(cfgStoreResp.Diagnostics) + if cfgStoreResp.Diagnostics.HasErrors() { + return nil, cty.NilVal, cty.NilVal, diags + } + + // Now we have a fully configured state store, ready to be used. + // To make it usable we need to return it in a backend.Backend interface. + b, err := backendPluggable.NewPluggable(provider, c.Type) + if err != nil { + diags = diags.Append(err) + return nil, cty.NilVal, cty.NilVal, diags + } + + return b, stateStoreConfigVal, providerConfigVal, diags +} + // Helper method to get aliases from the enhanced backend and alias them // in the Meta service discovery. It's unfortunate that the Meta backend // is modifying the service discovery at this level, but the owner @@ -1881,6 +2284,21 @@ hasn't changed and try again. At this point, no changes to your existing configuration or state have been made. ` +const errStateStoreInit = ` +Reason: %s +The "state store" is the interface that Terraform uses to store state when +performing operations on the local machine. If this message is showing up, +it means that the Terraform configuration you're using is using a custom +configuration for state storage in Terraform. +Changes to state store configurations require reinitialization. This allows +Terraform to set up the new configuration, copy existing state, etc. Please run +"terraform init" with either the "-reconfigure" or "-migrate-state" flags to +use the current configuration. +If the change reason above is incorrect, please verify your configuration +hasn't changed and try again. At this point, no changes to your existing +configuration or state have been made. +` + const errBackendInitCloud = ` Reason: %s. @@ -1901,6 +2319,14 @@ are usually due to simple file permission errors. Please look at the error above, resolve it, and try again. ` +const errStateStoreWorkspaceCreate = ` +Error creating the default workspace using pluggable state store %s: %s + +This could be a bug in the provider used for state storage, or a bug in +Terraform. Please file an issue with the provider developers before reporting +a bug for Terraform. +` + const outputBackendMigrateChange = ` Terraform detected that the backend type changed from %q to %q. ` diff --git a/internal/getproviders/providerreqs/version.go b/internal/getproviders/providerreqs/version.go index 579eceb9ac4c..cc78ccdcbb44 100644 --- a/internal/getproviders/providerreqs/version.go +++ b/internal/getproviders/providerreqs/version.go @@ -20,6 +20,8 @@ import ( "github.com/apparentlymart/go-versions/versions" "github.com/apparentlymart/go-versions/versions/constraints" + "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform/internal/addrs" ) @@ -86,6 +88,10 @@ func ParseVersion(str string) (Version, error) { return versions.ParseVersion(str) } +func GoVersionFromVersion(v Version) (*version.Version, error) { + return version.NewVersion(v.String()) +} + // MustParseVersion is a variant of ParseVersion that panics if it encounters // an error while parsing. func MustParseVersion(str string) Version { From 85c738e7f7e1a573a20a2118652086c011ab3871 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 7 Aug 2025 11:46:48 +0100 Subject: [PATCH 12/54] Fix call to `ConfigureProvider` by avoiding use of returned config value from `ValidateProviderConfig` --- internal/command/meta_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 2782d9c88d27..99a3e370d0d2 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -2012,7 +2012,7 @@ func (m *Meta) stateStoreInitFromConfig(c *configs.StateStore, opts *BackendOpts configureResp := provider.ConfigureProvider(providers.ConfigureProviderRequest{ // TODO TerraformVersion: , - Config: validateResp.PreparedConfig, + Config: providerConfigVal, // TODO ClientCapabilities: , }) diags = diags.Append(configureResp.Diagnostics) From 5d703ce27eb678f42d6495605bf34d5b20c574db Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 7 Aug 2025 11:56:22 +0100 Subject: [PATCH 13/54] Add TF version to `ConfigureProviderRequest` --- internal/command/meta_backend.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 99a3e370d0d2..3c1101b22c4a 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -45,6 +45,7 @@ import ( "github.com/hashicorp/terraform/internal/states/statemgr" "github.com/hashicorp/terraform/internal/terraform" "github.com/hashicorp/terraform/internal/tfdiags" + tfversion "github.com/hashicorp/terraform/version" ) // BackendOpts are the options used to initialize a backendrun.OperationsBackend. @@ -2011,9 +2012,9 @@ func (m *Meta) stateStoreInitFromConfig(c *configs.StateStore, opts *BackendOpts } configureResp := provider.ConfigureProvider(providers.ConfigureProviderRequest{ - // TODO TerraformVersion: , + TerraformVersion: tfversion.String(), Config: providerConfigVal, - // TODO ClientCapabilities: , + // TODO ClientCapabilities? }) diags = diags.Append(configureResp.Diagnostics) if configureResp.Diagnostics.HasErrors() { From 080dba5c2216095198b9e2ec113f659d7e94fb62 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 7 Aug 2025 17:17:42 +0100 Subject: [PATCH 14/54] Update test to assert that the newly-initialised backend is reflected in the backend state file --- internal/command/meta_backend.go | 16 ++++--- internal/command/meta_backend_test.go | 48 ++++++++++++++++--- .../command/testdata/state-store-new/main.tf | 4 +- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 3c1101b22c4a..aaae17f078c2 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1544,7 +1544,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L if len(localStates) > 0 { // Migrate any local states into the new state store - err = m.backendMigrateState(&backendMigrateOpts{ + err := m.backendMigrateState(&backendMigrateOpts{ SourceType: "local", DestinationType: c.Type, Source: localB, @@ -1596,6 +1596,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L // We must record a value into the backend state file, and we cannot include a value that changes (e.g. the Terraform core binary version) as migration // is impossible with builtin providers. // So, we use a hardcoded version number of 42. + var err error pVersion, err = version.NewVersion("0.42.0") if err != nil { diags = diags.Append(fmt.Errorf("Error when creating a backend state file containing a builtin provider. This is a bug in Terraform and should be reported: %w", @@ -1605,11 +1606,11 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L } else { pLock := opts.Locks.Provider(c.ProviderAddr) if pLock == nil { - diags = diags.Append(fmt.Errorf("The provider %s (%q) is not present in the lockfile, despite being used for state store %q. This is a bug in Terraform and should be reported: %w", + diags = diags.Append(fmt.Errorf("The provider %s (%q) is not present in the lockfile, despite being used for state store %q. This is a bug in Terraform and should be reported", c.Provider.Name, c.ProviderAddr, c.Type, - err)) + )) return nil, diags } var err error @@ -1646,13 +1647,14 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L err := m.selectWorkspace(b) if strings.Contains(err.Error(), "No existing workspaces") { // Make the default workspace. All other workspaces are user-created via the workspace commands. - bStateMgr, err := b.StateMgr(backend.DefaultStateName) - if err != nil { - diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug in Terraform and should be reported: %w", + bStateMgr, sDiags := b.StateMgr(backend.DefaultStateName) + diags = diags.Append(sDiags) + if sDiags.HasErrors() { + diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug and should be reported: %w", c.Type, c.Provider.Name, c.ProviderAddr, - err)) + sDiags.Err())) return nil, diags } emptyState := states.NewState() diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index f9ff25184347..af159e78649c 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2103,25 +2103,61 @@ func TestMetaBackend_configureNewStateStore(t *testing.T) { // // This imagines a provider called foo that contains // a pluggable state store implementation called bar. + pssName := "test_store" mock := testStateStoreMock(t) factory := func() (providers.Interface, error) { return mock, nil } - // Get the operations backend + // Create locks - these would normally be the locks derived from config + locks := depsfile.NewLocks() + constraint, err := providerreqs.ParseVersionConstraints(">9.0.0") + if err != nil { + t.Fatalf("test setup failed when making constraint: %s", err) + } + expectedVersionString := "9.9.9" + expectedProviderSource := "registry.terraform.io/hashicorp/test" + locks.SetProvider( + addrs.MustParseProviderSourceString(expectedProviderSource), + versions.MustParseVersion(expectedVersionString), + constraint, + []providerreqs.Hash{"h1:foo"}, + ) + + // Act - get the operations backend _, beDiags := m.Backend(&BackendOpts{ Init: true, StateStoreConfig: mod.StateStore, ProviderFactory: factory, + Locks: locks, }) - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") + if beDiags.HasErrors() { + t.Fatalf("unexpected error: %s", beDiags.Err()) } - wantErr := "Configuring a state store for the first time is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) + + // Check the backend state file exists & assert its contents + s := testDataStateRead(t, filepath.Join(DefaultDataDir, backendLocal.DefaultStateFilename)) + if s == nil { + t.Fatal("expected backend state file to be created, but it was missing") } + if s.StateStore.Type != pssName { + t.Fatalf("backend state file contains unexpected state store type, want %q, got %q", pssName, s.StateStore.Type) + } + if s.StateStore.Provider.Version.String() != expectedVersionString { + t.Fatalf("backend state file contains unexpected version, want %q, got %q", expectedVersionString, s.StateStore.Provider.Version) + } + if s.StateStore.Provider.Source.String() != expectedProviderSource { + t.Fatalf("backend state file contains unexpected source, want %q, got %q", expectedProviderSource, s.StateStore.Provider.Source) + } + expectedProviderConfig := "{ \"region\": \"mars\" }" + expectedStoreConfig := "{ \"value\": \"foobar\" }" + if cleanString(string(s.StateStore.Provider.ConfigRaw)) != expectedProviderConfig { + t.Fatalf("backend state file contains unexpected raw config data for the provider, want %q, got %q", expectedProviderConfig, cleanString(string(s.StateStore.Provider.ConfigRaw))) + } + if cleanString(string(s.StateStore.ConfigRaw)) != expectedStoreConfig { + t.Fatalf("backend state file contains unexpected raw config data for the state store, want %q, got %q", expectedStoreConfig, cleanString(string(s.StateStore.ConfigRaw))) + } } // Unsetting a saved state store diff --git a/internal/command/testdata/state-store-new/main.tf b/internal/command/testdata/state-store-new/main.tf index 37fb2ac835a6..b62d81df6447 100644 --- a/internal/command/testdata/state-store-new/main.tf +++ b/internal/command/testdata/state-store-new/main.tf @@ -5,7 +5,9 @@ terraform { } } state_store "test_store" { - provider "test" {} + provider "test" { + region = "mars" + } value = "foobar" } From d9f351bc96afc753ab64b754b4bc97aed3c9ada1 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 8 Aug 2025 11:28:46 +0100 Subject: [PATCH 15/54] Add godoc comment to `GoVersionFromVersion` --- internal/getproviders/providerreqs/version.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/getproviders/providerreqs/version.go b/internal/getproviders/providerreqs/version.go index cc78ccdcbb44..2fc15fc499de 100644 --- a/internal/getproviders/providerreqs/version.go +++ b/internal/getproviders/providerreqs/version.go @@ -88,6 +88,8 @@ func ParseVersion(str string) (Version, error) { return versions.ParseVersion(str) } +// GoVersionFromVersion converts a Version from the providerreqs package +// into a Version from the hashicorp/go-version module. func GoVersionFromVersion(v Version) (*version.Version, error) { return version.NewVersion(v.String()) } From bce0926b3d8bdc0cb360e8d8a684c85309abac07 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Mon, 4 Aug 2025 15:23:38 +0100 Subject: [PATCH 16/54] command: Prompt user for default workspace creation --- internal/command/meta_backend.go | 65 ++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index aaae17f078c2..63733863ddbf 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1640,32 +1640,57 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L } // Verify that selected workspace exists in the state store. - // TODO (SarahFrench/radeksimko) - TF core should be responsible for creating the new workspace. - // > Is this the correct place to do so? - // > Should we prompt the user to approve creating a new workspace? if opts.Init && b != nil { err := m.selectWorkspace(b) if strings.Contains(err.Error(), "No existing workspaces") { - // Make the default workspace. All other workspaces are user-created via the workspace commands. - bStateMgr, sDiags := b.StateMgr(backend.DefaultStateName) - diags = diags.Append(sDiags) - if sDiags.HasErrors() { - diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug and should be reported: %w", - c.Type, - c.Provider.Name, - c.ProviderAddr, - sDiags.Err())) - return nil, diags - } - emptyState := states.NewState() - if err := bStateMgr.WriteState(emptyState); err != nil { - diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) + ws, err := m.Workspace() + if err != nil { + diags = diags.Append(fmt.Errorf("Failed to check current workspace: %w", err)) return nil, diags } - if err := sMgr.PersistState(); err != nil { - diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) - return nil, diags + + if m.Input() && ws == backend.DefaultStateName { + input := m.UIInput() + desc := fmt.Sprintf("Terraform will create the %q workspace via %q.\n"+ + "Only 'yes' will be accepted to approve.", backend.DefaultStateName, c.Type) + v, err := input.Input(context.Background(), &terraform.InputOpts{ + Id: "approve", + Query: fmt.Sprintf("Workspace %q does not exit, would you like to create one?", backend.DefaultStateName), + Description: desc, + }) + if err != nil { + diags = diags.Append(fmt.Errorf("Failed to confirm default workspace creation: %w", err)) + return nil, diags + } + if v != "yes" { + diags = diags.Append(errors.New("Failed to create default workspace")) + return nil, diags + } + + // TODO: Confirm if defaulting to creation on first use (rather than error) is a good idea + // Make the default workspace. All other workspaces are user-created via the workspace commands. + bStateMgr, sDiags := b.StateMgr(backend.DefaultStateName) + diags = diags.Append(sDiags) + if sDiags.HasErrors() { + diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug in Terraform and should be reported: %w", + c.Type, + c.Provider.Name, + c.ProviderAddr, + sDiags.Err())) + return nil, diags + } + emptyState := states.NewState() + if err := bStateMgr.WriteState(emptyState); err != nil { + diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) + return nil, diags + } + if err := sMgr.PersistState(); err != nil { + diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) + return nil, diags + } } + // TODO: handle if input is not enabled + // TODO: handle if non-default workspace is not used } else if err != nil { diags = diags.Append(err) } From 7b650e030d5f132a25a76699b7735235fe94c89e Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 8 Aug 2025 12:19:11 +0100 Subject: [PATCH 17/54] Rename argument to backend and state store init methods to help distinguish between state managers --- internal/command/meta_backend.go | 58 +++++++++++++++++--------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 63733863ddbf..1f6fc428bd28 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1100,7 +1100,7 @@ func (m *Meta) backendFromState(_ context.Context) (backend.Backend, tfdiags.Dia // Unconfiguring a backend (moving from backend => local). func (m *Meta) backend_c_r_S( - c *configs.Backend, cHash int, sMgr *clistate.LocalState, output bool, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { + c *configs.Backend, cHash int, backendSMgr *clistate.LocalState, output bool, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics @@ -1111,7 +1111,8 @@ func (m *Meta) backend_c_r_S( vt = arguments.ViewHuman } - s := sMgr.State() + // Get backend state file data + s := backendSMgr.State() cloudMode := cloud.DetectConfigChangeType(s.Backend, c, false) diags = diags.Append(m.assertSupportedCloudInitOptions(cloudMode)) @@ -1136,7 +1137,7 @@ func (m *Meta) backend_c_r_S( } // Initialize the configured backend - b, moreDiags := m.savedBackend(sMgr) + b, moreDiags := m.savedBackend(backendSMgr) diags = diags.Append(moreDiags) if moreDiags.HasErrors() { return nil, diags @@ -1157,11 +1158,11 @@ func (m *Meta) backend_c_r_S( // Remove the stored metadata s.Backend = nil - if err := sMgr.WriteState(s); err != nil { + if err := backendSMgr.WriteState(s); err != nil { diags = diags.Append(fmt.Errorf(strings.TrimSpace(errBackendClearSaved), err)) return nil, diags } - if err := sMgr.PersistState(); err != nil { + if err := backendSMgr.PersistState(); err != nil { diags = diags.Append(fmt.Errorf(strings.TrimSpace(errBackendClearSaved), err)) return nil, diags } @@ -1177,7 +1178,7 @@ func (m *Meta) backend_c_r_S( } // Configuring a backend for the first time. -func (m *Meta) backend_C_r_s(c *configs.Backend, cHash int, sMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) backend_C_r_s(c *configs.Backend, cHash int, backendSMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics vt := arguments.ViewJSON @@ -1282,15 +1283,15 @@ func (m *Meta) backend_C_r_s(c *configs.Backend, cHash int, sMgr *clistate.Local if m.stateLock { view := views.NewStateLocker(vt, m.View) stateLocker := clistate.NewLocker(m.stateLockTimeout, view) - if err := stateLocker.Lock(sMgr, "backend from plan"); err != nil { + if err := stateLocker.Lock(backendSMgr, "backend from plan"); err != nil { diags = diags.Append(fmt.Errorf("Error locking state: %s", err)) return nil, diags } defer stateLocker.Unlock() } - // Store the metadata in our saved state location - s := sMgr.State() + // Store the backend's metadata in our the backend state file location + s := backendSMgr.State() if s == nil { s = workdir.NewBackendStateFile() } @@ -1325,11 +1326,12 @@ func (m *Meta) backend_C_r_s(c *configs.Backend, cHash int, sMgr *clistate.Local } } - if err := sMgr.WriteState(s); err != nil { + // Update backend state file + if err := backendSMgr.WriteState(s); err != nil { diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) return nil, diags } - if err := sMgr.PersistState(); err != nil { + if err := backendSMgr.PersistState(); err != nil { diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) return nil, diags } @@ -1345,7 +1347,7 @@ func (m *Meta) backend_C_r_s(c *configs.Backend, cHash int, sMgr *clistate.Local } // Changing a previously saved backend. -func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clistate.LocalState, output bool, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, backendSMgr *clistate.LocalState, output bool, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics vt := arguments.ViewJSON @@ -1355,8 +1357,8 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clista vt = arguments.ViewHuman } - // Get the old state - s := sMgr.State() + // Get the old backend state file data + s := backendSMgr.State() cloudMode := cloud.DetectConfigChangeType(s.Backend, c, false) diags = diags.Append(m.assertSupportedCloudInitOptions(cloudMode)) @@ -1402,7 +1404,7 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clista // state lives. if cloudMode != cloud.ConfigChangeInPlace { // Grab the existing backend - oldB, oldBDiags := m.savedBackend(sMgr) + oldB, oldBDiags := m.savedBackend(backendSMgr) diags = diags.Append(oldBDiags) if oldBDiags.HasErrors() { return nil, diags @@ -1424,7 +1426,7 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clista if m.stateLock { view := views.NewStateLocker(vt, m.View) stateLocker := clistate.NewLocker(m.stateLockTimeout, view) - if err := stateLocker.Lock(sMgr, "backend from plan"); err != nil { + if err := stateLocker.Lock(backendSMgr, "backend from plan"); err != nil { diags = diags.Append(fmt.Errorf("Error locking state: %s", err)) return nil, diags } @@ -1433,7 +1435,7 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clista } // Update the backend state - s = sMgr.State() + s = backendSMgr.State() if s == nil { s = workdir.NewBackendStateFile() } @@ -1455,11 +1457,12 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clista } } - if err := sMgr.WriteState(s); err != nil { + // Save data to backend state file + if err := backendSMgr.WriteState(s); err != nil { diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) return nil, diags } - if err := sMgr.PersistState(); err != nil { + if err := backendSMgr.PersistState(); err != nil { diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) return nil, diags } @@ -1491,7 +1494,7 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, sMgr *clista //------------------------------------------------------------------- // Configuring a state_store for the first time. -func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics vt := arguments.ViewJSON @@ -1577,7 +1580,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L if m.stateLock { view := views.NewStateLocker(vt, m.View) stateLocker := clistate.NewLocker(m.stateLockTimeout, view) - if err := stateLocker.Lock(sMgr, "state_store from plan"); err != nil { + if err := stateLocker.Lock(backendSMgr, "state_store from plan"); err != nil { diags = diags.Append(fmt.Errorf("Error locking state: %s", err)) return nil, diags } @@ -1585,7 +1588,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L } // Store the state_store metadata in our saved state location - s := sMgr.State() + s := backendSMgr.State() if s == nil { s = workdir.NewBackendStateFile() } @@ -1684,7 +1687,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) return nil, diags } - if err := sMgr.PersistState(); err != nil { + if err := backendSMgr.PersistState(); err != nil { diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) return nil, diags } @@ -1696,11 +1699,12 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L } } - if err := sMgr.WriteState(s); err != nil { + // Update backend state file + if err := backendSMgr.WriteState(s); err != nil { diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) return nil, diags } - if err := sMgr.PersistState(); err != nil { + if err := backendSMgr.PersistState(); err != nil { diags = diags.Append(fmt.Errorf(errBackendWriteSaved, err)) return nil, diags } @@ -1713,10 +1717,10 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, sMgr *clistate.L // TODO: This is extremely similar to Meta.backendFromState() but for legacy reasons this is the // function used by the migration APIs within this file. The other handles 'init -backend=false', // specifically. -func (m *Meta) savedBackend(sMgr *clistate.LocalState) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) savedBackend(backendSMgr *clistate.LocalState) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics - s := sMgr.State() + s := backendSMgr.State() // Get the backend f := backendInit.Backend(s.Backend.Type) From a9c3ff7e49839ec35d213bd5772809c1d2f33998 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 8 Aug 2025 12:20:32 +0100 Subject: [PATCH 18/54] Fix: avoid updating backend state file if an unhandled error has occurred --- internal/command/meta_backend.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 1f6fc428bd28..5b0c9b714f64 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1693,10 +1693,9 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli } } // TODO: handle if input is not enabled - // TODO: handle if non-default workspace is not used - } else if err != nil { - diags = diags.Append(err) - } + } + if diags.HasErrors() { + return nil, diags } // Update backend state file From 5232270fe0c43e9d19599d2ae9e39f93f7eb93e7 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 8 Aug 2025 12:42:39 +0100 Subject: [PATCH 19/54] Fix missing } --- internal/command/meta_backend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 5b0c9b714f64..42523a5203c1 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1693,6 +1693,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli } } // TODO: handle if input is not enabled + } } if diags.HasErrors() { return nil, diags From f39887f930a766976a183aa2d15bb39e76eab818 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 8 Aug 2025 13:08:41 +0100 Subject: [PATCH 20/54] Add handling of non-default workspace, remove input prompting for now. --- internal/command/meta_backend.go | 56 ++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 42523a5203c1..341de23a0021 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1646,29 +1646,48 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli if opts.Init && b != nil { err := m.selectWorkspace(b) if strings.Contains(err.Error(), "No existing workspaces") { + // If there are no workspaces, Terraform either needs to create the default workspace here, + // or instruct the user to run a `terraform workspace new` command. ws, err := m.Workspace() if err != nil { diags = diags.Append(fmt.Errorf("Failed to check current workspace: %w", err)) return nil, diags } - if m.Input() && ws == backend.DefaultStateName { - input := m.UIInput() - desc := fmt.Sprintf("Terraform will create the %q workspace via %q.\n"+ - "Only 'yes' will be accepted to approve.", backend.DefaultStateName, c.Type) - v, err := input.Input(context.Background(), &terraform.InputOpts{ - Id: "approve", - Query: fmt.Sprintf("Workspace %q does not exit, would you like to create one?", backend.DefaultStateName), - Description: desc, - }) - if err != nil { - diags = diags.Append(fmt.Errorf("Failed to confirm default workspace creation: %w", err)) - return nil, diags - } - if v != "yes" { - diags = diags.Append(errors.New("Failed to create default workspace")) - return nil, diags - } + switch { + case ws != backend.DefaultStateName: + // User needs to run a `terraform workspace new` command. + diags = append(diags, tfdiags.Sourceless( + tfdiags.Error, + fmt.Sprintf("Workspace %q has not been created yet", ws), + fmt.Sprintf("State store %q in provider %s (%q) reports that no workspaces currently exist. To create the custom workspace %q use the command `terraform workspace new %s`.", + c.Type, + c.Provider.Name, + c.ProviderAddr, + ws, + ws, + ), + )) + return nil, diags + + case ws == backend.DefaultStateName: + // TODO: do we want to prompt for input here (m.Input()), or create automatically unless -readonly flag present? + // input := m.UIInput() + // desc := fmt.Sprintf("Terraform will create the %q workspace via %q.\n"+ + // "Only 'yes' will be accepted to approve.", backend.DefaultStateName, c.Type) + // v, err := input.Input(context.Background(), &terraform.InputOpts{ + // Id: "approve", + // Query: fmt.Sprintf("Workspace %q does not exit, would you like to create one?", backend.DefaultStateName), + // Description: desc, + // }) + // if err != nil { + // diags = diags.Append(fmt.Errorf("Failed to confirm default workspace creation: %w", err)) + // return nil, diags + // } + // if v != "yes" { + // diags = diags.Append(errors.New("Failed to create default workspace")) + // return nil, diags + // } // TODO: Confirm if defaulting to creation on first use (rather than error) is a good idea // Make the default workspace. All other workspaces are user-created via the workspace commands. @@ -1691,6 +1710,9 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) return nil, diags } + default: + diags = diags.Append(err) + return nil, diags } // TODO: handle if input is not enabled } From 67d6f694da86c98df6c0ddf7832f29c93c2525be Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 8 Aug 2025 13:09:07 +0100 Subject: [PATCH 21/54] Rename state manager var, fix bug where PersistState was called on wrong state manager --- internal/command/meta_backend.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 341de23a0021..58a1e28e3c7e 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1691,7 +1691,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli // TODO: Confirm if defaulting to creation on first use (rather than error) is a good idea // Make the default workspace. All other workspaces are user-created via the workspace commands. - bStateMgr, sDiags := b.StateMgr(backend.DefaultStateName) + defaultSMgr, sDiags := b.StateMgr(backend.DefaultStateName) diags = diags.Append(sDiags) if sDiags.HasErrors() { diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug in Terraform and should be reported: %w", @@ -1702,11 +1702,11 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli return nil, diags } emptyState := states.NewState() - if err := bStateMgr.WriteState(emptyState); err != nil { + if err := defaultSMgr.WriteState(emptyState); err != nil { diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) return nil, diags } - if err := backendSMgr.PersistState(); err != nil { + if err := defaultSMgr.PersistState(nil); err != nil { diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) return nil, diags } From 4b848f7f69110c93feb7fa5e2908539cb6bfec84 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 8 Aug 2025 13:09:37 +0100 Subject: [PATCH 22/54] Make TODO indicating blocker --- internal/command/meta_backend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 58a1e28e3c7e..75de4528f4c0 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1706,6 +1706,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) return nil, diags } + // TODO - implement Read/Write state RPC methods if err := defaultSMgr.PersistState(nil); err != nil { diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) return nil, diags From 119b0ae1ab4b023d738c5b3e5c802396a4439e2b Mon Sep 17 00:00:00 2001 From: Sarah French Date: Tue, 12 Aug 2025 10:50:08 +0100 Subject: [PATCH 23/54] Add createDefaultWorkspace method --- internal/command/meta_backend.go | 51 +++++++++++++++++++------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 75de4528f4c0..145f67cb3782 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1691,26 +1691,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli // TODO: Confirm if defaulting to creation on first use (rather than error) is a good idea // Make the default workspace. All other workspaces are user-created via the workspace commands. - defaultSMgr, sDiags := b.StateMgr(backend.DefaultStateName) - diags = diags.Append(sDiags) - if sDiags.HasErrors() { - diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug in Terraform and should be reported: %w", - c.Type, - c.Provider.Name, - c.ProviderAddr, - sDiags.Err())) - return nil, diags - } - emptyState := states.NewState() - if err := defaultSMgr.WriteState(emptyState); err != nil { - diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) - return nil, diags - } - // TODO - implement Read/Write state RPC methods - if err := defaultSMgr.PersistState(nil); err != nil { - diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) - return nil, diags - } + m.createDefaultWorkspace(c, b) default: diags = diags.Append(err) return nil, diags @@ -1735,6 +1716,36 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli return b, diags } +// createDefaultWorkspace receives a backend made using a pluggable state store, and details about that store's config, +// and persists an empty state file in the default workspace. By creating this artifact we ensure that the default +// workspace is created and usable by Terraform in later operations. +func (m *Meta) createDefaultWorkspace(c *configs.StateStore, b backend.Backend) tfdiags.Diagnostics { + var diags tfdiags.Diagnostics + + defaultSMgr, sDiags := b.StateMgr(backend.DefaultStateName) + diags = diags.Append(sDiags) + if sDiags.HasErrors() { + diags = diags.Append(fmt.Errorf("Failed to create a state manager for state store %q in provider %s (%q). This is a bug in Terraform and should be reported: %w", + c.Type, + c.Provider.Name, + c.ProviderAddr, + sDiags.Err())) + return diags + } + emptyState := states.NewState() + if err := defaultSMgr.WriteState(emptyState); err != nil { + diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) + return diags + } + // TODO - implement Read/Write state RPC methods + if err := defaultSMgr.PersistState(nil); err != nil { + diags = diags.Append(fmt.Errorf(errStateStoreWorkspaceCreate, c.Type, err)) + return diags + } + + return diags +} + // Initializing a saved backend from the cache file (legacy state file) // // TODO: This is extremely similar to Meta.backendFromState() but for legacy reasons this is the From debdb73e792bdb0382f8187d76d55e316af5b45b Mon Sep 17 00:00:00 2001 From: Sarah French Date: Tue, 12 Aug 2025 11:31:30 +0100 Subject: [PATCH 24/54] Add locks to test --- internal/command/meta_backend_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index af159e78649c..85f87f350768 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2234,11 +2234,27 @@ func TestMetaBackend_reconfigureStateStoreChange(t *testing.T) { return mock, nil } + // Create locks - these would normally be the locks derived from config + locks := depsfile.NewLocks() + constraint, err := providerreqs.ParseVersionConstraints(">9.0.0") + if err != nil { + t.Fatalf("test setup failed when making constraint: %s", err) + } + expectedVersionString := "9.9.9" + expectedProviderSource := "registry.terraform.io/my-org/foo" + locks.SetProvider( + addrs.MustParseProviderSourceString(expectedProviderSource), + versions.MustParseVersion(expectedVersionString), + constraint, + []providerreqs.Hash{"h1:foo"}, + ) + // Get the operations backend _, beDiags := m.Backend(&BackendOpts{ Init: true, StateStoreConfig: mod.StateStore, ProviderFactory: factory, + Locks: locks, }) if !beDiags.HasErrors() { From f527f4f85406309ae179cb5a3daa07687ad65484 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 15 Aug 2025 15:07:02 +0100 Subject: [PATCH 25/54] Add `-create-default-workspace` flag to init command, for overriding behaviour when TF isn't in interactive mode This required refactoring the signature of the initBackend method; pass in all init args instead of a subset --- internal/command/arguments/init.go | 14 +++++- internal/command/init.go | 15 ++++--- internal/command/init_run_experiment.go | 2 +- internal/command/meta_backend.go | 59 ++++++++++++++++--------- 4 files changed, 59 insertions(+), 31 deletions(-) diff --git a/internal/command/arguments/init.go b/internal/command/arguments/init.go index 6bb74473847a..91b2846b7456 100644 --- a/internal/command/arguments/init.go +++ b/internal/command/arguments/init.go @@ -78,6 +78,10 @@ type Init struct { // TODO(SarahFrench/radeksimko): Remove this once the feature is no longer // experimental EnablePssExperiment bool + + // CreateDefaultWorkspace indicates whether the default workspace should be created by + // Terraform when initializing a state store for the first time. + CreateDefaultWorkspace bool } // ParseInit processes CLI arguments, returning an Init value and errors. @@ -111,7 +115,7 @@ func ParseInit(args []string) (*Init, tfdiags.Diagnostics) { cmdFlags.BoolVar(&init.Json, "json", false, "json") cmdFlags.Var(&init.BackendConfig, "backend-config", "") cmdFlags.Var(&init.PluginPath, "plugin-dir", "plugin directory") - + cmdFlags.BoolVar(&init.CreateDefaultWorkspace, "create-default-workspace", true, "when -input=false, use this flag to block creation of the default workspace") // Used for enabling experimental code that's invoked before configuration is parsed. cmdFlags.BoolVar(&init.EnablePssExperiment, "enable-pluggable-state-storage-experiment", false, "Enable the pluggable state storage experiment") @@ -139,6 +143,14 @@ func ParseInit(args []string) (*Init, tfdiags.Diagnostics) { )) } + if init.InputEnabled && !init.CreateDefaultWorkspace { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Warning, + "Invalid init options", + "The flag -create-default-workspace=false is ignored when Terraform is configured to ask users for input. Instead, add -input=false or remove the -create-default-workspace flag", + )) + } + init.Args = cmdFlags.Args() backendFlagSet := FlagIsSet(cmdFlags, "backend") diff --git a/internal/command/init.go b/internal/command/init.go index 79775f7f4633..1f78a6aff29f 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -51,7 +51,7 @@ func (c *InitCommand) Run(args []string) int { args = c.Meta.process(args) initArgs, initDiags := arguments.ParseInit(args) - view := views.NewInit(initArgs.ViewType, c.View) + view := views.NewInit(viewType, c.View) if initDiags.HasErrors() { diags = diags.Append(initDiags) @@ -246,12 +246,13 @@ func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, ext } opts = &BackendOpts{ - StateStoreConfig: root.StateStore, - Locks: locks, - ProviderFactory: factory, - ConfigOverride: configOverride, - Init: true, - ViewType: viewType, + StateStoreConfig: root.StateStore, + Locks: locks, + ProviderFactory: factory, + CreateDefaultWorkspace: initArgs.CreateDefaultWorkspace, + ConfigOverride: configOverride, + Init: true, + ViewType: viewType, } case root.Backend != nil: diff --git a/internal/command/init_run_experiment.go b/internal/command/init_run_experiment.go index 59b44e1ec3aa..f5a27e35a5c2 100644 --- a/internal/command/init_run_experiment.go +++ b/internal/command/init_run_experiment.go @@ -208,7 +208,7 @@ func (c *InitCommand) runPssInit(initArgs *arguments.Init, view views.Init) int // This handles case when config contains either backend or state_store blocks. // This is valid as either can be implementations of backend.Backend, which is what we // obtain here. - back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, configLocks, view) + back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs, configLocks, view) default: // load the previously-stored backend config back, backDiags = c.Meta.backendFromState(ctx) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 145f67cb3782..fe2284959e72 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -89,6 +89,10 @@ type BackendOpts struct { // ViewType will set console output format for the // initialization operation (JSON or human-readable). ViewType arguments.ViewType + + // CreateDefaultWorkspace signifies whether the operations backend should create + // the default workspace or not + CreateDefaultWorkspace bool } // BackendWithRemoteTerraformVersion is a shared interface between the 'remote' and 'cloud' backends @@ -1671,32 +1675,43 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli return nil, diags case ws == backend.DefaultStateName: - // TODO: do we want to prompt for input here (m.Input()), or create automatically unless -readonly flag present? - // input := m.UIInput() - // desc := fmt.Sprintf("Terraform will create the %q workspace via %q.\n"+ - // "Only 'yes' will be accepted to approve.", backend.DefaultStateName, c.Type) - // v, err := input.Input(context.Background(), &terraform.InputOpts{ - // Id: "approve", - // Query: fmt.Sprintf("Workspace %q does not exit, would you like to create one?", backend.DefaultStateName), - // Description: desc, - // }) - // if err != nil { - // diags = diags.Append(fmt.Errorf("Failed to confirm default workspace creation: %w", err)) - // return nil, diags - // } - // if v != "yes" { - // diags = diags.Append(errors.New("Failed to create default workspace")) - // return nil, diags - // } - - // TODO: Confirm if defaulting to creation on first use (rather than error) is a good idea - // Make the default workspace. All other workspaces are user-created via the workspace commands. - m.createDefaultWorkspace(c, b) + // Should we create the default state after prompting the user, or not? + if m.Input() { + // If input is enabled, we prompt the user before creating the default workspace. + input := m.UIInput() + desc := fmt.Sprintf("Terraform will create the %q workspace via state store %q.\n"+ + "Only 'yes' will be accepted to approve.", backend.DefaultStateName, c.Type) + v, err := input.Input(context.Background(), &terraform.InputOpts{ + Id: "approve", + Query: fmt.Sprintf("Workspace the %s workspace does not exit, would you like to create it?", backend.DefaultStateName), + Description: desc, + }) + if err != nil { + diags = diags.Append(fmt.Errorf("Failed to confirm %s workspace creation: %w", backend.DefaultStateName, err)) + return nil, diags + } + if v != "yes" { + diags = diags.Append(fmt.Errorf("Cancelled creation of the %s workspace", backend.DefaultStateName)) + return nil, diags + } + m.createDefaultWorkspace(c, b) + } else { + // If input is disabled, we don't prompt before creating the default workspace. + // However this can be blocked with other flags present. + if opts.CreateDefaultWorkspace { + m.createDefaultWorkspace(c, b) + } else { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "The default workspace does not exist", + Detail: "Terraform has been configured to skip creation of the default workspace in the state store. This may cause issues in subsequent Terraform operations", + }) + } + } default: diags = diags.Append(err) return nil, diags } - // TODO: handle if input is not enabled } } if diags.HasErrors() { From 094328884e681774d8bb4948a17a283bc3227c15 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 15 Aug 2025 15:24:07 +0100 Subject: [PATCH 26/54] WIP - updating tests, currently blocked --- internal/command/meta_backend_test.go | 222 ++++++++++++-------- internal/providers/testing/provider_mock.go | 23 +- 2 files changed, 156 insertions(+), 89 deletions(-) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 85f87f350768..7e0d49549fa1 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2081,82 +2081,154 @@ func Test_determineInitReason(t *testing.T) { } // Newly configured state store -// -// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch -// case for this scenario, and will need to be updated when that init feature is implemented. +// Working directory has state_store in config but no preexisting backend state file func TestMetaBackend_configureNewStateStore(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-new"), td) - t.Chdir(td) + cases := map[string]struct { + // setup + isInitCommand bool - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true + inputEnabled bool + inputText string - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + createDefaultWorkspace bool + // assertions + expectedError string + expectDefaultWorkspaceExists bool + }{ + "an init command prompts users for input when the default workspace needs to be created": { + inputEnabled: true, + createDefaultWorkspace: true, + inputText: "yes", + isInitCommand: true, + expectDefaultWorkspaceExists: true, + }, + "an init command with input disabled will create the default workspace automatically": { + inputEnabled: false, + createDefaultWorkspace: true, + isInitCommand: true, + expectDefaultWorkspaceExists: true, + }, + "an init command with input disabled and the flag -create-default-workspace=false will not make the default workspace": { + inputEnabled: false, + createDefaultWorkspace: false, + isInitCommand: true, + expectDefaultWorkspaceExists: false, + }, + // "during a non-init command, the command ends in with an error telling the user to run an init command": { + // isInitCommand: false, + // expectedError: "State store initialization required, please run \"terraform init\": Reason: Initial configuration of the requested state_store \"foo_bar\" in provider foo (\"registry.terraform.io/my-org/foo\")", + // }, } - // Get mock provider factory to be used during init - // - // This imagines a provider called foo that contains - // a pluggable state store implementation called bar. - pssName := "test_store" - mock := testStateStoreMock(t) - factory := func() (providers.Interface, error) { - return mock, nil - } + for tn, tc := range cases { + t.Run(tn, func(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-new"), td) + t.Chdir(td) - // Create locks - these would normally be the locks derived from config - locks := depsfile.NewLocks() - constraint, err := providerreqs.ParseVersionConstraints(">9.0.0") - if err != nil { - t.Fatalf("test setup failed when making constraint: %s", err) - } - expectedVersionString := "9.9.9" - expectedProviderSource := "registry.terraform.io/hashicorp/test" - locks.SetProvider( - addrs.MustParseProviderSourceString(expectedProviderSource), - versions.MustParseVersion(expectedVersionString), - constraint, - []providerreqs.Hash{"h1:foo"}, - ) + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true + m.input = tc.inputEnabled + if tc.inputEnabled { + defer testInteractiveInput(t, []string{tc.inputText})() + } - // Act - get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - StateStoreConfig: mod.StateStore, - ProviderFactory: factory, - Locks: locks, - }) - if beDiags.HasErrors() { - t.Fatalf("unexpected error: %s", beDiags.Err()) - } + // Get the state store's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } - // Check the backend state file exists & assert its contents - s := testDataStateRead(t, filepath.Join(DefaultDataDir, backendLocal.DefaultStateFilename)) - if s == nil { - t.Fatal("expected backend state file to be created, but it was missing") - } + // Get mock provider factory to be used during init + // + // This imagines a provider called foo that contains + // a pluggable state store implementation called bar. + pssName := "test_store" + mock := testStateStoreMock(t) + factory := func() (providers.Interface, error) { + return mock, nil + } - if s.StateStore.Type != pssName { - t.Fatalf("backend state file contains unexpected state store type, want %q, got %q", pssName, s.StateStore.Type) - } - if s.StateStore.Provider.Version.String() != expectedVersionString { - t.Fatalf("backend state file contains unexpected version, want %q, got %q", expectedVersionString, s.StateStore.Provider.Version) - } - if s.StateStore.Provider.Source.String() != expectedProviderSource { - t.Fatalf("backend state file contains unexpected source, want %q, got %q", expectedProviderSource, s.StateStore.Provider.Source) - } - expectedProviderConfig := "{ \"region\": \"mars\" }" - expectedStoreConfig := "{ \"value\": \"foobar\" }" - if cleanString(string(s.StateStore.Provider.ConfigRaw)) != expectedProviderConfig { - t.Fatalf("backend state file contains unexpected raw config data for the provider, want %q, got %q", expectedProviderConfig, cleanString(string(s.StateStore.Provider.ConfigRaw))) - } - if cleanString(string(s.StateStore.ConfigRaw)) != expectedStoreConfig { - t.Fatalf("backend state file contains unexpected raw config data for the state store, want %q, got %q", expectedStoreConfig, cleanString(string(s.StateStore.ConfigRaw))) + // Create locks - these would normally be the locks derived from config + locks := depsfile.NewLocks() + constraint, err := providerreqs.ParseVersionConstraints(">9.0.0") + if err != nil { + t.Fatalf("test setup failed when making constraint: %s", err) + } + expectedVersionString := "9.9.9" + expectedProviderSource := "registry.terraform.io/hashicorp/test" + locks.SetProvider( + addrs.MustParseProviderSourceString(expectedProviderSource), + versions.MustParseVersion(expectedVersionString), + constraint, + []providerreqs.Hash{"h1:foo"}, + ) + + // Act - get the operations backend + b, beDiags := m.Backend(&BackendOpts{ + Init: tc.isInitCommand, // Changes with test case + StateStoreConfig: mod.StateStore, + ProviderFactory: factory, + Locks: locks, + CreateDefaultWorkspace: tc.createDefaultWorkspace, + }) + if beDiags.HasErrors() { + if tc.expectedError == "" { + t.Fatalf("unexpected error: %s", beDiags.Err()) + } + if !strings.Contains(cleanString(beDiags.Err().Error()), tc.expectedError) { + t.Fatalf("expected error to contain %s, but instead got: %s", tc.expectedError, cleanString(beDiags.Err().Error())) + } + return // error is as expected + } + if tc.expectedError != "" && !beDiags.HasErrors() { + t.Fatal("expected error missing") + } + + // Check the backend state file exists & assert its contents + s := testDataStateRead(t, filepath.Join(DefaultDataDir, backendLocal.DefaultStateFilename)) + if s == nil { + t.Fatal("expected backend state file to be created, but it was missing") + } + + if s.StateStore.Type != pssName { + t.Fatalf("backend state file contains unexpected state store type, want %q, got %q", pssName, s.StateStore.Type) + } + if s.StateStore.Provider.Version.String() != expectedVersionString { + t.Fatalf("backend state file contains unexpected version, want %q, got %q", expectedVersionString, s.StateStore.Provider.Version) + } + if s.StateStore.Provider.Source.String() != expectedProviderSource { + t.Fatalf("backend state file contains unexpected source, want %q, got %q", expectedProviderSource, s.StateStore.Provider.Source) + } + expectedProviderConfig := "{ \"region\": \"mars\" }" + expectedStoreConfig := "{ \"bar\": \"foobar\" }" + if cleanString(string(s.StateStore.Provider.ConfigRaw)) != expectedProviderConfig { + t.Fatalf("backend state file contains unexpected raw config data for the provider, want %q, got %q", expectedProviderConfig, cleanString(string(s.StateStore.Provider.ConfigRaw))) + } + if cleanString(string(s.StateStore.ConfigRaw)) != expectedStoreConfig { + t.Fatalf("backend state file contains unexpected raw config data for the state store, want %q, got %q", expectedStoreConfig, cleanString(string(s.StateStore.ConfigRaw))) + } + + w, wDiags := b.Workspaces() + if wDiags.HasErrors() { + t.Fatalf("unexpected error: %s", wDiags.Err()) + } + if len(w) == 0 { + if tc.expectDefaultWorkspaceExists { + t.Fatal("expected the default workspace to exist, but there are no workspaces") + } + return + } + if len(w) > 0 { + if tc.expectDefaultWorkspaceExists { + if len(w) == 1 && w[0] != "default" { + t.Fatalf("expected the default workspace to exist, but instead got: %v", w) + } + } + t.Fatalf("expected the default workspace to be the only existing workspace, but instead got: %v", w) + } + }) } } @@ -2234,27 +2306,11 @@ func TestMetaBackend_reconfigureStateStoreChange(t *testing.T) { return mock, nil } - // Create locks - these would normally be the locks derived from config - locks := depsfile.NewLocks() - constraint, err := providerreqs.ParseVersionConstraints(">9.0.0") - if err != nil { - t.Fatalf("test setup failed when making constraint: %s", err) - } - expectedVersionString := "9.9.9" - expectedProviderSource := "registry.terraform.io/my-org/foo" - locks.SetProvider( - addrs.MustParseProviderSourceString(expectedProviderSource), - versions.MustParseVersion(expectedVersionString), - constraint, - []providerreqs.Hash{"h1:foo"}, - ) - // Get the operations backend _, beDiags := m.Backend(&BackendOpts{ Init: true, StateStoreConfig: mod.StateStore, ProviderFactory: factory, - Locks: locks, }) if !beDiags.HasErrors() { diff --git a/internal/providers/testing/provider_mock.go b/internal/providers/testing/provider_mock.go index 9b38da2ba83b..82f92e79953e 100644 --- a/internal/providers/testing/provider_mock.go +++ b/internal/providers/testing/provider_mock.go @@ -5,12 +5,15 @@ package testing import ( "fmt" + "maps" + "slices" "sync" "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" "github.com/zclconf/go-cty/cty/msgpack" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/terraform/internal/configs/hcl2shim" "github.com/hashicorp/terraform/internal/providers" ) @@ -156,6 +159,9 @@ type MockProvider struct { WriteStateBytesFn func(providers.WriteStateBytesRequest) providers.WriteStateBytesResponse WriteStateBytesResponse providers.WriteStateBytesResponse + // states is an internal field that tracks which workspaces have been created in a test + states map[string]interface{} + GetStatesCalled bool GetStatesResponse *providers.GetStatesResponse GetStatesRequest providers.GetStatesRequest @@ -1032,11 +1038,8 @@ func (p *MockProvider) GetStates(r providers.GetStatesRequest) (resp providers.G return p.GetStatesFn(r) } - // If the mock has no further inputs, return an empty list. - // The state store should be reporting a minimum of the default workspace usually, - // but this should be achieved by querying data storage and identifying the artifact - // for that workspace, and reporting that the workspace exists. - resp.States = []string{} + // If the mock has no further inputs, return the internal states list + resp.States = slices.Sorted(maps.Keys(p.states)) return resp } @@ -1063,7 +1066,15 @@ func (p *MockProvider) DeleteState(r providers.DeleteStateRequest) (resp provide return p.DeleteStateFn(r) } - // There's no logic we can include here in the absence of other fields on the mock. + if _, match := p.states[r.StateId]; match { + delete(p.states, r.StateId) + } else { + resp.Diagnostics.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Workspace cannot be deleted", + Detail: fmt.Sprintf("The workspace %q does not exist, so cannot be deleted", r.StateId), + }) + } // If the response contains no diagnostics then the deletion is assumed to be successful. return resp From 87d7af4d86a0a02fb979ec0d5300badde570779c Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 19 Sep 2025 17:47:44 +0100 Subject: [PATCH 27/54] Random fix from rebasing/remaking branch... --- internal/command/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/init.go b/internal/command/init.go index 1f78a6aff29f..ed41e180cbbc 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -51,7 +51,7 @@ func (c *InitCommand) Run(args []string) int { args = c.Meta.process(args) initArgs, initDiags := arguments.ParseInit(args) - view := views.NewInit(viewType, c.View) + view := views.NewInit(initArgs.ViewType, c.View) if initDiags.HasErrors() { diags = diags.Append(initDiags) From 5a2f682adff3ec06ed9afd0afc2efb1e46a0d9b7 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 18 Aug 2025 12:37:07 +0100 Subject: [PATCH 28/54] Update tests with new flag --- internal/command/arguments/init_test.go | 31 ++++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/command/arguments/init_test.go b/internal/command/arguments/init_test.go index 93e13b7b6281..2c78e97254fb 100644 --- a/internal/command/arguments/init_test.go +++ b/internal/command/arguments/init_test.go @@ -40,10 +40,11 @@ func TestParseInit_basicValid(t *testing.T) { FlagName: "-backend-config", Items: &flagNameValue, }, - Vars: &Vars{}, - InputEnabled: true, - CompactWarnings: false, - TargetFlags: nil, + Vars: &Vars{}, + InputEnabled: true, + CompactWarnings: false, + TargetFlags: nil, + CreateDefaultWorkspace: true, }, }, "setting multiple options": { @@ -72,11 +73,12 @@ func TestParseInit_basicValid(t *testing.T) { FlagName: "-backend-config", Items: &flagNameValue, }, - Vars: &Vars{}, - InputEnabled: true, - Args: []string{}, - CompactWarnings: true, - TargetFlags: nil, + Vars: &Vars{}, + InputEnabled: true, + Args: []string{}, + CompactWarnings: true, + TargetFlags: nil, + CreateDefaultWorkspace: true, }, }, "with cloud option": { @@ -101,11 +103,12 @@ func TestParseInit_basicValid(t *testing.T) { FlagName: "-backend-config", Items: &[]FlagNameValue{{Name: "-backend-config", Value: "backend.config"}}, }, - Vars: &Vars{}, - InputEnabled: false, - Args: []string{}, - CompactWarnings: false, - TargetFlags: []string{"foo_bar.baz"}, + Vars: &Vars{}, + InputEnabled: false, + Args: []string{}, + CompactWarnings: false, + TargetFlags: []string{"foo_bar.baz"}, + CreateDefaultWorkspace: true, }, }, } From ec5ae5706a53009bd33b9b108b1d250b4a1567b2 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 18 Aug 2025 12:37:39 +0100 Subject: [PATCH 29/54] Pivot to creating the default workspace without prompts This change was following discussions with Product --- internal/command/meta_backend.go | 37 +++++---------------------- internal/command/meta_backend_test.go | 31 +++++----------------- 2 files changed, 14 insertions(+), 54 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index fe2284959e72..6d3fd1a2eb65 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1675,38 +1675,15 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli return nil, diags case ws == backend.DefaultStateName: - // Should we create the default state after prompting the user, or not? - if m.Input() { - // If input is enabled, we prompt the user before creating the default workspace. - input := m.UIInput() - desc := fmt.Sprintf("Terraform will create the %q workspace via state store %q.\n"+ - "Only 'yes' will be accepted to approve.", backend.DefaultStateName, c.Type) - v, err := input.Input(context.Background(), &terraform.InputOpts{ - Id: "approve", - Query: fmt.Sprintf("Workspace the %s workspace does not exit, would you like to create it?", backend.DefaultStateName), - Description: desc, - }) - if err != nil { - diags = diags.Append(fmt.Errorf("Failed to confirm %s workspace creation: %w", backend.DefaultStateName, err)) - return nil, diags - } - if v != "yes" { - diags = diags.Append(fmt.Errorf("Cancelled creation of the %s workspace", backend.DefaultStateName)) - return nil, diags - } + // Users control if the default workspace is created through the -create-default-workspace flag (defaults to true) + if opts.CreateDefaultWorkspace { m.createDefaultWorkspace(c, b) } else { - // If input is disabled, we don't prompt before creating the default workspace. - // However this can be blocked with other flags present. - if opts.CreateDefaultWorkspace { - m.createDefaultWorkspace(c, b) - } else { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagWarning, - Summary: "The default workspace does not exist", - Detail: "Terraform has been configured to skip creation of the default workspace in the state store. This may cause issues in subsequent Terraform operations", - }) - } + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "The default workspace does not exist", + Detail: "Terraform has been configured to skip creation of the default workspace in the state store.", + }) } default: diags = diags.Append(err) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 7e0d49549fa1..08bd0bac11bf 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2085,39 +2085,26 @@ func Test_determineInitReason(t *testing.T) { func TestMetaBackend_configureNewStateStore(t *testing.T) { cases := map[string]struct { // setup - isInitCommand bool - - inputEnabled bool - inputText string - + isInitCommand bool createDefaultWorkspace bool // assertions expectedError string expectDefaultWorkspaceExists bool }{ - "an init command prompts users for input when the default workspace needs to be created": { - inputEnabled: true, - createDefaultWorkspace: true, - inputText: "yes", - isInitCommand: true, - expectDefaultWorkspaceExists: true, - }, - "an init command with input disabled will create the default workspace automatically": { - inputEnabled: false, + "an init command creates the default workspace by default": { createDefaultWorkspace: true, isInitCommand: true, expectDefaultWorkspaceExists: true, }, - "an init command with input disabled and the flag -create-default-workspace=false will not make the default workspace": { - inputEnabled: false, + "an init command with the flag -create-default-workspace=false will not make the default workspace": { createDefaultWorkspace: false, isInitCommand: true, expectDefaultWorkspaceExists: false, }, - // "during a non-init command, the command ends in with an error telling the user to run an init command": { - // isInitCommand: false, - // expectedError: "State store initialization required, please run \"terraform init\": Reason: Initial configuration of the requested state_store \"foo_bar\" in provider foo (\"registry.terraform.io/my-org/foo\")", - // }, + "during a non-init command, the command ends in with an error telling the user to run an init command": { + isInitCommand: false, + expectedError: "State store initialization required, please run \"terraform init\": Reason: Initial configuration of the requested state_store \"foo_bar\" in provider foo (\"registry.terraform.io/my-org/foo\")", + }, } for tn, tc := range cases { @@ -2129,10 +2116,6 @@ func TestMetaBackend_configureNewStateStore(t *testing.T) { // Setup the meta m := testMetaBackend(t, nil) m.AllowExperimentalFeatures = true - m.input = tc.inputEnabled - if tc.inputEnabled { - defer testInteractiveInput(t, []string{tc.inputText})() - } // Get the state store's config mod, loadDiags := m.loadSingleModule(td) From 4ce302e00803dcdfe664e65c81a52d3795a9bcbd Mon Sep 17 00:00:00 2001 From: Sarah French Date: Tue, 26 Aug 2025 16:45:58 +0100 Subject: [PATCH 30/54] Fix: Make sure diags from attempting to create the default workspace are returned --- internal/command/meta_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 6d3fd1a2eb65..107b3b60fb03 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1677,7 +1677,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli case ws == backend.DefaultStateName: // Users control if the default workspace is created through the -create-default-workspace flag (defaults to true) if opts.CreateDefaultWorkspace { - m.createDefaultWorkspace(c, b) + diags = diags.Append(m.createDefaultWorkspace(c, b)) } else { diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagWarning, From d8ce375fcef1e5ae974cb1c42aff44be8af4aecf Mon Sep 17 00:00:00 2001 From: Sarah French Date: Tue, 26 Aug 2025 16:46:58 +0100 Subject: [PATCH 31/54] Add rough implementation of Get and Put methods on remote gRPC state client --- internal/states/remote/remote_grpc.go | 31 +++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/states/remote/remote_grpc.go b/internal/states/remote/remote_grpc.go index cc3d1ce7d57c..7be226fd4fdc 100644 --- a/internal/states/remote/remote_grpc.go +++ b/internal/states/remote/remote_grpc.go @@ -53,16 +53,39 @@ type grpcClient struct { // and returns a copy of the downloaded state data. // // Implementation of remote.Client -func (g *grpcClient) Get() (*Payload, tfdiags.Diagnostics) { - panic("not implemented yet") +func (g *grpcClient) Get() (*Payload, error) { + // TODO - replace with method implementation added to main branch + req := providers.ReadStateBytesRequest{ + TypeName: g.typeName, + StateId: g.stateId, + } + resp := g.provider.ReadStateBytes(req) + + if len(resp.Bytes) == 0 { + // No state to return + return nil, resp.Diagnostics.Err() + } + + payload := &Payload{ + Data: resp.Bytes, + MD5: []byte("foobar"), + } + return payload, resp.Diagnostics.Err() } // Put invokes the WriteStateBytes gRPC method in the plugin protocol // and to transfer state data to the remote location. // // Implementation of remote.Client -func (g *grpcClient) Put(state []byte) tfdiags.Diagnostics { - panic("not implemented yet") +func (g *grpcClient) Put(state []byte) error { + // TODO - replace with method implementation added to main branch + req := providers.WriteStateBytesRequest{ + TypeName: g.typeName, + StateId: g.stateId, + } + resp := g.provider.WriteStateBytes(req) + + return resp.Diagnostics.Err() } // Delete invokes the DeleteState gRPC method in the plugin protocol From b89f2157855f2f0600994e66a62d60c54ed84ef1 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Tue, 26 Aug 2025 16:47:26 +0100 Subject: [PATCH 32/54] Update tests and mock; this is unblocked now Get and Put are implemented --- internal/command/meta_backend_test.go | 8 +++++--- internal/providers/testing/provider_mock.go | 11 ++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 08bd0bac11bf..84ee85140bd1 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2205,11 +2205,13 @@ func TestMetaBackend_configureNewStateStore(t *testing.T) { } if len(w) > 0 { if tc.expectDefaultWorkspaceExists { - if len(w) == 1 && w[0] != "default" { - t.Fatalf("expected the default workspace to exist, but instead got: %v", w) + if len(w) != 1 || w[0] != "default" { + t.Fatalf("expected only the default workspace to exist, but instead got: %v", w) } + return // we've got the expected default workspace } - t.Fatalf("expected the default workspace to be the only existing workspace, but instead got: %v", w) + + t.Fatalf("got unexpected workspaces: %v", w) } }) } diff --git a/internal/providers/testing/provider_mock.go b/internal/providers/testing/provider_mock.go index 82f92e79953e..953577a58973 100644 --- a/internal/providers/testing/provider_mock.go +++ b/internal/providers/testing/provider_mock.go @@ -159,8 +159,9 @@ type MockProvider struct { WriteStateBytesFn func(providers.WriteStateBytesRequest) providers.WriteStateBytesResponse WriteStateBytesResponse providers.WriteStateBytesResponse - // states is an internal field that tracks which workspaces have been created in a test - states map[string]interface{} + // MockStates is an internal field that tracks which workspaces have been created in a test + // The map keys are state ids (workspaces) and the value depends on the test. + MockStates map[string]interface{} GetStatesCalled bool GetStatesResponse *providers.GetStatesResponse @@ -1039,7 +1040,7 @@ func (p *MockProvider) GetStates(r providers.GetStatesRequest) (resp providers.G } // If the mock has no further inputs, return the internal states list - resp.States = slices.Sorted(maps.Keys(p.states)) + resp.States = slices.Sorted(maps.Keys(p.MockStates)) return resp } @@ -1066,8 +1067,8 @@ func (p *MockProvider) DeleteState(r providers.DeleteStateRequest) (resp provide return p.DeleteStateFn(r) } - if _, match := p.states[r.StateId]; match { - delete(p.states, r.StateId) + if _, match := p.MockStates[r.StateId]; match { + delete(p.MockStates, r.StateId) } else { resp.Diagnostics.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, From 40947deb8d6ee5209780064fd4b90612b1236d6a Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 29 Aug 2025 11:47:42 +0100 Subject: [PATCH 33/54] Move init command test cases up to the 'command testing' level. This is currently only possible for testing init commands interacting with the PSS command --- internal/command/init_test.go | 168 +++++++++++++ internal/command/meta_backend_test.go | 221 ++++++++---------- .../testdata/init-with-state-store/main.tf | 8 +- 3 files changed, 270 insertions(+), 127 deletions(-) diff --git a/internal/command/init_test.go b/internal/command/init_test.go index 361702972b94..ec9a3c49b68a 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -20,6 +20,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/command/arguments" "github.com/hashicorp/terraform/internal/command/views" "github.com/hashicorp/terraform/internal/configs" @@ -3227,6 +3228,111 @@ func TestInit_testsWithModule(t *testing.T) { } } +// Testing init's behaviors when run in an empty working directory +func TestInit_stateStore_newWorkingDir(t *testing.T) { + t.Run("an init command creates the default workspace by default", func(t *testing.T) { + // Create a temporary, uninitialized working directory with configuration including a state store + td := t.TempDir() + testCopyDir(t, testFixturePath("init-with-state-store"), td) + t.Chdir(td) + + mockProvider := mockPluggableStateStorageProvider() + mockProviderAddress := addrs.NewDefaultProvider("test") + providerSource, close := newMockProviderSource(t, map[string][]string{ + "hashicorp/test": {"1.0.0"}, + }) + defer close() + + ui := new(cli.MockUi) + view, done := testView(t) + c := &InitCommand{ + Meta: Meta{ + Ui: ui, + View: view, + AllowExperimentalFeatures: true, + testingOverrides: &testingOverrides{ + Providers: map[addrs.Provider]providers.Factory{ + mockProviderAddress: providers.FactoryFixed(mockProvider), + }, + }, + ProviderSource: providerSource, + }, + } + + args := []string{"-enable-pluggable-state-storage-experiment=true"} + code := c.Run(args) + testOutput := done(t) + if code != 0 { + t.Fatalf("expected code 0 exit code, got %d, output: \n%s", code, testOutput.All()) + } + + // Check output + output := testOutput.All() + expectedOutput := `Initializing the state store...` + if !strings.Contains(output, expectedOutput) { + t.Fatalf("expected output to include %q, but got':\n %s", expectedOutput, output) + } + + // Assert the default workspace was created + if _, exists := mockProvider.MockStates[backend.DefaultStateName]; !exists { + t.Fatal("expected the default workspace to be created during init, but it is missing") + } + }) + + t.Run("an init command with the flag -create-default-workspace=false will not make the default workspace", func(t *testing.T) { + // Create a temporary, uninitialized working directory with configuration including a state store + td := t.TempDir() + testCopyDir(t, testFixturePath("init-with-state-store"), td) + t.Chdir(td) + + mockProvider := mockPluggableStateStorageProvider() + mockProviderAddress := addrs.NewDefaultProvider("test") + providerSource, close := newMockProviderSource(t, map[string][]string{ + "hashicorp/test": {"1.0.0"}, + }) + defer close() + + ui := new(cli.MockUi) + view, done := testView(t) + c := &InitCommand{ + Meta: Meta{ + Ui: ui, + View: view, + AllowExperimentalFeatures: true, + testingOverrides: &testingOverrides{ + Providers: map[addrs.Provider]providers.Factory{ + mockProviderAddress: providers.FactoryFixed(mockProvider), + }, + }, + ProviderSource: providerSource, + }, + } + + args := []string{"-enable-pluggable-state-storage-experiment=true", "-create-default-workspace=false"} + code := c.Run(args) + testOutput := done(t) + if code != 0 { + t.Fatalf("expected code 0 exit code, got %d, output: \n%s", code, testOutput.All()) + } + + // Check output + output := testOutput.All() + expectedOutput := `Initializing the state store...` + if !strings.Contains(output, expectedOutput) { + t.Fatalf("expected output to include %q, but got':\n %s", expectedOutput, output) + } + + // Assert the default workspace was created + if _, exists := mockProvider.MockStates[backend.DefaultStateName]; exists { + t.Fatal("expected Terraform to skip creating the default workspace, but it has been created") + } + }) + + // TODO: Add test cases below once PSS feature isn't experimental. + // Currently these tests are handled at a lower level in `internal/command/meta_backend_test.go`: + // > "during a non-init command, the command ends in with an error telling the user to run an init command" +} + // newMockProviderSource is a helper to succinctly construct a mock provider // source that contains a set of packages matching the given provider versions // that are available for installation (from temporary local files). @@ -3367,3 +3473,65 @@ func expectedPackageInstallPath(name, version string, exe bool) string { baseDir, fmt.Sprintf("registry.terraform.io/hashicorp/%s/%s/%s", name, version, platform), )) } + +func mockPluggableStateStorageProvider() *testing_provider.MockProvider { + // Create a mock provider to use for PSS + // Get mock provider factory to be used during init + // + // This imagines a provider called `test` that contains + // a pluggable state store implementation called `store`. + pssName := "test_store" + mock := testing_provider.MockProvider{ + GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ + Provider: providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "region": {Type: cty.String, Optional: true}, + }, + }, + }, + DataSources: map[string]providers.Schema{}, + ResourceTypes: map[string]providers.Schema{}, + ListResourceTypes: map[string]providers.Schema{}, + StateStores: map[string]providers.Schema{ + pssName: { + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "bar": { + Type: cty.String, + Required: true, + }, + }, + }, + }, + }, + }, + } + mock.WriteStateBytesFn = func(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { + // Workspaces exist once the artefact representing it is written + if _, exist := mock.MockStates[req.StateId]; !exist { + // Ensure non-nil map + if mock.MockStates == nil { + mock.MockStates = make(map[string]interface{}) + } + + mock.MockStates[req.StateId] = req.Bytes + } + return providers.WriteStateBytesResponse{ + Diagnostics: nil, // success + } + } + mock.ReadStateBytesFn = func(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { + state := []byte{} + if v, exist := mock.MockStates[req.StateId]; exist { + if s, ok := v.([]byte); ok { + state = s + } + } + return providers.ReadStateBytesResponse{ + Bytes: state, + Diagnostics: nil, // success + } + } + return &mock +} diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 84ee85140bd1..8e0f113ab908 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2082,138 +2082,113 @@ func Test_determineInitReason(t *testing.T) { // Newly configured state store // Working directory has state_store in config but no preexisting backend state file -func TestMetaBackend_configureNewStateStore(t *testing.T) { - cases := map[string]struct { - // setup - isInitCommand bool - createDefaultWorkspace bool - // assertions - expectedError string - expectDefaultWorkspaceExists bool - }{ - "an init command creates the default workspace by default": { - createDefaultWorkspace: true, - isInitCommand: true, - expectDefaultWorkspaceExists: true, - }, - "an init command with the flag -create-default-workspace=false will not make the default workspace": { - createDefaultWorkspace: false, - isInitCommand: true, - expectDefaultWorkspaceExists: false, - }, - "during a non-init command, the command ends in with an error telling the user to run an init command": { - isInitCommand: false, - expectedError: "State store initialization required, please run \"terraform init\": Reason: Initial configuration of the requested state_store \"foo_bar\" in provider foo (\"registry.terraform.io/my-org/foo\")", - }, - } - - for tn, tc := range cases { - t.Run(tn, func(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-new"), td) - t.Chdir(td) - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true +// +// Tests that, during a non-init command, the command ends in with an error telling the user to run an init command +func TestMetaBackend_nonInitCommandInterrupted_uninitializedStateStore(t *testing.T) { - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-new"), td) + t.Chdir(td) - // Get mock provider factory to be used during init - // - // This imagines a provider called foo that contains - // a pluggable state store implementation called bar. - pssName := "test_store" - mock := testStateStoreMock(t) - factory := func() (providers.Interface, error) { - return mock, nil - } + // Setup the meta + m := testMetaBackend(t, nil) + m.AllowExperimentalFeatures = true - // Create locks - these would normally be the locks derived from config - locks := depsfile.NewLocks() - constraint, err := providerreqs.ParseVersionConstraints(">9.0.0") - if err != nil { - t.Fatalf("test setup failed when making constraint: %s", err) - } - expectedVersionString := "9.9.9" - expectedProviderSource := "registry.terraform.io/hashicorp/test" - locks.SetProvider( - addrs.MustParseProviderSourceString(expectedProviderSource), - versions.MustParseVersion(expectedVersionString), - constraint, - []providerreqs.Hash{"h1:foo"}, - ) + // Get the state store's config + mod, loadDiags := m.loadSingleModule(td) + if loadDiags.HasErrors() { + t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) + } - // Act - get the operations backend - b, beDiags := m.Backend(&BackendOpts{ - Init: tc.isInitCommand, // Changes with test case - StateStoreConfig: mod.StateStore, - ProviderFactory: factory, - Locks: locks, - CreateDefaultWorkspace: tc.createDefaultWorkspace, - }) - if beDiags.HasErrors() { - if tc.expectedError == "" { - t.Fatalf("unexpected error: %s", beDiags.Err()) - } - if !strings.Contains(cleanString(beDiags.Err().Error()), tc.expectedError) { - t.Fatalf("expected error to contain %s, but instead got: %s", tc.expectedError, cleanString(beDiags.Err().Error())) - } - return // error is as expected - } - if tc.expectedError != "" && !beDiags.HasErrors() { - t.Fatal("expected error missing") + // Get mock provider factory to be used during init + // + // This imagines a provider called foo that contains + // a pluggable state store implementation called bar. + pssName := "test_store" + mock := &testing_provider.MockProvider{ + GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ + Provider: providers.Schema{ + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "region": {Type: cty.String, Optional: true}, + }, + }, + }, + DataSources: map[string]providers.Schema{}, + ResourceTypes: map[string]providers.Schema{}, + ListResourceTypes: map[string]providers.Schema{}, + StateStores: map[string]providers.Schema{ + pssName: { + Body: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "value": { + Type: cty.String, + Required: true, + }, + }, + }, + }, + }, + }, + } + mock.WriteStateBytesFn = func(req providers.WriteStateBytesRequest) providers.WriteStateBytesResponse { + // Workspaces exist once the artefact representing it is written + if _, exist := mock.MockStates[req.StateId]; !exist { + // Ensure non-nil map + if mock.MockStates == nil { + mock.MockStates = make(map[string]interface{}) } - // Check the backend state file exists & assert its contents - s := testDataStateRead(t, filepath.Join(DefaultDataDir, backendLocal.DefaultStateFilename)) - if s == nil { - t.Fatal("expected backend state file to be created, but it was missing") + mock.MockStates[req.StateId] = req.Bytes + } + return providers.WriteStateBytesResponse{ + Diagnostics: nil, // success + } + } + mock.ReadStateBytesFn = func(req providers.ReadStateBytesRequest) providers.ReadStateBytesResponse { + state := []byte{} + if v, exist := mock.MockStates[req.StateId]; exist { + if s, ok := v.([]byte); ok { + state = s } + } + return providers.ReadStateBytesResponse{ + Bytes: state, + Diagnostics: nil, // success + } + } + factory := func() (providers.Interface, error) { + return mock, nil + } - if s.StateStore.Type != pssName { - t.Fatalf("backend state file contains unexpected state store type, want %q, got %q", pssName, s.StateStore.Type) - } - if s.StateStore.Provider.Version.String() != expectedVersionString { - t.Fatalf("backend state file contains unexpected version, want %q, got %q", expectedVersionString, s.StateStore.Provider.Version) - } - if s.StateStore.Provider.Source.String() != expectedProviderSource { - t.Fatalf("backend state file contains unexpected source, want %q, got %q", expectedProviderSource, s.StateStore.Provider.Source) - } - expectedProviderConfig := "{ \"region\": \"mars\" }" - expectedStoreConfig := "{ \"bar\": \"foobar\" }" - if cleanString(string(s.StateStore.Provider.ConfigRaw)) != expectedProviderConfig { - t.Fatalf("backend state file contains unexpected raw config data for the provider, want %q, got %q", expectedProviderConfig, cleanString(string(s.StateStore.Provider.ConfigRaw))) - } - if cleanString(string(s.StateStore.ConfigRaw)) != expectedStoreConfig { - t.Fatalf("backend state file contains unexpected raw config data for the state store, want %q, got %q", expectedStoreConfig, cleanString(string(s.StateStore.ConfigRaw))) - } + // Create locks - these would normally be the locks derived from config + locks := depsfile.NewLocks() + constraint, err := providerreqs.ParseVersionConstraints(">9.0.0") + if err != nil { + t.Fatalf("test setup failed when making constraint: %s", err) + } + expectedVersionString := "9.9.9" + expectedProviderSource := "registry.terraform.io/hashicorp/test" + locks.SetProvider( + addrs.MustParseProviderSourceString(expectedProviderSource), + versions.MustParseVersion(expectedVersionString), + constraint, + []providerreqs.Hash{"h1:foo"}, + ) - w, wDiags := b.Workspaces() - if wDiags.HasErrors() { - t.Fatalf("unexpected error: %s", wDiags.Err()) - } - if len(w) == 0 { - if tc.expectDefaultWorkspaceExists { - t.Fatal("expected the default workspace to exist, but there are no workspaces") - } - return - } - if len(w) > 0 { - if tc.expectDefaultWorkspaceExists { - if len(w) != 1 || w[0] != "default" { - t.Fatalf("expected only the default workspace to exist, but instead got: %v", w) - } - return // we've got the expected default workspace - } - - t.Fatalf("got unexpected workspaces: %v", w) - } - }) + // Act - get the operations backend + _, beDiags := m.Backend(&BackendOpts{ + Init: false, + StateStoreConfig: mod.StateStore, + ProviderFactory: factory, + Locks: locks, + }) + if !beDiags.HasErrors() { + t.Fatal("expected an error but got none") + } + expectedError := "State store initialization required, please run \"terraform init\": Reason: Initial configuration of the requested state_store \"test_store\" in provider test (\"registry.terraform.io/hashicorp/test\")" + if !strings.Contains(cleanString(beDiags.Err().Error()), expectedError) { + t.Fatalf("expected error to contain %s, but instead got: %s", expectedError, cleanString(beDiags.Err().Error())) } } diff --git a/internal/command/testdata/init-with-state-store/main.tf b/internal/command/testdata/init-with-state-store/main.tf index 9939e9dece2b..604feb44fed7 100644 --- a/internal/command/testdata/init-with-state-store/main.tf +++ b/internal/command/testdata/init-with-state-store/main.tf @@ -1,11 +1,11 @@ terraform { required_providers { - foo = { - source = "my-org/foo" + test = { + source = "hashicorp/test" } } - state_store "foo_foo" { - provider "foo" {} + state_store "test_store" { + provider "test" {} } } From 9bd81095eacc0a91ab06530cc1abeb05b9d7cb92 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 19 Sep 2025 17:56:53 +0100 Subject: [PATCH 34/54] Fix Get and Put impl. in remote_grpc --- internal/states/remote/remote_grpc.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/states/remote/remote_grpc.go b/internal/states/remote/remote_grpc.go index 7be226fd4fdc..2bd95e12f12a 100644 --- a/internal/states/remote/remote_grpc.go +++ b/internal/states/remote/remote_grpc.go @@ -53,7 +53,7 @@ type grpcClient struct { // and returns a copy of the downloaded state data. // // Implementation of remote.Client -func (g *grpcClient) Get() (*Payload, error) { +func (g *grpcClient) Get() (*Payload, tfdiags.Diagnostics) { // TODO - replace with method implementation added to main branch req := providers.ReadStateBytesRequest{ TypeName: g.typeName, @@ -63,21 +63,21 @@ func (g *grpcClient) Get() (*Payload, error) { if len(resp.Bytes) == 0 { // No state to return - return nil, resp.Diagnostics.Err() + return nil, resp.Diagnostics } payload := &Payload{ Data: resp.Bytes, MD5: []byte("foobar"), } - return payload, resp.Diagnostics.Err() + return payload, resp.Diagnostics } // Put invokes the WriteStateBytes gRPC method in the plugin protocol // and to transfer state data to the remote location. // // Implementation of remote.Client -func (g *grpcClient) Put(state []byte) error { +func (g *grpcClient) Put(state []byte) tfdiags.Diagnostics { // TODO - replace with method implementation added to main branch req := providers.WriteStateBytesRequest{ TypeName: g.typeName, @@ -85,7 +85,7 @@ func (g *grpcClient) Put(state []byte) error { } resp := g.provider.WriteStateBytes(req) - return resp.Diagnostics.Err() + return resp.Diagnostics } // Delete invokes the DeleteState gRPC method in the plugin protocol From bfaaddd4a87535af8475bf67eb093ed68ada7fd1 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 29 Aug 2025 18:56:08 +0100 Subject: [PATCH 35/54] Make Backend method return diagnostics, so warnings aren't swallowed. --- internal/command/meta_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 107b3b60fb03..2868c8b5a786 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -240,7 +240,7 @@ func (m *Meta) Backend(opts *BackendOpts) (backendrun.OperationsBackend, tfdiags } } - return local, nil + return local, diags } // selectWorkspace gets a list of existing workspaces and then checks From d1aaca0bc270406998f7abceeb15a9477c4a0e44 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 29 Aug 2025 18:56:13 +0100 Subject: [PATCH 36/54] Make test assert that warning about skipping workspace creation is included. --- internal/command/init_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/init_test.go b/internal/command/init_test.go index ec9a3c49b68a..f5d0fc0c1843 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -3317,7 +3317,7 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { // Check output output := testOutput.All() - expectedOutput := `Initializing the state store...` + expectedOutput := `Terraform has been configured to skip creation of the default workspace` if !strings.Contains(output, expectedOutput) { t.Fatalf("expected output to include %q, but got':\n %s", expectedOutput, output) } From 791071f2248781e60c93169781998e2384b23f93 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 29 Aug 2025 18:56:38 +0100 Subject: [PATCH 37/54] Update warning detail to include calls to action --- internal/command/meta_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 2868c8b5a786..7c71276b356f 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1682,7 +1682,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagWarning, Summary: "The default workspace does not exist", - Detail: "Terraform has been configured to skip creation of the default workspace in the state store.", + Detail: "Terraform has been configured to skip creation of the default workspace in the state store. To create it, either run an 'init' command without `-create-default-workspace=true`, or create it using a 'workspace new' command", }) } default: From 5575e7a5041bf9f7aab07efb7b757c0787bca6e3 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 5 Sep 2025 15:16:00 +0100 Subject: [PATCH 38/54] Fix use of hashes when initializing a new workingdir with a state store --- internal/command/meta_backend.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 7c71276b356f..5c87becf073a 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -649,11 +649,13 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di // Get the local 'backend' or 'state_store' configuration. var backendConfig *configs.Backend var stateStoreConfig *configs.StateStore - var cHash int + var cHash int // backend hash + var stateStoreHash int + var stateStoreProviderHash int if opts.StateStoreConfig != nil { // state store has been parsed from config and is included in opts var ssDiags tfdiags.Diagnostics - stateStoreConfig, cHash, _, ssDiags = m.stateStoreConfig(opts) + stateStoreConfig, stateStoreHash, stateStoreProviderHash, ssDiags = m.stateStoreConfig(opts) diags = diags.Append(ssDiags) if ssDiags.HasErrors() { return nil, diags @@ -820,7 +822,7 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di return nil, diags } - return m.stateStore_C_s(stateStoreConfig, cHash, sMgr, opts) + return m.stateStore_C_s(stateStoreConfig, stateStoreHash, stateStoreProviderHash, sMgr, opts) // Migration from state store to backend case backendConfig != nil && s.Backend.Empty() && @@ -1498,7 +1500,7 @@ func (m *Meta) backend_C_r_S_changed(c *configs.Backend, cHash int, backendSMgr //------------------------------------------------------------------- // Configuring a state_store for the first time. -func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { +func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, providerHash int, backendSMgr *clistate.LocalState, opts *BackendOpts) (backend.Backend, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics vt := arguments.ViewJSON @@ -1584,7 +1586,7 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli if m.stateLock { view := views.NewStateLocker(vt, m.View) stateLocker := clistate.NewLocker(m.stateLockTimeout, view) - if err := stateLocker.Lock(backendSMgr, "state_store from plan"); err != nil { + if err := stateLocker.Lock(backendSMgr, "init is initializing state_store first time"); err != nil { diags = diags.Append(fmt.Errorf("Error locking state: %s", err)) return nil, diags } @@ -1633,10 +1635,11 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, cHash int, backendSMgr *cli } s.StateStore = &workdir.StateStoreConfigState{ Type: c.Type, - Hash: uint64(cHash), + Hash: uint64(stateStoreHash), Provider: &workdir.ProviderConfigState{ Source: &c.ProviderAddr, Version: pVersion, + Hash: uint64(providerHash), }, } s.StateStore.SetConfig(storeConfigVal, b.ConfigSchema()) From 6a5fe6c81a7069d3d8ce27547fe93f4a2342f61b Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 5 Sep 2025 15:54:35 +0100 Subject: [PATCH 39/54] Update test to assert backend state file contents --- internal/command/init_test.go | 56 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/internal/command/init_test.go b/internal/command/init_test.go index f5d0fc0c1843..fc3f35271c94 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -17,12 +17,15 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/cli" version "github.com/hashicorp/go-version" + tfaddr "github.com/hashicorp/terraform-registry-address" "github.com/zclconf/go-cty/cty" "github.com/hashicorp/terraform/internal/addrs" "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/command/arguments" + "github.com/hashicorp/terraform/internal/command/clistate" "github.com/hashicorp/terraform/internal/command/views" + "github.com/hashicorp/terraform/internal/command/workdir" "github.com/hashicorp/terraform/internal/configs" "github.com/hashicorp/terraform/internal/configs/configschema" "github.com/hashicorp/terraform/internal/depsfile" @@ -3230,7 +3233,7 @@ func TestInit_testsWithModule(t *testing.T) { // Testing init's behaviors when run in an empty working directory func TestInit_stateStore_newWorkingDir(t *testing.T) { - t.Run("an init command creates the default workspace by default", func(t *testing.T) { + t.Run("the init command creates a backend state file, and creates the default workspace by default", func(t *testing.T) { // Create a temporary, uninitialized working directory with configuration including a state store td := t.TempDir() testCopyDir(t, testFixturePath("init-with-state-store"), td) @@ -3245,18 +3248,19 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { ui := new(cli.MockUi) view, done := testView(t) - c := &InitCommand{ - Meta: Meta{ - Ui: ui, - View: view, - AllowExperimentalFeatures: true, - testingOverrides: &testingOverrides{ - Providers: map[addrs.Provider]providers.Factory{ - mockProviderAddress: providers.FactoryFixed(mockProvider), - }, + meta := Meta{ + Ui: ui, + View: view, + AllowExperimentalFeatures: true, + testingOverrides: &testingOverrides{ + Providers: map[addrs.Provider]providers.Factory{ + mockProviderAddress: providers.FactoryFixed(mockProvider), }, - ProviderSource: providerSource, }, + ProviderSource: providerSource, + } + c := &InitCommand{ + Meta: meta, } args := []string{"-enable-pluggable-state-storage-experiment=true"} @@ -3277,6 +3281,36 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { if _, exists := mockProvider.MockStates[backend.DefaultStateName]; !exists { t.Fatal("expected the default workspace to be created during init, but it is missing") } + + // Assert contents of the backend state file + statePath := filepath.Join(meta.DataDir(), DefaultStateFilename) + sMgr := &clistate.LocalState{Path: statePath} + if err := sMgr.RefreshState(); err != nil { + t.Fatal("Failed to load state:", err) + } + s := sMgr.State() + if s == nil { + t.Fatal("expected backend state file to be created, but there isn't one") + } + v1_0_0, _ := version.NewVersion("1.0.0") + expectedState := &workdir.StateStoreConfigState{ + Type: "test_store", + ConfigRaw: []byte("{\n \"bar\": null\n }"), + Hash: uint64(3976463117), // Hash of empty config + Provider: &workdir.ProviderConfigState{ + Version: v1_0_0, + Source: &tfaddr.Provider{ + Hostname: tfaddr.DefaultProviderRegistryHost, + Namespace: "hashicorp", + Type: "test", + }, + ConfigRaw: []byte("{\n \"region\": null\n }"), + Hash: uint64(3976463117), // Hash of empty config + }, + } + if diff := cmp.Diff(s.StateStore, expectedState); diff != "" { + t.Fatalf("unexpected diff in backend state file's description of state store:\n%s", diff) + } }) t.Run("an init command with the flag -create-default-workspace=false will not make the default workspace", func(t *testing.T) { From dfe139c38b0b751c0cf0c984b1d899c758fc35c4 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 5 Sep 2025 17:57:25 +0100 Subject: [PATCH 40/54] Replace test for reconfiguring a state_store when the configuration has changed --- internal/command/init_test.go | 95 ++++++++++++++++++++++++++- internal/command/meta_backend_test.go | 53 --------------- 2 files changed, 94 insertions(+), 54 deletions(-) diff --git a/internal/command/init_test.go b/internal/command/init_test.go index fc3f35271c94..e60a61a462a0 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -3367,6 +3367,99 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { // > "during a non-init command, the command ends in with an error telling the user to run an init command" } +func TestInit_stateStore_configChanges(t *testing.T) { + t.Run("the -reconfigure flag makes Terraform ignore the backend state file during initialization", func(t *testing.T) { + // Create a temporary working directory with state store configuration + // that doesn't match the backend state file + td := t.TempDir() + testCopyDir(t, testFixturePath("state-store-reconfigure"), td) + t.Chdir(td) + + mockProvider := mockPluggableStateStorageProvider() + mockProviderAddress := addrs.NewDefaultProvider("test") + providerSource, close := newMockProviderSource(t, map[string][]string{ + "hashicorp/test": {"1.0.0"}, + }) + defer close() + + ui := new(cli.MockUi) + view, done := testView(t) + meta := Meta{ + Ui: ui, + View: view, + AllowExperimentalFeatures: true, + testingOverrides: &testingOverrides{ + Providers: map[addrs.Provider]providers.Factory{ + mockProviderAddress: providers.FactoryFixed(mockProvider), + }, + }, + ProviderSource: providerSource, + } + c := &InitCommand{ + Meta: meta, + } + + args := []string{ + "-enable-pluggable-state-storage-experiment=true", + "-reconfigure", + } + code := c.Run(args) + testOutput := done(t) + if code != 0 { + t.Fatalf("expected code 0 exit code, got %d, output: \n%s", code, testOutput.All()) + } + + // Check output + output := testOutput.All() + expectedOutputs := []string{ + "Initializing the state store...", + "Terraform has been successfully initialized!", + } + for _, expected := range expectedOutputs { + if !strings.Contains(output, expected) { + t.Fatalf("expected output to include %q, but got':\n %s", expected, output) + } + } + + // Assert the default workspace was created + if _, exists := mockProvider.MockStates[backend.DefaultStateName]; !exists { + t.Fatal("expected the default workspace to be created during init, but it is missing") + } + + // Assert contents of the backend state file + statePath := filepath.Join(meta.DataDir(), DefaultStateFilename) + sMgr := &clistate.LocalState{Path: statePath} + if err := sMgr.RefreshState(); err != nil { + t.Fatal("Failed to load state:", err) + } + s := sMgr.State() + if s == nil { + t.Fatal("expected backend state file to be created, but there isn't one") + } + v1_0_0, _ := version.NewVersion("1.0.0") + expectedState := &workdir.StateStoreConfigState{ + Type: "test_store", + ConfigRaw: []byte("{\n \"value\": \"changed-value\"\n }"), + Hash: uint64(1417640992), // Hash affected by config + Provider: &workdir.ProviderConfigState{ + Version: v1_0_0, + Source: &tfaddr.Provider{ + Hostname: tfaddr.DefaultProviderRegistryHost, + Namespace: "hashicorp", + Type: "test", + }, + ConfigRaw: []byte("{\n \"region\": null\n }"), + Hash: uint64(3976463117), // Hash of empty config + }, + } + if diff := cmp.Diff(s.StateStore, expectedState); diff != "" { + t.Fatalf("unexpected diff in backend state file's description of state store:\n%s", diff) + } + }) + + // TODO - add more test cases, e.g. these context changes in the context of updating providers +} + // newMockProviderSource is a helper to succinctly construct a mock provider // source that contains a set of packages matching the given provider versions // that are available for installation (from temporary local files). @@ -3531,7 +3624,7 @@ func mockPluggableStateStorageProvider() *testing_provider.MockProvider { pssName: { Body: &configschema.Block{ Attributes: map[string]*configschema.Attribute{ - "bar": { + "value": { Type: cty.String, Required: true, }, diff --git a/internal/command/meta_backend_test.go b/internal/command/meta_backend_test.go index 8e0f113ab908..799d7a7faa29 100644 --- a/internal/command/meta_backend_test.go +++ b/internal/command/meta_backend_test.go @@ -2230,59 +2230,6 @@ func TestMetaBackend_configuredStateStoreUnset(t *testing.T) { } } -// Reconfiguring with an already configured state store. -// This should ignore the existing state_store config, and configure the new -// state store is if this is the first time. -// -// TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch -// case for this scenario, and will need to be updated when that init feature is implemented. -func TestMetaBackend_reconfigureStateStoreChange(t *testing.T) { - td := t.TempDir() - testCopyDir(t, testFixturePath("state-store-reconfigure"), td) - t.Chdir(td) - - // Setup the meta - m := testMetaBackend(t, nil) - m.AllowExperimentalFeatures = true - - // this should not ask for input - m.input = false - - // cli flag -reconfigure - m.reconfigure = true - - // Get the state store's config - mod, loadDiags := m.loadSingleModule(td) - if loadDiags.HasErrors() { - t.Fatalf("unexpected error when loading test config: %s", loadDiags.Err()) - } - - // Get mock provider factory to be used during init - // - // This imagines a provider called foo that contains - // a pluggable state store implementation called bar. - mock := testStateStoreMock(t) - factory := func() (providers.Interface, error) { - return mock, nil - } - - // Get the operations backend - _, beDiags := m.Backend(&BackendOpts{ - Init: true, - StateStoreConfig: mod.StateStore, - ProviderFactory: factory, - }) - - if !beDiags.HasErrors() { - t.Fatal("expected an error to be returned during partial implementation of PSS") - } - wantErr := "Configuring a state store for the first time is not implemented yet" - if !strings.Contains(beDiags.Err().Error(), wantErr) { - t.Fatalf("expected the returned error to contain %q, but got: %s", wantErr, beDiags.Err()) - } - -} - // Changing a configured state store // // TODO(SarahFrench/radeksimko): currently this test only confirms that we're hitting the switch From 6e92e42dd2c4fe4458e46e7e9489a3ddc5ecfa2f Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 5 Sep 2025 17:57:36 +0100 Subject: [PATCH 41/54] Fix test, add more assertions about output --- internal/command/init_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/command/init_test.go b/internal/command/init_test.go index e60a61a462a0..cc8aa1a8119b 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -3272,9 +3272,14 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { // Check output output := testOutput.All() - expectedOutput := `Initializing the state store...` - if !strings.Contains(output, expectedOutput) { - t.Fatalf("expected output to include %q, but got':\n %s", expectedOutput, output) + expectedOutputs := []string{ + "Initializing the state store...", + "Terraform has been successfully initialized!", + } + for _, expected := range expectedOutputs { + if !strings.Contains(output, expected) { + t.Fatalf("expected output to include %q, but got':\n %s", expected, output) + } } // Assert the default workspace was created @@ -3295,7 +3300,7 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { v1_0_0, _ := version.NewVersion("1.0.0") expectedState := &workdir.StateStoreConfigState{ Type: "test_store", - ConfigRaw: []byte("{\n \"bar\": null\n }"), + ConfigRaw: []byte("{\n \"value\": null\n }"), Hash: uint64(3976463117), // Hash of empty config Provider: &workdir.ProviderConfigState{ Version: v1_0_0, From 3e435831f6f720bbde3ab58d81d2de82d5ee89d7 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 8 Sep 2025 10:05:10 +0100 Subject: [PATCH 42/54] Update comments on tests --- internal/command/init_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/command/init_test.go b/internal/command/init_test.go index cc8aa1a8119b..c3911a18621b 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -3372,6 +3372,8 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { // > "during a non-init command, the command ends in with an error telling the user to run an init command" } +// Testing init's behaviors when run in a working directory where the configuration +// doesn't match the backend state file. func TestInit_stateStore_configChanges(t *testing.T) { t.Run("the -reconfigure flag makes Terraform ignore the backend state file during initialization", func(t *testing.T) { // Create a temporary working directory with state store configuration From 7b9af2caed01c533a2fbdef7a16321ea2a0ebc96 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 15 Sep 2025 17:12:34 +0100 Subject: [PATCH 43/54] Fix diagnostic using an error unnecessarily --- internal/command/meta_backend.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 5c87becf073a..32130ac9a968 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1615,11 +1615,10 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, provide } else { pLock := opts.Locks.Provider(c.ProviderAddr) if pLock == nil { - diags = diags.Append(fmt.Errorf("The provider %s (%q) is not present in the lockfile, despite being used for state store %q. This is a bug in Terraform and should be reported", + diags = diags.Append(fmt.Errorf("The provider %s (%q) is not present in the lockfile, despite being used for state store %q. This is a bug in Terraform and should be reported.", c.Provider.Name, c.ProviderAddr, - c.Type, - )) + c.Type)) return nil, diags } var err error From 40030d4cf048c7df50e2f5cdcd4b258545849f3f Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 15 Sep 2025 20:34:48 +0100 Subject: [PATCH 44/54] Fix - make sure all errors are handled, and avoid nil error causing a panic --- internal/command/meta_backend.go | 77 +++++++++++++++++--------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 32130ac9a968..31915332d045 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -1651,45 +1651,50 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, provide // Verify that selected workspace exists in the state store. if opts.Init && b != nil { err := m.selectWorkspace(b) - if strings.Contains(err.Error(), "No existing workspaces") { - // If there are no workspaces, Terraform either needs to create the default workspace here, - // or instruct the user to run a `terraform workspace new` command. - ws, err := m.Workspace() - if err != nil { - diags = diags.Append(fmt.Errorf("Failed to check current workspace: %w", err)) - return nil, diags - } + if err != nil { + if strings.Contains(err.Error(), "No existing workspaces") { + // If there are no workspaces, Terraform either needs to create the default workspace here, + // or instruct the user to run a `terraform workspace new` command. + ws, err := m.Workspace() + if err != nil { + diags = diags.Append(fmt.Errorf("Failed to check current workspace: %w", err)) + return nil, diags + } - switch { - case ws != backend.DefaultStateName: - // User needs to run a `terraform workspace new` command. - diags = append(diags, tfdiags.Sourceless( - tfdiags.Error, - fmt.Sprintf("Workspace %q has not been created yet", ws), - fmt.Sprintf("State store %q in provider %s (%q) reports that no workspaces currently exist. To create the custom workspace %q use the command `terraform workspace new %s`.", - c.Type, - c.Provider.Name, - c.ProviderAddr, - ws, - ws, - ), - )) - return nil, diags + switch { + case ws != backend.DefaultStateName: + // User needs to run a `terraform workspace new` command. + diags = append(diags, tfdiags.Sourceless( + tfdiags.Error, + fmt.Sprintf("Workspace %q has not been created yet", ws), + fmt.Sprintf("State store %q in provider %s (%q) reports that no workspaces currently exist. To create the custom workspace %q use the command `terraform workspace new %s`.", + c.Type, + c.Provider.Name, + c.ProviderAddr, + ws, + ws, + ), + )) + return nil, diags - case ws == backend.DefaultStateName: - // Users control if the default workspace is created through the -create-default-workspace flag (defaults to true) - if opts.CreateDefaultWorkspace { - diags = diags.Append(m.createDefaultWorkspace(c, b)) - } else { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagWarning, - Summary: "The default workspace does not exist", - Detail: "Terraform has been configured to skip creation of the default workspace in the state store. To create it, either run an 'init' command without `-create-default-workspace=true`, or create it using a 'workspace new' command", - }) + case ws == backend.DefaultStateName: + // Users control if the default workspace is created through the -create-default-workspace flag (defaults to true) + if opts.CreateDefaultWorkspace { + diags = diags.Append(m.createDefaultWorkspace(c, b)) + } else { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "The default workspace does not exist", + Detail: "Terraform has been configured to skip creation of the default workspace in the state store. To create it, either run an 'init' command without `-create-default-workspace=true`, or create it using a 'workspace new' command", + }) + } + default: + diags = diags.Append(err) + return nil, diags } - default: - diags = diags.Append(err) - return nil, diags + } else { + // For all other errors, report via diagnostics + diags = diags.Append(fmt.Errorf("Failed to select a workspace: %w", err)) } } } From 77e5466d3b94aed2bbece9145f352bdeabcb0957 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 15 Sep 2025 20:35:40 +0100 Subject: [PATCH 45/54] Allow init commands to succeed if using PSS and a reattached provider. --- internal/command/meta_backend.go | 85 +++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 18 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 31915332d045..5aaed8775190 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -14,6 +14,7 @@ import ( "fmt" "log" "maps" + "os" "path/filepath" "slices" "strconv" @@ -1604,33 +1605,58 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, provide // If we're handling the builtin "terraform" provider then there's no version information to store in the dependency lock file, so don't access it. // We must record a value into the backend state file, and we cannot include a value that changes (e.g. the Terraform core binary version) as migration // is impossible with builtin providers. - // So, we use a hardcoded version number of 42. - var err error - pVersion, err = version.NewVersion("0.42.0") + // So, we use an arbitrary stand-in version. + standInVersion, err := version.NewVersion("0.0.1") if err != nil { - diags = diags.Append(fmt.Errorf("Error when creating a backend state file containing a builtin provider. This is a bug in Terraform and should be reported: %w", + diags = diags.Append(fmt.Errorf("Error when creating a backend state file. This is a bug in Terraform and should be reported: %w", err)) return nil, diags } + pVersion = standInVersion } else { - pLock := opts.Locks.Provider(c.ProviderAddr) - if pLock == nil { - diags = diags.Append(fmt.Errorf("The provider %s (%q) is not present in the lockfile, despite being used for state store %q. This is a bug in Terraform and should be reported.", - c.Provider.Name, - c.ProviderAddr, - c.Type)) - return nil, diags - } - var err error - pVersion, err = providerreqs.GoVersionFromVersion(pLock.Version()) + isReattached, err := isProviderReattached(c.ProviderAddr) if err != nil { - diags = diags.Append(fmt.Errorf("Failed obtain the in-use version of provider %s (%q) when recording backend state for state store %q. This is a bug in Terraform and should be reported: %w", - c.Provider.Name, - c.ProviderAddr, - c.Type, + diags = diags.Append(fmt.Errorf("Error determining if the state storage provider is reattached or not. This is a bug in Terraform and should be reported: %w", err)) return nil, diags } + if isReattached { + // If the provider is unmanaged then it won't be in the locks. + // If there are no locks then there's no version information to for us to access and use when creating the backend state file. + // So, we use an arbitrary stand-in version. + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "State storage provider is not managed by Terraform", + Detail: "Terraform is using a provider supplied via TF_REATTACH_PROVIDERS for initializing state storage. This will affect Terraform's ability to detect when state migrations are required.", + }) + standInVersion, err := version.NewVersion("0.0.1") + if err != nil { + diags = diags.Append(fmt.Errorf("Error when creating a backend state file. This is a bug in Terraform and should be reported: %w", + err)) + return nil, diags + } + pVersion = standInVersion + } else { + // The provider is not built in and is being managed by Terraform + // This is the most common scenario, by far. + pLock := opts.Locks.Provider(c.ProviderAddr) + if pLock == nil { + diags = diags.Append(fmt.Errorf("The provider %s (%q) is not present in the lockfile, despite being used for state store %q. This is a bug in Terraform and should be reported.", + c.Provider.Name, + c.ProviderAddr, + c.Type)) + return nil, diags + } + pVersion, err = providerreqs.GoVersionFromVersion(pLock.Version()) + if err != nil { + diags = diags.Append(fmt.Errorf("Failed obtain the in-use version of provider %s (%q) when recording backend state for state store %q. This is a bug in Terraform and should be reported: %w", + c.Provider.Name, + c.ProviderAddr, + c.Type, + err)) + return nil, diags + } + } } s.StateStore = &workdir.StateStoreConfigState{ Type: c.Type, @@ -1715,6 +1741,29 @@ func (m *Meta) stateStore_C_s(c *configs.StateStore, stateStoreHash int, provide return b, diags } +// isProviderReattached determines if a given provider is being supplied to Terraform via the TF_REATTACH_PROVIDERS +// environment variable. +func isProviderReattached(provider addrs.Provider) (bool, error) { + in := os.Getenv("TF_REATTACH_PROVIDERS") + if in != "" { + var m map[string]any + err := json.Unmarshal([]byte(in), &m) + if err != nil { + return false, fmt.Errorf("Invalid format for TF_REATTACH_PROVIDERS: %w", err) + } + for p, _ := range m { + a, diags := addrs.ParseProviderSourceString(p) + if diags.HasErrors() { + return false, fmt.Errorf("Error parsing %q as a provider address: %w", a, diags.Err()) + } + if a.Equals(provider) { + return true, nil + } + } + } + return false, nil +} + // createDefaultWorkspace receives a backend made using a pluggable state store, and details about that store's config, // and persists an empty state file in the default workspace. By creating this artifact we ensure that the default // workspace is created and usable by Terraform in later operations. From f7ad3ec3441b452d1542a77b52aafd9cb43b6236 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Tue, 16 Sep 2025 12:17:36 +0100 Subject: [PATCH 46/54] Fix: Pass state bytes into WriteStateBytesRequest Otherwise the provider never receives any state data to write! --- internal/states/remote/remote_grpc.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/states/remote/remote_grpc.go b/internal/states/remote/remote_grpc.go index 2bd95e12f12a..4c9f9c1b2cc5 100644 --- a/internal/states/remote/remote_grpc.go +++ b/internal/states/remote/remote_grpc.go @@ -82,6 +82,7 @@ func (g *grpcClient) Put(state []byte) tfdiags.Diagnostics { req := providers.WriteStateBytesRequest{ TypeName: g.typeName, StateId: g.stateId, + Bytes: state, } resp := g.provider.WriteStateBytes(req) From ac42ff1c8295bcbbfcf5f293d404ca6bd6d08b7b Mon Sep 17 00:00:00 2001 From: Sarah French Date: Fri, 19 Sep 2025 12:41:52 +0100 Subject: [PATCH 47/54] Make error message more specific --- internal/command/meta_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 5aaed8775190..6d3a680ab22e 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -695,7 +695,7 @@ func (m *Meta) backendFromConfig(opts *BackendOpts) (backend.Backend, tfdiags.Di statePath := filepath.Join(m.DataDir(), DefaultStateFilename) sMgr := &clistate.LocalState{Path: statePath} if err := sMgr.RefreshState(); err != nil { - diags = diags.Append(fmt.Errorf("Failed to load state: %s", err)) + diags = diags.Append(fmt.Errorf("Failed to load the backend state file: %s", err)) return nil, diags } From 8b11c1d9225773c0cd965a467db8953378ed1c4c Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 22 Sep 2025 18:40:01 +0100 Subject: [PATCH 48/54] Add test for getStateStoreProviderFactory --- internal/command/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/init.go b/internal/command/init.go index ed41e180cbbc..f788818c36a8 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -247,7 +247,7 @@ func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, ext opts = &BackendOpts{ StateStoreConfig: root.StateStore, - Locks: locks, + Locks: configLocks, ProviderFactory: factory, CreateDefaultWorkspace: initArgs.CreateDefaultWorkspace, ConfigOverride: configOverride, From 0ca84fcdd909d5541e36c0602c1d3a7591ae3b70 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Tue, 23 Sep 2025 15:25:35 +0100 Subject: [PATCH 49/54] Implement configurable state chunk size --- internal/command/meta_backend.go | 35 ++++++++++++++++++++++++++++++- internal/plugin6/grpc_provider.go | 1 + 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 6d3a680ab22e..2a446288a854 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -49,6 +49,20 @@ import ( tfversion "github.com/hashicorp/terraform/version" ) +const ( + // defaultStateStoreChunkSize is the default chunk size proposed + // to the provider. + // This can be tweaked but should provide reasonable performance + // trade-offs for average network conditions and state file sizes. + defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB + + // maxStateStoreChunkSize is the highest chunk size provider may choose + // which we still consider reasonable/safe. + // This reflects terraform-plugin-go's max. RPC message size of 256MB + // and leaves plenty of space for other variable data like diagnostics. + maxStateStoreChunkSize int64 = 128 << 20 // 128 MB +) + // BackendOpts are the options used to initialize a backendrun.OperationsBackend. type BackendOpts struct { // BackendConfig is a representation of the backend configuration block given in @@ -2127,7 +2141,6 @@ func (m *Meta) stateStoreInitFromConfig(c *configs.StateStore, opts *BackendOpts configureResp := provider.ConfigureProvider(providers.ConfigureProviderRequest{ TerraformVersion: tfversion.String(), Config: providerConfigVal, - // TODO ClientCapabilities? }) diags = diags.Append(configureResp.Diagnostics) if configureResp.Diagnostics.HasErrors() { @@ -2149,12 +2162,32 @@ func (m *Meta) stateStoreInitFromConfig(c *configs.StateStore, opts *BackendOpts cfgStoreResp := provider.ConfigureStateStore(providers.ConfigureStateStoreRequest{ TypeName: c.Type, Config: stateStoreConfigVal, + Capabilities: providers.StateStoreClientCapabilities{ + ChunkSize: defaultStateStoreChunkSize, + }, }) diags = diags.Append(cfgStoreResp.Diagnostics) if cfgStoreResp.Diagnostics.HasErrors() { return nil, cty.NilVal, cty.NilVal, diags } + chunkSize := cfgStoreResp.Capabilities.ChunkSize + if chunkSize > maxStateStoreChunkSize { + diags = diags.Append(fmt.Errorf("Failed to negotiate acceptable chunk size. "+ + "Expected size <= %d bytes, provider wants %d bytes", + maxStateStoreChunkSize, chunkSize, + )) + return nil, cty.NilVal, cty.NilVal, diags + } + + p, ok := provider.(providers.StateStoreChunkSizeSetter) + if !ok { + msg := fmt.Sprintf("Unable to set chunk size for provider %s; this is a bug in Terraform - please report it", c.Type) + panic(msg) + } + // casting to int here is okay because the number should never exceed int32 + p.SetStateStoreChunkSize(c.Type, int(chunkSize)) + // Now we have a fully configured state store, ready to be used. // To make it usable we need to return it in a backend.Backend interface. b, err := backendPluggable.NewPluggable(provider, c.Type) diff --git a/internal/plugin6/grpc_provider.go b/internal/plugin6/grpc_provider.go index a475ec66549a..82f1b33b4d38 100644 --- a/internal/plugin6/grpc_provider.go +++ b/internal/plugin6/grpc_provider.go @@ -1562,6 +1562,7 @@ func (p *GRPCProvider) ReadStateBytes(r providers.ReadStateBytesRequest) (resp p buf := &bytes.Buffer{} var expectedTotalLength int + // TODO: Send warning if client misbehaves and uses (lower) chunk size that we didn't agree on for { chunk, err := client.Recv() if err == io.EOF { From edffe4c93e8c230bfde75286e0d7cf5a855a1178 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 13 Aug 2025 15:50:16 +0200 Subject: [PATCH 50/54] Implement ReadStateBytes + WriteStateBytes --- internal/providers/provider.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/internal/providers/provider.go b/internal/providers/provider.go index 83036bfd3b07..2397063738e4 100644 --- a/internal/providers/provider.go +++ b/internal/providers/provider.go @@ -903,6 +903,34 @@ type WriteStateBytesResponse struct { Diagnostics tfdiags.Diagnostics } +type ReadStateBytesRequest struct { + // TypeName is the name of the state store to read state from + TypeName string + // StateId is the ID of a state file to read + StateId string +} + +type ReadStateBytesResponse struct { + // Bytes represents all received bytes of the given state file + Bytes []byte + // Diagnostics contains any warnings or errors from the method call. + Diagnostics tfdiags.Diagnostics +} + +type WriteStateBytesRequest struct { + // TypeName is the name of the state store to write state to + TypeName string + // Bytes represents all bytes of the given state file to write + Bytes []byte + // StateId is the ID of a state file to write + StateId string +} + +type WriteStateBytesResponse struct { + // Diagnostics contains any warnings or errors from the method call. + Diagnostics tfdiags.Diagnostics +} + type GetStatesRequest struct { // TypeName is the name of the state store to request the list of states from TypeName string From 5257853059b3c1302c4743b6f5f3b15354143ed6 Mon Sep 17 00:00:00 2001 From: Sarah French <15078782+SarahFrench@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:31:24 +0100 Subject: [PATCH 51/54] Update proto file definition of Read/WriteStateBytes RPCs (#37529) * Update Read/WriteStateBytes RPCs to match https://github.com/hashicorp/terraform-plugin-go/pull/531 * Run `make protobuf` * Run `make generate` * Update use of `proto.ReadStateBytes_ResponseChunk` in tests * Fix how diagnostics are handled alongside EOF error, update ReadStateBytes test * More fixes - test setup was incorrect I think? I assume that a response would be returned full of zero-values when EOF is encountered. --- internal/providers/provider.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/internal/providers/provider.go b/internal/providers/provider.go index 2397063738e4..83036bfd3b07 100644 --- a/internal/providers/provider.go +++ b/internal/providers/provider.go @@ -903,34 +903,6 @@ type WriteStateBytesResponse struct { Diagnostics tfdiags.Diagnostics } -type ReadStateBytesRequest struct { - // TypeName is the name of the state store to read state from - TypeName string - // StateId is the ID of a state file to read - StateId string -} - -type ReadStateBytesResponse struct { - // Bytes represents all received bytes of the given state file - Bytes []byte - // Diagnostics contains any warnings or errors from the method call. - Diagnostics tfdiags.Diagnostics -} - -type WriteStateBytesRequest struct { - // TypeName is the name of the state store to write state to - TypeName string - // Bytes represents all bytes of the given state file to write - Bytes []byte - // StateId is the ID of a state file to write - StateId string -} - -type WriteStateBytesResponse struct { - // Diagnostics contains any warnings or errors from the method call. - Diagnostics tfdiags.Diagnostics -} - type GetStatesRequest struct { // TypeName is the name of the state store to request the list of states from TypeName string From 4d39c04aa903ea860cc1869a8c5d576bdb654c99 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Tue, 23 Sep 2025 15:25:35 +0100 Subject: [PATCH 52/54] Implement configurable state chunk size --- internal/command/meta_backend.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 2a446288a854..1a315e58371d 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -63,6 +63,20 @@ const ( maxStateStoreChunkSize int64 = 128 << 20 // 128 MB ) +const ( + // defaultStateStoreChunkSize is the default chunk size proposed + // to the provider. + // This can be tweaked but should provide reasonable performance + // trade-offs for average network conditions and state file sizes. + defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB + + // maxStateStoreChunkSize is the highest chunk size provider may choose + // which we still consider reasonable/safe. + // This reflects terraform-plugin-go's max. RPC message size of 256MB + // and leaves plenty of space for other variable data like diagnostics. + maxStateStoreChunkSize int64 = 128 << 20 // 128 MB +) + // BackendOpts are the options used to initialize a backendrun.OperationsBackend. type BackendOpts struct { // BackendConfig is a representation of the backend configuration block given in From 89f809a3fd60b777f7418318b36cb88a658eef14 Mon Sep 17 00:00:00 2001 From: Sarah French Date: Thu, 25 Sep 2025 16:19:16 +0100 Subject: [PATCH 53/54] Fix repeat declarations --- internal/command/meta_backend.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/command/meta_backend.go b/internal/command/meta_backend.go index 1a315e58371d..2a446288a854 100644 --- a/internal/command/meta_backend.go +++ b/internal/command/meta_backend.go @@ -63,20 +63,6 @@ const ( maxStateStoreChunkSize int64 = 128 << 20 // 128 MB ) -const ( - // defaultStateStoreChunkSize is the default chunk size proposed - // to the provider. - // This can be tweaked but should provide reasonable performance - // trade-offs for average network conditions and state file sizes. - defaultStateStoreChunkSize int64 = 8 << 20 // 8 MB - - // maxStateStoreChunkSize is the highest chunk size provider may choose - // which we still consider reasonable/safe. - // This reflects terraform-plugin-go's max. RPC message size of 256MB - // and leaves plenty of space for other variable data like diagnostics. - maxStateStoreChunkSize int64 = 128 << 20 // 128 MB -) - // BackendOpts are the options used to initialize a backendrun.OperationsBackend. type BackendOpts struct { // BackendConfig is a representation of the backend configuration block given in From 24a44d749275f10a13eba3c7d6980822ef8cf65a Mon Sep 17 00:00:00 2001 From: Sarah French Date: Mon, 29 Sep 2025 16:31:11 +0100 Subject: [PATCH 54/54] Post-rebase fix --- internal/command/init.go | 5 ++++- internal/command/init_run.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/command/init.go b/internal/command/init.go index f788818c36a8..cbf68b969403 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -159,7 +159,10 @@ func (c *InitCommand) initCloud(ctx context.Context, root *configs.Module, extra return back, true, diags } -func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, extraConfig arguments.FlagNameValueSlice, viewType arguments.ViewType, configLocks *depsfile.Locks, view views.Init) (be backend.Backend, output bool, diags tfdiags.Diagnostics) { +func (c *InitCommand) initBackend(ctx context.Context, root *configs.Module, initArgs *arguments.Init, configLocks *depsfile.Locks, view views.Init) (be backend.Backend, output bool, diags tfdiags.Diagnostics) { + extraConfig := initArgs.BackendConfig + viewType := initArgs.ViewType + ctx, span := tracer.Start(ctx, "initialize backend") _ = ctx // prevent staticcheck from complaining to avoid a maintenance hazard of having the wrong ctx in scope here defer span.End() diff --git a/internal/command/init_run.go b/internal/command/init_run.go index e2f4044dcad2..8fb35aea44c3 100644 --- a/internal/command/init_run.go +++ b/internal/command/init_run.go @@ -174,7 +174,7 @@ func (c *InitCommand) run(initArgs *arguments.Init, view views.Init) int { // initBackend has new parameters that aren't relevant to the original (unpluggable) version of the init command logic here. // So for this version of the init command, we pass in empty locks intentionally. emptyLocks := depsfile.NewLocks() - back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs.BackendConfig, initArgs.ViewType, emptyLocks, view) + back, backendOutput, backDiags = c.initBackend(ctx, rootModEarly, initArgs, emptyLocks, view) default: // load the previously-stored backend config back, backDiags = c.Meta.backendFromState(ctx)