diff --git a/src/SIL.XForge.Scripture/Services/MachineProjectService.cs b/src/SIL.XForge.Scripture/Services/MachineProjectService.cs index 13ba79b4770..3f234e990e7 100644 --- a/src/SIL.XForge.Scripture/Services/MachineProjectService.cs +++ b/src/SIL.XForge.Scripture/Services/MachineProjectService.cs @@ -8,7 +8,9 @@ using System.Threading; using System.Threading.Tasks; using Autofac.Extras.DynamicProxy; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json.Linq; @@ -36,6 +38,7 @@ namespace SIL.XForge.Scripture.Services; public class MachineProjectService( ICorporaClient corporaClient, IDataFilesClient dataFilesClient, + IWebHostEnvironment env, IExceptionHandler exceptionHandler, IFileSystemService fileSystemService, ILogger logger, @@ -54,6 +57,12 @@ IRepository userSecrets internal const string Nmt = "nmt"; internal const string SmtTransfer = "smt-transfer"; + // These tags are used for the ClearML task to differentiate Scripture Forge jobs + internal const string TagDevelopment = "sf-dev"; + internal const string TagTest = "sf-test"; + internal const string TagStaging = "sf-qa"; + internal const string TagProduction = "sf-live"; + /// /// Adds the SMT project to Serval, if the required data is present. /// @@ -588,7 +597,7 @@ await RecreateOrUpdateTranslationEngineIfRequiredAsync( else { translationEngineId = projectSecret.ServalData.TranslationEngineId!; - translationBuildConfig = new TranslationBuildConfig(); + translationBuildConfig = new TranslationBuildConfig { Options = AddTags() }; } // Start the build @@ -1112,19 +1121,14 @@ IList corporaSyncInfo ) { // Load the Serval Config from the Draft Config - JObject? options = null; - if (!string.IsNullOrWhiteSpace(servalConfig)) - { - options = JObject.Parse(servalConfig); - } + JObject? options = !string.IsNullOrWhiteSpace(servalConfig) ? JObject.Parse(servalConfig) : []; + + // Add the tags + AddTags(options); // If Fast Training is enabled, override the max_steps if (buildConfig.FastTraining) { - // Ensure that there is a servalConfig JSON object - options ??= []; - - // 20 is the number of steps used on Serval QA by default options["max_steps"] = 20; } @@ -2129,6 +2133,54 @@ CancellationToken cancellationToken return true; } + /// + /// Adds the tags to the build_options + /// + /// The build options. + /// The build options + private JObject AddTags(JObject? options = null) + { + // Generate the tag based on the environment + string tag; + if (env.EnvironmentName == Environments.Production) + { + tag = TagProduction; + } + else if (env.EnvironmentName == Environments.Staging) + { + tag = TagStaging; + } + else if (env.EnvironmentName == Environments.Development) + { + tag = TagDevelopment; + } + else + { + tag = TagTest; + } + + // Set the tag in the options, taking into account potential existing values + options ??= []; + if (options["tags"] == null) + { + options["tags"] = tag; + } + else if (options["tags"].Type == JTokenType.String && options["tags"].ToString() != tag) + { + options["tags"] = new JArray(options["tags"].ToString(), tag); + } + else if (options["tags"].Type == JTokenType.Array) + { + JArray tags = (JArray)options["tags"]; + if (!tags.Any(t => t.Type == JTokenType.String && t.ToString() == tag)) + { + tags.Add(tag); + } + } + + return options; + } + /// /// Gets the translation engine identifier from the project secret, /// depending on whether we are pre-translating or not. diff --git a/test/SIL.XForge.Scripture.Tests/Services/MachineProjectServiceTests.cs b/test/SIL.XForge.Scripture.Tests/Services/MachineProjectServiceTests.cs index 9975246aff7..29197374314 100644 --- a/test/SIL.XForge.Scripture.Tests/Services/MachineProjectServiceTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Services/MachineProjectServiceTests.cs @@ -6,6 +6,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json.Linq; @@ -1470,6 +1472,104 @@ public void GetTargetLanguage_Success() Assert.AreEqual(targetWritingSystemTag, actual); } + [Test] + public void GetTranslationBuildConfig_AddsTagForDev() + { + // Set up test environment + var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Development } }; + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig: null, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual(MachineProjectService.TagDevelopment, (string)(actual.Options as JObject)?["tags"]); + } + + [Test] + public void GetTranslationBuildConfig_AddsTagForTest() + { + // Set up test environment + var env = new TestEnvironment(); + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig: null, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual(MachineProjectService.TagTest, (string)(actual.Options as JObject)?["tags"]); + } + + [Test] + public void GetTranslationBuildConfig_AddsTagForStaging() + { + // Set up test environment + var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Staging } }; + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig: null, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual(MachineProjectService.TagStaging, (string)(actual.Options as JObject)?["tags"]); + } + + [Test] + public void GetTranslationBuildConfig_AddsTagForProduction() + { + // Set up test environment + var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } }; + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig: null, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual(MachineProjectService.TagProduction, (string)(actual.Options as JObject)?["tags"]); + } + + [Test] + public void GetTranslationBuildConfig_DoesNotAddTagIfAlreadyInServalConfigAsArray() + { + // Set up test environment + var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } }; + const string servalConfig = $$"""{"tags":["{{MachineProjectService.TagProduction}}"]}"""; + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual(new JArray(MachineProjectService.TagProduction), (JArray)(actual.Options as JObject)?["tags"]); + } + + [Test] + public void GetTranslationBuildConfig_DoesNotReplaceTagIfAlreadyInServalConfigAsString() + { + // Set up test environment + var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } }; + const string servalConfig = $$"""{"tags":"{{MachineProjectService.TagProduction}}"}"""; + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual(MachineProjectService.TagProduction, (string)(actual.Options as JObject)?["tags"]); + } + [Test] public void GetTranslationBuildConfig_DoesNotSpecifyAdditionalTrainingDataIfNoFilesSpecified() { @@ -1514,6 +1614,49 @@ public void GetTranslationBuildConfig_MergesFastTrainingConfiguration() Assert.AreEqual(20, (int)(actual.Options as JObject)?["max_steps"]); } + [Test] + public void GetTranslationBuildConfig_MergesTagFromServalConfig() + { + // Set up test environment + var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } }; + const string tag = "my_project"; + const string servalConfig = $$"""{"tags":"{{tag}}"}"""; + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual( + new JArray(tag, MachineProjectService.TagProduction), + (JArray)(actual.Options as JObject)?["tags"] + ); + } + + [Test] + public void GetTranslationBuildConfig_MergesTagsFromServalConfig() + { + // Set up test environment + var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } }; + const string tag1 = "my_first_tag"; + const string tag2 = "my_second_tag"; + const string servalConfig = $$"""{"tags":["{{tag1}}", "{{tag2}}"]}"""; + + // SUT + TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig( + new ServalData(), + servalConfig, + new BuildConfig(), + corporaSyncInfo: [] + ); + Assert.AreEqual( + new JArray(tag1, tag2, MachineProjectService.TagProduction), + (JArray)(actual.Options as JObject)?["tags"] + ); + } + [Test] public void GetTranslationBuildConfig_NoScriptureRange() { @@ -3849,6 +3992,7 @@ private class TestEnvironment public TestEnvironment(TestEnvironmentOptions? options = null) { options ??= new TestEnvironmentOptions(); + Environment = Substitute.For(); ExceptionHandler = Substitute.For(); MockLogger = new MockLogger(); CorporaClient = Substitute.For(); @@ -4103,6 +4247,7 @@ public TestEnvironment(TestEnvironmentOptions? options = null) Service = Substitute.ForPartsOf( CorporaClient, DataFilesClient, + Environment, ExceptionHandler, FileSystemService, MockLogger, @@ -4119,6 +4264,7 @@ public TestEnvironment(TestEnvironmentOptions? options = null) public MachineProjectService Service { get; } public ICorporaClient CorporaClient { get; } public IDataFilesClient DataFilesClient { get; } + public IWebHostEnvironment Environment { get; } public IFileSystemService FileSystemService { get; } public IParatextService ParatextService { get; } public SFMemoryRealtimeService RealtimeService { get; }