Skip to content

Commit e00aae8

Browse files
pmachapmanpmachapman
committed
Add tags to Serval jobs based on the Scripture Forge environment
1 parent 6bb1732 commit e00aae8

File tree

2 files changed

+174
-10
lines changed

2 files changed

+174
-10
lines changed

‎src/SIL.XForge.Scripture/Services/MachineProjectService.cs‎

Lines changed: 62 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
using System.Threading;
99
using System.Threading.Tasks;
1010
using Autofac.Extras.DynamicProxy;
11+
using Microsoft.AspNetCore.Hosting;
1112
using Microsoft.AspNetCore.Http;
13+
using Microsoft.Extensions.Hosting;
1214
using Microsoft.Extensions.Logging;
1315
using Microsoft.Extensions.Options;
1416
using Newtonsoft.Json.Linq;
@@ -36,6 +38,7 @@ namespace SIL.XForge.Scripture.Services;
3638
public class MachineProjectService(
3739
ICorporaClient corporaClient,
3840
IDataFilesClient dataFilesClient,
41+
IWebHostEnvironment env,
3942
IExceptionHandler exceptionHandler,
4043
IFileSystemService fileSystemService,
4144
ILogger<MachineProjectService> logger,
@@ -54,6 +57,12 @@ IRepository<UserSecret> userSecrets
5457
internal const string Nmt = "nmt";
5558
internal const string SmtTransfer = "smt-transfer";
5659

60+
// These tags are used for the ClearML task to differentiate Scripture Forge jobs
61+
internal const string TagDevelopment = "sf-dev";
62+
internal const string TagTest = "sf-test";
63+
internal const string TagStaging = "sf-qa";
64+
internal const string TagProduction = "sf-live";
65+
5766
/// <summary>
5867
/// Adds the SMT project to Serval, if the required data is present.
5968
/// </summary>
@@ -588,7 +597,7 @@ await RecreateOrUpdateTranslationEngineIfRequiredAsync(
588597
else
589598
{
590599
translationEngineId = projectSecret.ServalData.TranslationEngineId!;
591-
translationBuildConfig = new TranslationBuildConfig();
600+
translationBuildConfig = new TranslationBuildConfig { Options = AddTags() };
592601
}
593602

594603
// Start the build
@@ -1112,19 +1121,14 @@ IList<ServalCorpusSyncInfo> corporaSyncInfo
11121121
)
11131122
{
11141123
// Load the Serval Config from the Draft Config
1115-
JObject? options = null;
1116-
if (!string.IsNullOrWhiteSpace(servalConfig))
1117-
{
1118-
options = JObject.Parse(servalConfig);
1119-
}
1124+
JObject? options = !string.IsNullOrWhiteSpace(servalConfig) ? JObject.Parse(servalConfig) : [];
1125+
1126+
// Add the tags
1127+
AddTags(options);
11201128

11211129
// If Fast Training is enabled, override the max_steps
11221130
if (buildConfig.FastTraining)
11231131
{
1124-
// Ensure that there is a servalConfig JSON object
1125-
options ??= [];
1126-
1127-
// 20 is the number of steps used on Serval QA by default
11281132
options["max_steps"] = 20;
11291133
}
11301134

@@ -2129,6 +2133,54 @@ CancellationToken cancellationToken
21292133
return true;
21302134
}
21312135

2136+
/// <summary>
2137+
/// Adds the tags to the build_options
2138+
/// </summary>
2139+
/// <param name="options">The build options.</param>
2140+
/// <returns>The build options</returns>
2141+
private JObject AddTags(JObject? options = null)
2142+
{
2143+
// Generate the tag based on the environment
2144+
string tag;
2145+
if (env.EnvironmentName == Environments.Production)
2146+
{
2147+
tag = TagProduction;
2148+
}
2149+
else if (env.EnvironmentName == Environments.Staging)
2150+
{
2151+
tag = TagStaging;
2152+
}
2153+
else if (env.EnvironmentName == Environments.Development)
2154+
{
2155+
tag = TagDevelopment;
2156+
}
2157+
else
2158+
{
2159+
tag = TagTest;
2160+
}
2161+
2162+
// Set the tag in the options, taking into account potential existing values
2163+
options ??= [];
2164+
if (options["tags"] == null)
2165+
{
2166+
options["tags"] = tag;
2167+
}
2168+
else if (options["tags"].Type == JTokenType.String)
2169+
{
2170+
options["tags"] = new JArray(options["tags"].ToString(), tag);
2171+
}
2172+
else if (options["tags"].Type == JTokenType.Array)
2173+
{
2174+
JArray tags = (JArray)options["tags"];
2175+
if (!tags.Any(t => t.Type == JTokenType.String && t.ToString() == tag))
2176+
{
2177+
tags.Add(tag);
2178+
}
2179+
}
2180+
2181+
return options;
2182+
}
2183+
21322184
/// <summary>
21332185
/// Gets the translation engine identifier from the project secret,
21342186
/// depending on whether we are pre-translating or not.

‎test/SIL.XForge.Scripture.Tests/Services/MachineProjectServiceTests.cs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
using System.Text;
77
using System.Threading;
88
using System.Threading.Tasks;
9+
using Microsoft.AspNetCore.Hosting;
10+
using Microsoft.Extensions.Hosting;
911
using Microsoft.Extensions.Logging;
1012
using Microsoft.Extensions.Options;
1113
using Newtonsoft.Json.Linq;
@@ -1470,6 +1472,70 @@ public void GetTargetLanguage_Success()
14701472
Assert.AreEqual(targetWritingSystemTag, actual);
14711473
}
14721474

1475+
[Test]
1476+
public void GetTranslationBuildConfig_AddsTagForDev()
1477+
{
1478+
// Set up test environment
1479+
var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Development } };
1480+
1481+
// SUT
1482+
TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig(
1483+
new ServalData(),
1484+
servalConfig: null,
1485+
new BuildConfig(),
1486+
corporaSyncInfo: []
1487+
);
1488+
Assert.AreEqual(MachineProjectService.TagDevelopment, (string)(actual.Options as JObject)?["tags"]);
1489+
}
1490+
1491+
[Test]
1492+
public void GetTranslationBuildConfig_AddsTagForTest()
1493+
{
1494+
// Set up test environment
1495+
var env = new TestEnvironment();
1496+
1497+
// SUT
1498+
TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig(
1499+
new ServalData(),
1500+
servalConfig: null,
1501+
new BuildConfig(),
1502+
corporaSyncInfo: []
1503+
);
1504+
Assert.AreEqual(MachineProjectService.TagTest, (string)(actual.Options as JObject)?["tags"]);
1505+
}
1506+
1507+
[Test]
1508+
public void GetTranslationBuildConfig_AddsTagForStaging()
1509+
{
1510+
// Set up test environment
1511+
var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Staging } };
1512+
1513+
// SUT
1514+
TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig(
1515+
new ServalData(),
1516+
servalConfig: null,
1517+
new BuildConfig(),
1518+
corporaSyncInfo: []
1519+
);
1520+
Assert.AreEqual(MachineProjectService.TagStaging, (string)(actual.Options as JObject)?["tags"]);
1521+
}
1522+
1523+
[Test]
1524+
public void GetTranslationBuildConfig_AddsTagForProduction()
1525+
{
1526+
// Set up test environment
1527+
var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } };
1528+
1529+
// SUT
1530+
TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig(
1531+
new ServalData(),
1532+
servalConfig: null,
1533+
new BuildConfig(),
1534+
corporaSyncInfo: []
1535+
);
1536+
Assert.AreEqual(MachineProjectService.TagProduction, (string)(actual.Options as JObject)?["tags"]);
1537+
}
1538+
14731539
[Test]
14741540
public void GetTranslationBuildConfig_DoesNotSpecifyAdditionalTrainingDataIfNoFilesSpecified()
14751541
{
@@ -1514,6 +1580,49 @@ public void GetTranslationBuildConfig_MergesFastTrainingConfiguration()
15141580
Assert.AreEqual(20, (int)(actual.Options as JObject)?["max_steps"]);
15151581
}
15161582

1583+
[Test]
1584+
public void GetTranslationBuildConfig_MergesTagFromServalConfig()
1585+
{
1586+
// Set up test environment
1587+
var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } };
1588+
const string tag = "my_project";
1589+
const string servalConfig = $$"""{"tags":"{{tag}}"}""";
1590+
1591+
// SUT
1592+
TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig(
1593+
new ServalData(),
1594+
servalConfig,
1595+
new BuildConfig(),
1596+
corporaSyncInfo: []
1597+
);
1598+
Assert.AreEqual(
1599+
new JArray(tag, MachineProjectService.TagProduction),
1600+
((JArray)(actual.Options as JObject)?["tags"])
1601+
);
1602+
}
1603+
1604+
[Test]
1605+
public void GetTranslationBuildConfig_MergesTagsFromServalConfig()
1606+
{
1607+
// Set up test environment
1608+
var env = new TestEnvironment { Environment = { EnvironmentName = Environments.Production } };
1609+
const string tag1 = "my_first_tag";
1610+
const string tag2 = "my_second_tag";
1611+
const string servalConfig = $$"""{"tags":["{{tag1}}", "{{tag2}}"]}""";
1612+
1613+
// SUT
1614+
TranslationBuildConfig actual = env.Service.GetTranslationBuildConfig(
1615+
new ServalData(),
1616+
servalConfig,
1617+
new BuildConfig(),
1618+
corporaSyncInfo: []
1619+
);
1620+
Assert.AreEqual(
1621+
new JArray(tag1, tag2, MachineProjectService.TagProduction),
1622+
((JArray)(actual.Options as JObject)?["tags"])
1623+
);
1624+
}
1625+
15171626
[Test]
15181627
public void GetTranslationBuildConfig_NoScriptureRange()
15191628
{
@@ -3849,6 +3958,7 @@ private class TestEnvironment
38493958
public TestEnvironment(TestEnvironmentOptions? options = null)
38503959
{
38513960
options ??= new TestEnvironmentOptions();
3961+
Environment = Substitute.For<IWebHostEnvironment>();
38523962
ExceptionHandler = Substitute.For<IExceptionHandler>();
38533963
MockLogger = new MockLogger<MachineProjectService>();
38543964
CorporaClient = Substitute.For<ICorporaClient>();
@@ -4103,6 +4213,7 @@ public TestEnvironment(TestEnvironmentOptions? options = null)
41034213
Service = Substitute.ForPartsOf<MachineProjectService>(
41044214
CorporaClient,
41054215
DataFilesClient,
4216+
Environment,
41064217
ExceptionHandler,
41074218
FileSystemService,
41084219
MockLogger,
@@ -4119,6 +4230,7 @@ public TestEnvironment(TestEnvironmentOptions? options = null)
41194230
public MachineProjectService Service { get; }
41204231
public ICorporaClient CorporaClient { get; }
41214232
public IDataFilesClient DataFilesClient { get; }
4233+
public IWebHostEnvironment Environment { get; }
41224234
public IFileSystemService FileSystemService { get; }
41234235
public IParatextService ParatextService { get; }
41244236
public SFMemoryRealtimeService RealtimeService { get; }

0 commit comments

Comments
 (0)