Skip to content

CNDB-15968: Fix flaky SAI tests by ensuring index queryability on all coordinators. Rename ambiguous waitForIndexqueryable() to waitForIndexQueryableOnFirstNode() and add coordinator-aware alternatives #2127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 19, 2025
Merged

CNDB-15968: Fix flaky SAI tests by ensuring index queryability on all coordinators. Rename ambiguous waitForIndexqueryable() to waitForIndexQueryableOnFirstNode() and add coordinator-aware alternatives #2127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6595564
CNDB-15968: Fix flaky SAI tests by ensuring index queryability on all…
ekaterinadimitrova2 Nov 15, 2025
10eaa91
CNDB-15968: Create a constant INDEX_BUILD_TIMEOUT_SECONDS in SAIUtil
ekaterinadimitrova2 Nov 18, 2025
a0c3dcc
CNDB-15969: Opt in for the noisy renaming change instead of using a d…
ekaterinadimitrova2 Nov 18, 2025
c9f9778
CNDB-15968: Remove not needed comment
ekaterinadimitrova2 Nov 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions test/distributed/org/apache/cassandra/distributed/test/QueryInfoTrackerDistributedTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public void testTrackingReadsWithEndpointGrouping()
String table = rfOneKs + ".saiTbl";
cluster.schemaChange("CREATE TABLE " + table + " (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT)");
cluster.schemaChange("CREATE CUSTOM INDEX IF NOT EXISTS test_idx ON " + table + " (v1) USING 'StorageAttachedIndex'");
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);

int rowsCount = 1000;

Expand Down Expand Up @@ -151,7 +151,7 @@ public void testANNQueryWithIndexRestrictionAndLIMIT()
cluster.schemaChange("CREATE TABLE " + table + " (p int PRIMARY KEY, v int, ni int, vec VECTOR<FLOAT, 2>)");
cluster.schemaChange("CREATE CUSTOM INDEX ON " + table + "(vec) USING 'StorageAttachedIndex'");
cluster.schemaChange("CREATE CUSTOM INDEX ON " + table + "(v) USING 'StorageAttachedIndex'");
SAIUtil.waitForIndexQueryable(cluster, rfOneKs);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, rfOneKs);

for (int rowIdx = 0; rowIdx < 100; rowIdx++)
{
Expand Down
3 changes: 2 additions & 1 deletion test/distributed/org/apache/cassandra/distributed/test/sai/ANNOptionsDistributedTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ private static void testSelectWithAnnOptions(Cluster cluster, String expectedErr
{
cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int PRIMARY KEY, n int, v vector<float, 2>)"));
cluster.schemaChange(withKeyspace("CREATE CUSTOM INDEX ON %s.t(v) USING 'StorageAttachedIndex'"));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
// Wait for index to be queryable on all nodes since test queries from all coordinators
SAIUtil.waitForIndexQueryableOnAllNodes(cluster, KEYSPACE);

String selectRerankk = withKeyspace("SELECT * FROM %s.t ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'rerank_k': 10}");
String selectUsePruning = withKeyspace("SELECT * FROM %s.t ORDER BY v ANN OF [1, 1] LIMIT 10 WITH ann_options = {'use_pruning': false}");
Expand Down
8 changes: 4 additions & 4 deletions test/distributed/org/apache/cassandra/distributed/test/sai/AnalyzerDistributedTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public void testAnalyzerSearch()
{
cluster.schemaChange(formatQuery("CREATE TABLE %s (pk int PRIMARY KEY, not_analyzed int, val text)"));
cluster.schemaChange(formatQuery("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex' WITH OPTIONS = {'index_analyzer': 'standard'}"));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);

var iterations = 15000;
for (int i = 0; i < iterations; i++)
Expand Down Expand Up @@ -137,7 +137,7 @@ public void testIndexAndQueryAnalyzerSearch()
"'query_analyzer': '{" +
" \"tokenizer\" : { \"name\" : \"whitespace\", \"args\" : {} }," +
" \"filters\" : [ {\"name\" : \"lowercase\",\"args\": {}} ]}'}"));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);

execute("INSERT INTO %s(c1,c2) VALUES (1, 'astra quick fox')");
execute("INSERT INTO %s(c1,c2) VALUES (2, 'astra quick foxes')");
Expand All @@ -159,7 +159,7 @@ public void testEdgeNgramFilterWithOR() throws Throwable
"{\"name\":\"edgengram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" +
"\t\"charFilters\":[]" +
"}'};"));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);

execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')");
execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')");
Expand Down Expand Up @@ -206,7 +206,7 @@ public void testNgramFilterWithOR() throws Throwable
"{\"name\":\"ngram\", \"args\":{\"minGramSize\":\"2\", \"maxGramSize\":\"30\"}}],\n" +
"\t\"charFilters\":[]" +
"}'};"));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);

execute("INSERT INTO %s (id, val) VALUES ('1', 'MAL0133AU')");
execute("INSERT INTO %s (id, val) VALUES ('2', 'WFS2684AU')");
Expand Down
2 changes: 1 addition & 1 deletion test/distributed/org/apache/cassandra/distributed/test/sai/BM25DistributedTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void before()
table = "table_" + seq.getAndIncrement();
cluster.schemaChange(formatQuery(CREATE_TABLE));
cluster.schemaChange(formatQuery(CREATE_INDEX));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion test/distributed/org/apache/cassandra/distributed/test/sai/ConcurrencyFactorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public void testInitialConcurrencySelection()
cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, state ascii, gdp bigint, PRIMARY KEY (pk)) WITH compaction = " +
" {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }", KEYSPACE, SAI_TABLE));
cluster.schemaChange(String.format("CREATE CUSTOM INDEX ON %s.%s (gdp) USING 'StorageAttachedIndex'", KEYSPACE, SAI_TABLE));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
SAIUtil.waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);

insertRows(1_000_000_000L, 16_000_000_000L, 1_000_000_000L);

Expand Down
6 changes: 4 additions & 2 deletions test/distributed/org/apache/cassandra/distributed/test/sai/GenericOrderByTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ public void testOrderBy() throws Throwable
{
cluster.schemaChange(withKeyspace("CREATE TABLE %s.t(k int, c int, v int, PRIMARY KEY(k, c))"));
cluster.schemaChange(withKeyspace("CREATE CUSTOM INDEX ON %s.t(v) USING 'StorageAttachedIndex'"));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
// Wait for index to be queryable on all nodes since test queries from all coordinators
SAIUtil.waitForIndexQueryableOnAllNodes(cluster, KEYSPACE);

ICoordinator coordinator = cluster.coordinator(1);

Expand Down Expand Up @@ -111,7 +112,8 @@ public void testOrderBy() throws Throwable

// with indexed column filter
cluster.schemaChange(withKeyspace("CREATE CUSTOM INDEX ON %s.t(c) USING 'StorageAttachedIndex'"));
SAIUtil.waitForIndexQueryable(cluster, KEYSPACE);
// Wait for index to be queryable on all nodes since test queries from all coordinators
SAIUtil.waitForIndexQueryableOnAllNodes(cluster, KEYSPACE);
assertRowsWithLimit(cluster, "SELECT * FROM %s.t WHERE c=1 ORDER BY v ASC",
row(1, 1, 1),
row(2, 1, 6),
Expand Down
8 changes: 4 additions & 4 deletions test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@

import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.distributed.test.sai.SAIUtil.waitForIndexQueryable;
import static org.apache.cassandra.distributed.test.sai.SAIUtil.waitForIndexQueryableOnFirstNode;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -90,7 +90,7 @@ public void verifyIndexStatusPropagation() throws Exception
cluster.schemaChange(String.format(CREATE_TABLE, ks, cf1));
cluster.schemaChange(String.format(CREATE_INDEX, index1, ks, cf1, "v1"));
cluster.schemaChange(String.format(CREATE_INDEX, index2, ks, cf1, "v2"));
waitForIndexQueryable(cluster, ks);
waitForIndexQueryableOnFirstNode(cluster, ks);
cluster.forEach(node -> {
expectedNodeIndexQueryability.put(NodeIndex.create(ks, index1, node), Index.Status.BUILD_SUCCEEDED);
expectedNodeIndexQueryability.put(NodeIndex.create(ks, index2, node), Index.Status.BUILD_SUCCEEDED);
Expand Down Expand Up @@ -199,7 +199,7 @@ public void testIndexExceptionsTwoIndexesOn3NodeCluster() throws Exception

cluster.schemaChange(String.format(CREATE_INDEX, index1, ks2, cf1, "v1"));
cluster.schemaChange(String.format(CREATE_INDEX, index2, ks2, cf1, "v2"));
waitForIndexQueryable(cluster, ks2);
waitForIndexQueryableOnFirstNode(cluster, ks2);

// Mark only index2 as building on node3, leave index1 in BUILD_SUCCEEDED state
markIndexBuilding(cluster.get(3), ks2, cf1, index2);
Expand Down Expand Up @@ -247,7 +247,7 @@ private void shouldSkipNonQueryableNode(int nodes, List<Integer>... nonQueryable
cluster.schemaChange(String.format(CREATE_TABLE, KEYSPACE, table));
cluster.schemaChange(String.format(CREATE_INDEX, "", KEYSPACE, table, "v1"));
cluster.schemaChange(String.format(CREATE_INDEX, "", KEYSPACE, table, "v2"));
waitForIndexQueryable(cluster, KEYSPACE);
waitForIndexQueryableOnFirstNode(cluster, KEYSPACE);

// create 100 rows in 1 sstable
int rows = 100;
Expand Down
Loading