CASSANDRA-20499 fixes

.build/README.md

@@ -100,6 +100,7 @@ Running other types of tests with docker:
     .build/docker/run-tests.sh -a test
     .build/docker/run-tests.sh -a stress-test
     .build/docker/run-tests.sh -a fqltool-test
+    .build/docker/run-tests.sh -a sstableloader-test
     .build/docker/run-tests.sh -a microbench
     .build/docker/run-tests.sh -a test-cdc
     .build/docker/run-tests.sh -a test-compression

.build/build-accord.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project basedir=".." name="apache-cassandra-accord-build"
+         xmlns:if="ant:if">
+    <target name="_build-accord" depends="init">
+      <exec executable="${basedir}/${accord.dir}/gradlew" dir="${basedir}/${accord.dir}" logError="true" failonerror="true" failifexecutionfails="true">
+        <!-- Need to call clean as a version change with a jar that didn't have code changes will not produce a new jar with the new version... so need clean to make sure build is stable -->
+        <arg value="clean" />
+        <arg value="build" />
+        <arg value="copyDependencies" /> <!-- For IDE support -->
+        <arg value="publishToMavenLocal" />
+        <arg value="-x" />
+        <arg value="test" />
+        <!-- Disable rat/checkstyle as C*'s logic will include the accord source -->
+        <arg value="-x" />
+        <arg value="rat" />
+        <arg value="-x" />
+        <arg value="checkstyleMain" />
+        <arg value="-x" />
+        <arg value="checkstyleTest" />
+        <arg value="-x" />
+        <arg value="javadoc" />
+        <arg value="--stacktrace" />
+        <arg value="-Paccord_group=org.apache.cassandra" />
+        <arg value="-Paccord_artifactId=cassandra-accord" />
+        <arg value="-Paccord_version=${version}" />
+      </exec>
+      <!-- when dependencies are copied into the build dir they may be ignored if already present, so cleanup to pick up latest changes -->
+      <delete>
+        <!-- Why does this not use ${build.dir.lib}/jar and ${test.lib}/jar?  The reason is that filesets never finds files in that case -->
+        <!-- To handle the case that build or lib dirs do not exist yet, had to push the logic into includes, other solution was to create a task and use if conditions there -->
+        <fileset dir="${basedir}" includes="build/lib/jars/cassandra-accord-*.jar,build/test/lib/jars/cassandra-accord-*.jar" />
+      </delete>
+    </target>
+</project>

.build/build-checkstyle.xml

@@ -19,7 +19,7 @@
     <project basedir="." name="apache-cassandra-checkstyle"
      xmlns:rat="antlib:org.apache.rat.anttasks">
 
-    <target name="init-checkstyle" depends="resolver-retrieve-build,build-project" unless="no-checkstyle">
+    <target name="init-checkstyle" depends="resolver-retrieve-build,_build_subprojects,build-project" unless="no-checkstyle">
         <path id="checkstyle.lib.path">
             <fileset dir="${test.lib}/jars" includes="*.jar"/>
         </path>
@@ -45,7 +45,7 @@
         </checkstyle>
     </target>
 
-    <target name="checkstyle-test" depends="init-checkstyle,resolver-retrieve-build,build-project" description="Run custom checkstyle code analysis on tests" unless="no-checkstyle">
+    <target name="checkstyle-test" depends="init-checkstyle,resolver-retrieve-build,_build_subprojects,build-project" description="Run custom checkstyle code analysis on tests" unless="no-checkstyle">
         <property name="checkstyle.log.dir" value="${build.dir}/checkstyle" />
         <property name="checkstyle_test.report.file" value="${checkstyle.log.dir}/checkstyle_report_test.xml"/>
         <mkdir  dir="${checkstyle.log.dir}" />

.build/build-owasp.xml

@@ -19,7 +19,7 @@
 <project basedir="." name="apache-cassandra-owasp-tasks"
          xmlns:unless="ant:unless"
          xmlns:if="ant:if">
-    <property name="dependency-check.version" value="10.0.4"/>
+    <property name="dependency-check.version" value="12.1.0"/>
     <property name="dependency-check.home" value="${tmp.dir}/dependency-check-ant-${dependency-check.version}"/>
     <property name="dependency-check.archive.dir" value="${local.repository}/org/owasp/dependency-check-ant/${dependency-check.version}"/>
     <property name="dependency-check.archive.name" value="dependency-check-ant-${dependency-check.version}-release.zip"/>
@@ -34,7 +34,7 @@
             unless="dependency-check-ant.archive.present">
 
         <mkdir dir="${dependency-check.archive.dir}"/>
-        <get src="https://github.com/jeremylong/DependencyCheck/releases/download/v${dependency-check.version}/${dependency-check.archive.name}"
+        <get src="https://github.com/dependency-check/DependencyCheck/releases/download/v${dependency-check.version}/${dependency-check.archive.name}"
              dest="${dependency-check.archive.dir}/${dependency-check.archive.name}" retries="3"/>
     </target>
 
@@ -105,7 +105,7 @@
         </dependency-check>
     </target>
 
-    <target name="dependency-check" depends="resolver-dist-lib">
+    <target name="dependency-check" depends="resolver-dist-lib,generate-snyk-file">
         <fileset dir="lib" id="dependencies_to_check">
             <include name="**/*.jar"/>
         </fileset>

.build/build-rat.xml

@@ -30,6 +30,7 @@
         <exec executable="git" failifexecutionfails="false" failonerror="false" resultproperty="git.success" output="${build.dir}/.ratinclude">
             <arg line="ls-tree -r HEAD --name-only"/>
         </exec>
+        <exec executable="${build.helpers.dir}/rat-include-accord.sh" failifexecutionfails="false" failonerror="false" resultproperty="git.success"  output="${build.dir}/.ratinclude" append="true" />
         <condition property="rat.skip" value="true">
             <not>
                 <equals arg1="${git.success}" arg2="0"/>
@@ -78,6 +79,7 @@
                  <exclude NAME="CHANGES.txt"/>
                  <exclude NAME="CASSANDRA-14092.txt"/>
                  <exclude NAME="debian/TODO"/>
+                 <exclude NAME="accord_demo.txt"/>
                  <!-- legal files -->
                  <exclude NAME="NOTICE.txt"/>
                  <exclude NAME="LICENSE.txt"/>

.build/build-resolver.xml

@@ -178,7 +178,7 @@
         <property name="resolver-ant-tasks.initialized" value="true"/>
     </target>
 
-    <target name="resolver-retrieve-build" depends="resolver-init,write-poms">
+    <target name="resolver-retrieve-build" depends="resolver-init,write-poms,_build_subprojects">
         <resolvepom file="${build.dir}/${final.name}.pom" id="all-pom" />
         <resolvepom file="${build.dir}/tmp-${final.name}-deps.pom" id="pom-deps" />
 
@@ -206,7 +206,7 @@
         </unzip>
     </target>
 
-    <target name="resolver-dist-lib" depends="resolver-retrieve-build">
+    <target name="resolver-dist-lib" depends="resolver-retrieve-build,_build_subprojects">
         <resolvepom file="${build.dir}/${final.name}.pom" id="all-pom" />
 
         <retry retrycount="3" retrydelay="10" >
@@ -251,8 +251,8 @@
         <delete file="${build.lib}/AmazonCorrettoCryptoProvider-2.2.0-linux-x86_64.jar" failonerror="false"/>
         <delete file="${build.lib}/AmazonCorrettoCryptoProvider-2.2.0-linux-aarch_64.jar" failonerror="false"/>
 
-        <delete file="${build.lib}/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar" failonerror="false"/>
-        <delete file="${build.dir.lib}/jars/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar" failonerror="false"/>
+        <delete file="${build.lib}/netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar" failonerror="false"/>
+        <delete file="${build.dir.lib}/jars/netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar" failonerror="false"/>
     </target>
 
     <target name="_resolver-dist-lib_get_files">

.build/build-sonar.xml

@@ -226,9 +226,9 @@
             <arg value="-Dsonar.working.directory=${sonar.workdir}/scanner"/>
             <arg value="-Dsonar.sourceEncoding=UTF-8"/>
             <arg value="-Dsonar.scm.disabled=true"/>
-            <arg value="-Dsonar.sources=${build.src.java},${fqltool.build.src},${stress.build.src}"/>
+            <arg value="-Dsonar.sources=${build.src.java},${fqltool.build.src},${sstableloader.build.src},${stress.build.src}"/>
             <arg value="-Dsonar.java.libraries=${build.dir.lib}/jars/jsr305-*.jar"/>
-            <arg value="-Dsonar.java.binaries=${build.classes.main},${fqltool.build.classes},${stress.build.classes}"/>
+            <arg value="-Dsonar.java.binaries=${build.classes.main},${fqltool.build.classes},${sstableloader.build.classes},${stress.build.classes}"/>
             <arg value="-Dsonar.java.source=11"/>
             <arg value="-Dsonar.projectVersion=${timestamp}"/>
             <arg value="-Dsonar.qualitygate.wait=true"/>

.build/cassandra-build-deps-template.xml → .build/cassandra-build-maven-pom.xml

@@ -155,5 +155,10 @@
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcutil-jdk18on</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cassandra</groupId>
+      <artifactId>cassandra-accord</artifactId>
+      <classifier>tests</classifier>
+    </dependency>
   </dependencies>
 </project>

.build/cassandra-deps-template.xml → .build/cassandra-deps-maven-pom.xml

@@ -116,6 +116,10 @@
       <groupId>org.mindrot</groupId>
       <artifactId>jbcrypt</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cassandra</groupId>
+      <artifactId>cassandra-accord</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.airlift</groupId>
       <artifactId>airline</artifactId>
@@ -145,7 +149,7 @@
       <artifactId>logback-classic</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.datastax.cassandra</groupId>
+      <groupId>org.apache.cassandra</groupId>
       <artifactId>cassandra-driver-core</artifactId>
       <classifier>shaded</classifier>
     </dependency>

.build/checkstyle_suppressions.xml

@@ -21,5 +21,4 @@
         "https://checkstyle.org/dtds/suppressions_1_1.dtd">
 
 <suppressions>
-  <suppress checks="RegexpSinglelineJava" files="Semaphore\.java"/>
 </suppressions>

.build/docker/bullseye-build.docker

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 FROM debian:bullseye
-MAINTAINER Apache Cassandra <[email protected]>
+LABEL org.opencontainers.image.authors="Apache Cassandra <[email protected]>"
 
 # CONTEXT is expected to be cassandra/.build
 
@@ -52,3 +52,22 @@ RUN pip install --upgrade pip
 
 # dependencies for .build/ci/ci_parser.py
 RUN pip install beautifulsoup4==4.12.3 jinja2==3.1.3
+
+# install golang.  GO_VERSION_SHA must be updated with VERSION
+RUN sh -c '\
+  GO_VERSION="1.24.3" ;\
+  GO_VERSION_SHAS="3333f6ea53afa971e9078895eaa4ac7204a8c6b5c68c10e6bc9a33e8e391bdd8 a463cb59382bd7ae7d8f4c68846e73c4d589f223c589ac76871b66811ded7836 13e6fe3fcf65689d77d40e633de1e31c6febbdbcb846eb05fc2434ed2213e92b 64a3fa22142f627e78fac3018ce3d4aeace68b743eff0afda8aae0411df5e4fb" ;\
+  GO_OS=linux ;\
+  [ $(uname) = "Darwin" ] && GO_OS=darwin ;\
+  GO_PLATFORM=amd64 ;\
+  [ $(uname -m) = "aarch64" ] && GO_PLATFORM=arm64 ;\
+  GO_TAR="go${GO_VERSION}.${GO_OS}-${GO_PLATFORM}.tar.gz" ;\
+  curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 30 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\
+  GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\
+  echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\
+  tar -C /usr/local -xzf $GO_TAR ;\
+  rm $GO_TAR'
+
+ENV GOROOT="/usr/local/go"
+ENV GOPATH="$BUILD_HOME/go"
+ENV PATH="$PATH:/usr/local/go/bin"

.build/docker/run-tests.sh

@@ -40,7 +40,7 @@ error() {
 
 # legacy argument handling
 case ${1} in
-  "build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
+  "build_dtest_jars" | "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
     test_type="-a ${1}"
     if [[ -z ${2} ]]; then
       test_list=""
@@ -182,7 +182,7 @@ docker_flags="-m 5g --memory-swap 5g"
 case ${test_target/-repeat/} in
     "build_dtest_jars")
     ;;
-    "stress-test" | "fqltool-test" )
+    "stress-test" | "fqltool-test" | "sstableloader-test" )
         [[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
     ;;
     # test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway

.build/docker/ubuntu2004_test.docker

@@ -124,22 +124,23 @@ RUN /bin/bash -c "source ${BUILD_HOME}/env3.8/bin/activate && \
     ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \
     ccm create -n 1 -v git:cassandra-4.0 test && ccm remove test"
 
-# Initialize ccm versions.  right side of each sequence needs to be updated with new releases.
-#  this can be checked with:
-#  `curl -s https://downloads.apache.org/cassandra/ | grep -oP '(?<=href=\")[0-9]+\.[0-9]+\.[0-9]+(?=)' | sort -rV | uniq -w 3`
+# Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
 RUN bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
-    for i in {1..14} ; do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i test && ccm remove test ; done && \
-    for i in {1..7}  ; do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done'
+    latest_4_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
+    for i in $(seq 1 $latest_4_0); do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i test && ccm remove test ; done && \
+    latest_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.1\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
+    for i in $(seq 1 $latest_4_1); do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done'
 
 # 5+ requires java11
 RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture)
 
-# Initialize the CCM git repo, after removing the git cache, as this also can fail to clone
+# Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
 RUN rm -fr ${BUILD_HOME}/.ccm/repository/_git_cache_apache
 RUN /bin/bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
     ccm create --quiet -n 1 -v git:cassandra-5.0 test && ccm remove test && \
     ccm create --quiet -n 1 -v git:trunk test && ccm remove test && \
-    for i in {1..2}  ; do echo $i ; ccm create --quiet -n 1 -v binary:5.0.$i test && ccm remove test ; done'
+    latest_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")5\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
+    for i in $(seq 1 $latest_5_0); do echo $i ; ccm create --quiet -n 1 -v binary:5.0.$i test && ccm remove test ; done'
 
 # the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them
 #  and other directories we don't need in image

.build/git/git-hooks/post-checkout/100-update-submodules.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Redirect output to stderr.
+exec 1>&2
+
+#set -o xtrace
+set -o errexit
+set -o pipefail
+set -o nounset
+
+bin="$(cd "$(dirname "$0")" > /dev/null; pwd)"
+
+_main() {
+  # In case the usage happens at a different layer, make sure to cd to the toplevel
+  local root_dir
+  root_dir="$(git rev-parse --show-toplevel)"
+  cd "$root_dir"
+
+  if [[ ! -e .gitmodules ]]; then
+    # nothing to see here, look away!
+    return 0
+  fi
+  git submodule update --init --recursive
+}
+
+_main "$@"

.build/git/git-hooks/post-switch

@@ -0,0 +1 @@
+post-checkout