Terraform updates

contrib/terraform-testing-infrastructure/README.md

@@ -161,19 +161,19 @@ The table below lists the variables and their default values that are used in th
 | accumulo\_root\_password | The password for the accumulo root user. A randomly generated password will be used if none is specified here. | `string` | `null` | no |
 | accumulo\_testing\_branch\_name | The name of the branch to build and install | `string` | `"main"` | no |
 | accumulo\_testing\_repo | URL of the Accumulo Testing git repo | `string` | `"https://github.com/apache/accumulo-testing.git"` | no |
-| accumulo\_version | The branch of Accumulo to download and install | `string` | `"2.1.0-SNAPSHOT"` | no |
+| accumulo\_version | The branch of Accumulo to download and install | `string` | `"4.0.0-SNAPSHOT"` | no |
 | ami\_name\_pattern | The pattern of the name of the AMI to use | `any` | n/a | yes |
 | ami\_owner | The id of the AMI owner | `any` | n/a | yes |
 | authorized\_ssh\_key\_files | List of SSH public key files for the developers that will log into the cluster | `list(string)` | `[]` | no |
 | authorized\_ssh\_keys | List of SSH keys for the developers that will log into the cluster | `list(string)` | n/a | yes |
 | cloudinit\_merge\_type | Describes the merge behavior for overlapping config blocks in cloud-init. | `string` | `null` | no |
 | create\_route53\_records | Indicates whether or not route53 records will be created | `bool` | `false` | no |
 | hadoop\_dir | The Hadoop directory on each EC2 node | `string` | `"/data/hadoop"` | no |
-| hadoop\_version | The version of Hadoop to download and install | `string` | `"3.3.4"` | no |
+| hadoop\_version | The version of Hadoop to download and install | `string` | `"3.5.0"` | no |
 | instance\_count | The number of EC2 instances to create | `string` | `"2"` | no |
 | instance\_type | The type of EC2 instances to create | `string` | `"m5.2xlarge"` | no |
 | local\_sources\_dir | Directory on local machine that contains Maven, ZooKeeper or Hadoop binary distributions or Accumulo source tarball | `string` | `""` | no |
-| maven\_version | The version of Maven to download and install | `string` | `"3.8.8"` | no |
+| maven\_version | The version of Maven to download and install | `string` | `"3.9.16"` | no |
 | optional\_cloudinit\_config | An optional config block for the cloud-init script. If you set this, you should consider setting cloudinit\_merge\_type to handle merging with the default script as you need. | `string` | `null` | no |
 | private\_network | Indicates whether or not the user is on a private network and access to hosts should be through the private IP addresses rather than public ones. | `bool` | `false` | no |
 | root\_volume\_gb | The size, in GB, of the EC2 instance root volume | `string` | `"300"` | no |
@@ -183,7 +183,7 @@ The table below lists the variables and their default values that are used in th
 | us\_east\_1b\_subnet | The AWS subnet id for the us-east-1b subnet | `any` | n/a | yes |
 | us\_east\_1e\_subnet | The AWS subnet id for the us-east-1e subnet | `any` | n/a | yes |
 | zookeeper\_dir | The ZooKeeper directory on each EC2 node | `string` | `"/data/zookeeper"` | no |
-| zookeeper\_version | The version of ZooKeeper to download and install | `string` | `"3.8.0"` | no |
+| zookeeper\_version | The version of ZooKeeper to download and install | `string` | `"3.9.5"` | no |
 
 The following outputs are returned by the `aws` Terraform configuration.
 
@@ -330,11 +330,11 @@ This Terraform configuration:
      and builds the software using Maven, then untars the binary tarball to
      `${software_root}/accumulo/accumulo-${accumulo_version}`
   5. Downloads the [OpenTelemetry](https://opentelemetry.io/) Java Agent jar file and copies it to
-     `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar`
+     `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar`
   6. Copies the Accumulo `test` jar to `${software_root}/accumulo/accumulo-${accumulo_version}/lib`
      so that `org.apache.accumulo.test.metrics.TestStatsDRegistryFactory` is on the classpath
   7. Downloads the [Micrometer](https://micrometer.io/) StatsD Registry jar file and copies it to
-     `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.12.1.jar`
+     `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar`
   8. Clones, if necessary, the Apache Accumulo Testing Git repo from `${accumulo_testing_repo}`
      into `${software_root}/sources/accumulo-testing-repo`. It switches to the
      `${accumulo_testing_branch_name}` branch and builds the software using Maven.

contrib/terraform-testing-infrastructure/aws/main.tf

@@ -69,6 +69,7 @@ terraform {
     key            = "accumulo-testing/terraform.tfstate"
     region         = "us-east-1"
     dynamodb_table = "accumulo-testing-tf-locks"
+    use_lockfile   = true
     encrypt        = true
   }
 }

contrib/terraform-testing-infrastructure/aws/variables.tf

@@ -131,25 +131,25 @@ variable "accumulo_dir" {
 }
 
 variable "maven_version" {
-  default     = "3.8.8"
+  default     = "3.9.16"
   description = "The version of Maven to download and install"
   nullable    = false
 }
 
 variable "zookeeper_version" {
-  default     = "3.8.0"
+  default     = "3.9.5"
   description = "The version of ZooKeeper to download and install"
   nullable    = false
 }
 
 variable "hadoop_version" {
-  default     = "3.3.4"
+  default     = "3.5.0"
   description = "The version of Hadoop to download and install"
   nullable    = false
 }
 
 variable "accumulo_version" {
-  default     = "2.1.0-SNAPSHOT"
+  default     = "4.0.0-SNAPSHOT"
   description = "The branch of Accumulo to download and install"
   nullable    = false
 }

contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl

@@ -60,6 +60,7 @@ packages:
   - java-11-openjdk-devel
 %{ if centos7  == "false" ~}
   - java-17-openjdk-devel
+  - java-21-openjdk-devel
 %{ endif ~}
   - git
 %{ endif ~}

contrib/terraform-testing-infrastructure/modules/config-files/main.tf

@@ -56,6 +56,7 @@ locals {
 
   java_11_home        = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-11-openjdk-amd64" : "/usr/lib/jvm/java-11-openjdk"
   java_17_home        = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-17-openjdk-amd64" : "/usr/lib/jvm/java-17-openjdk"
+  java_21_home        = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-21-openjdk-amd64" : "/usr/lib/jvm/java-21-openjdk"
   accumulo_root_pw = coalesce(var.accumulo_root_password, random_string.accumulo_root_password.result)
 
   template_vars = {
@@ -64,6 +65,7 @@ locals {
     worker_ips                   = var.worker_ips
     java_11_home                 = local.java_11_home
     java_17_home                 = local.java_17_home
+    java_21_home                 = local.java_21_home
     accumulo_branch_name         = var.accumulo_branch_name
     accumulo_dir                 = var.accumulo_dir
     accumulo_repo                = var.accumulo_repo

contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl

@@ -41,7 +41,7 @@ general.opentelemetry.enabled=true
 # Micrometer settings
 general.micrometer.enabled=true
 general.micrometer.jvm.metrics.enabled=true
-general.micrometer.factory=org.apache.accumulo.test.metrics.TestStatsDRegistryFactory
+general.micrometer.factory=org.apache.accumulo.core.spi.metrics.AccumuloMonitorMeterRegistryFactory,org.apache.accumulo.test.metrics.TestStatsDRegistryFactory
 
 # Per-Table Encryption
 instance.crypto.opts.factory=org.apache.accumulo.core.spi.crypto.PerTableCryptoServiceFactory

contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl

@@ -18,7 +18,9 @@
 #
 
 manager:
-  - ${manager_ip}
+  servers_per_host: 1
+  hosts:
+    - ${manager_ip}
 
 monitor:
   - ${manager_ip}

contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl

@@ -22,7 +22,7 @@ Description=Hadoop DataNode start/stop
 After=remote-fs.target
 
 [Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
 Environment=HADOOP_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
 Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
 User=hadoop

contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl

@@ -22,7 +22,7 @@ Description=Hadoop NameNode start/stop
 After=remote-fs.target
 
 [Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
 Environment=HADOOP_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
 Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
 User=hadoop

contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl

@@ -10,7 +10,7 @@ export ACCUMULO_LOG_DIR=${accumulo_dir}/logs
 export M2_HOME=${software_root}/apache-maven/apache-maven-${maven_version}
 export PDSH_RCMD_TYPE=ssh
 
-export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar -Dotel.traces.exporter=otlp -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.otlp.endpoint=http://${manager_ip}:4317 -Dotel.exporter.otlp.protocol=grpc -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125"
+export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar -Dotel.traces.exporter=otlp -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.otlp.endpoint=http://${manager_ip}:4317 -Dotel.exporter.otlp.protocol=grpc -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125"
 
 # User specific environment and startup programs
 PATH=$PATH:$HOME/.local/bin:$HOME/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ACCUMULO_HOME/bin:$M2_HOME/bin

contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl

@@ -40,8 +40,8 @@ pdsh -S -g worker sudo systemctl start telegraf
 #
 # Initialize Accumulo with the supplied instance name and root user password
 #
-accumulo init --instance-name "$1" --password "$2"
-accumulo init --add-resource-groups accumulo_meta,user_small,user_large
+accumulo inst init --instance-name "$1" --password "$2"
+accumulo inst init --add-resource-groups accumulo_meta,user_small,user_large
 
 #
 # Start jaegertracing container

contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl

@@ -20,7 +20,8 @@
 
 set -eo pipefail
 
-export JAVA_HOME=${java_17_home}
+# Use Java 21 for building
+export JAVA_HOME=${java_21_home}
 
 SOURCES_DIR="${software_root}/sources"
 
@@ -88,28 +89,13 @@ if [ -f $ACCUMULO_SRC ]; then
 else
 #
 # Download, build, and install Accumulo
-  echo "Cloning Accumulo-Access repo from https://github.com/apache/accumulo-access.git"
-  cd $SOURCES_DIR
-  rm -rf accumulo-access-repo
-  git clone https://github.com/apache/accumulo-access.git accumulo-access-repo
-  cd accumulo-access-repo
-  git checkout main
-  ${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -ntp clean install -PskipQA
-
   echo "Binary tarball not found, cloning Accumulo repo from ${accumulo_repo}"
   cd $SOURCES_DIR
   rm -rf accumulo-repo
   git clone ${accumulo_repo} accumulo-repo
   cd accumulo-repo
   git checkout ${accumulo_branch_name}
-  # Java 17 required for Accumulo build
-  OLDPATH="$PATH"
-  export JAVA_HOME=${java_17_home}
-  export PATH=$JAVA_HOME/bin:$PATH
   ${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -Dapache.snapshots=true -ntp clean install -PskipQA
-  # Reset JAVA HOME to Java 17 for the remainder of the script
-  export JAVA_HOME=${java_17_home}
-  export PATH=$OLDPATH
   mkdir -p ${software_root}/accumulo
   tar zxf assemble/target/accumulo-${accumulo_version}-bin.tar.gz -C ${software_root}/accumulo
 fi
@@ -124,8 +110,8 @@ openssl rand -out ${software_root}/accumulo/accumulo-${accumulo_version}/conf/pe
 #
 # OpenTelemetry dependencies
 #
-if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar ]; then
-  wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/2.26.1/opentelemetry-javaagent-2.26.1.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar
+if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar ]; then
+  wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/2.31.1/opentelemetry-javaagent-2.31.1.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar
 fi
 #
 # Micrometer dependencies
@@ -137,8 +123,8 @@ if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/accumulo-te
     echo "accumulo-test-${accumulo_version}.jar not found, metrics won't work..."
   fi
 fi
-if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.14.5.jar ]; then
-  wget https://search.maven.org/remotecontent?filepath=io/micrometer/micrometer-registry-statsd/1.14.5/micrometer-registry-statsd-1.14.5.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.14.5.jar
+if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar ]; then
+  wget https://search.maven.org/remotecontent?filepath=io/micrometer/micrometer-registry-statsd/1.16.7/micrometer-registry-statsd-1.16.7.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar
 fi
 
 #

contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl

@@ -22,7 +22,7 @@ Description=Yarn NodeManager start/stop
 After=hadoop-datanode.service
 
 [Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
 Environment=YARN_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
 Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
 User=hadoop

contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl

@@ -22,7 +22,7 @@ Description=Yarn ResourceManager start/stop
 After=hadoop-namenode.service
 
 [Service]
-Environment=JAVA_HOME=${java_11_home}
+Environment=JAVA_HOME=${java_17_home}
 Environment=YARN_HOME=${software_root}/hadoop/hadoop-${hadoop_version}
 Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs
 User=hadoop

contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile

@@ -21,12 +21,12 @@ FROM redhat/ubi8 as builder
 COPY build_output/maven /opt/apache-maven
 COPY build_output/repo /opt/apache-maven
 COPY settings.xml /tmp/settings.xml
-ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk
 ENV TIMELY_VERSION="3.0.0-SNAPSHOT"
-ARG HADOOP_VERSION="3.3.6"
+ARG HADOOP_VERSION="3.5.0"
 
 RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-    dnf install -y java-17-openjdk-devel git wget collectd collectd-java && \
+    dnf install -y java-21-openjdk-devel git wget collectd collectd-java && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
@@ -38,16 +38,25 @@ RUN cd /opt && \
     /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml clean install
 
 RUN cd /opt && \
-    git clone https://github.com/NationalSecurityAgency/datawave-in-memory-accumulo.git datawave-in-memory-accumulo && \
-    cd datawave-in-memory-accumulo && \
-    git checkout accumulo4 && \
+    git clone https://github.com/NationalSecurityAgency/datawave.git datawave && \
+    cd datawave && \
+    git checkout c1fd5d0a787bfa73301a586d0e68598426a32ed7 && \
+    cd contrib/datawave-utils/read-properties && \
+    /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install && \
+    git checkout 7fa6fc2261cc79bce9b468d6ae8b248e44f8ff4f && \
+    cd ../code-style && \
+    /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install && \
+    git checkout accumulo4-2026 && \
+    cd ../../.. && \
+    /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install -N && \
+    cd core/in-memory-accumulo && \
     /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install
 
 RUN cd /opt && \
     git clone https://github.com/NationalSecurityAgency/timely.git timely && \
     cd timely && \
     git checkout accumulo4 && \
-    /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean package -Pcollectd -DskipTests
+    /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean package -Pcollectd -DskipTests -Denforcer.skip=true -Dspotbugs.skip=true
 
 RUN mkdir /tmp/timely-server && \
     tar zxf /opt/timely/server/target/timely-server-${TIMELY_VERSION}-dist.tar.gz --strip-components=1 -C /tmp/timely-server && \

contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh

@@ -18,7 +18,7 @@
 # under the License.
 #
 
-HADOOP_VERSION=${1:-"3.3.6"}
+HADOOP_VERSION=${1:-"3.5.0"}
 MVN_REPO=$2
 MVN_DIR=$3