diff --git a/contrib/terraform-testing-infrastructure/README.md b/contrib/terraform-testing-infrastructure/README.md index 72e1b555..d84989c5 100644 --- a/contrib/terraform-testing-infrastructure/README.md +++ b/contrib/terraform-testing-infrastructure/README.md @@ -161,7 +161,7 @@ The table below lists the variables and their default values that are used in th | accumulo\_root\_password | The password for the accumulo root user. A randomly generated password will be used if none is specified here. | `string` | `null` | no | | accumulo\_testing\_branch\_name | The name of the branch to build and install | `string` | `"main"` | no | | accumulo\_testing\_repo | URL of the Accumulo Testing git repo | `string` | `"https://github.com/apache/accumulo-testing.git"` | no | -| accumulo\_version | The branch of Accumulo to download and install | `string` | `"2.1.0-SNAPSHOT"` | no | +| accumulo\_version | The branch of Accumulo to download and install | `string` | `"4.0.0-SNAPSHOT"` | no | | ami\_name\_pattern | The pattern of the name of the AMI to use | `any` | n/a | yes | | ami\_owner | The id of the AMI owner | `any` | n/a | yes | | authorized\_ssh\_key\_files | List of SSH public key files for the developers that will log into the cluster | `list(string)` | `[]` | no | @@ -169,11 +169,11 @@ The table below lists the variables and their default values that are used in th | cloudinit\_merge\_type | Describes the merge behavior for overlapping config blocks in cloud-init. | `string` | `null` | no | | create\_route53\_records | Indicates whether or not route53 records will be created | `bool` | `false` | no | | hadoop\_dir | The Hadoop directory on each EC2 node | `string` | `"/data/hadoop"` | no | -| hadoop\_version | The version of Hadoop to download and install | `string` | `"3.3.4"` | no | +| hadoop\_version | The version of Hadoop to download and install | `string` | `"3.5.0"` | no | | instance\_count | The number of EC2 instances to create | `string` | `"2"` | no | | instance\_type | The type of EC2 instances to create | `string` | `"m5.2xlarge"` | no | | local\_sources\_dir | Directory on local machine that contains Maven, ZooKeeper or Hadoop binary distributions or Accumulo source tarball | `string` | `""` | no | -| maven\_version | The version of Maven to download and install | `string` | `"3.8.8"` | no | +| maven\_version | The version of Maven to download and install | `string` | `"3.9.16"` | no | | optional\_cloudinit\_config | An optional config block for the cloud-init script. If you set this, you should consider setting cloudinit\_merge\_type to handle merging with the default script as you need. | `string` | `null` | no | | private\_network | Indicates whether or not the user is on a private network and access to hosts should be through the private IP addresses rather than public ones. | `bool` | `false` | no | | root\_volume\_gb | The size, in GB, of the EC2 instance root volume | `string` | `"300"` | no | @@ -183,7 +183,7 @@ The table below lists the variables and their default values that are used in th | us\_east\_1b\_subnet | The AWS subnet id for the us-east-1b subnet | `any` | n/a | yes | | us\_east\_1e\_subnet | The AWS subnet id for the us-east-1e subnet | `any` | n/a | yes | | zookeeper\_dir | The ZooKeeper directory on each EC2 node | `string` | `"/data/zookeeper"` | no | -| zookeeper\_version | The version of ZooKeeper to download and install | `string` | `"3.8.0"` | no | +| zookeeper\_version | The version of ZooKeeper to download and install | `string` | `"3.9.5"` | no | The following outputs are returned by the `aws` Terraform configuration. @@ -330,11 +330,11 @@ This Terraform configuration: and builds the software using Maven, then untars the binary tarball to `${software_root}/accumulo/accumulo-${accumulo_version}` 5. Downloads the [OpenTelemetry](https://opentelemetry.io/) Java Agent jar file and copies it to - `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar` + `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar` 6. Copies the Accumulo `test` jar to `${software_root}/accumulo/accumulo-${accumulo_version}/lib` so that `org.apache.accumulo.test.metrics.TestStatsDRegistryFactory` is on the classpath 7. Downloads the [Micrometer](https://micrometer.io/) StatsD Registry jar file and copies it to - `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.12.1.jar` + `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar` 8. Clones, if necessary, the Apache Accumulo Testing Git repo from `${accumulo_testing_repo}` into `${software_root}/sources/accumulo-testing-repo`. It switches to the `${accumulo_testing_branch_name}` branch and builds the software using Maven. diff --git a/contrib/terraform-testing-infrastructure/aws/main.tf b/contrib/terraform-testing-infrastructure/aws/main.tf index 3719b209..8e68c48f 100644 --- a/contrib/terraform-testing-infrastructure/aws/main.tf +++ b/contrib/terraform-testing-infrastructure/aws/main.tf @@ -69,6 +69,7 @@ terraform { key = "accumulo-testing/terraform.tfstate" region = "us-east-1" dynamodb_table = "accumulo-testing-tf-locks" + use_lockfile = true encrypt = true } } diff --git a/contrib/terraform-testing-infrastructure/aws/variables.tf b/contrib/terraform-testing-infrastructure/aws/variables.tf index 3100f12a..6e53953b 100644 --- a/contrib/terraform-testing-infrastructure/aws/variables.tf +++ b/contrib/terraform-testing-infrastructure/aws/variables.tf @@ -131,25 +131,25 @@ variable "accumulo_dir" { } variable "maven_version" { - default = "3.8.8" + default = "3.9.16" description = "The version of Maven to download and install" nullable = false } variable "zookeeper_version" { - default = "3.8.0" + default = "3.9.5" description = "The version of ZooKeeper to download and install" nullable = false } variable "hadoop_version" { - default = "3.3.4" + default = "3.5.0" description = "The version of Hadoop to download and install" nullable = false } variable "accumulo_version" { - default = "2.1.0-SNAPSHOT" + default = "4.0.0-SNAPSHOT" description = "The branch of Accumulo to download and install" nullable = false } diff --git a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl index 55a9ed40..ee1ff9f0 100644 --- a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl @@ -60,6 +60,7 @@ packages: - java-11-openjdk-devel %{ if centos7 == "false" ~} - java-17-openjdk-devel + - java-21-openjdk-devel %{ endif ~} - git %{ endif ~} diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/main.tf b/contrib/terraform-testing-infrastructure/modules/config-files/main.tf index 291a2777..71146a90 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/main.tf +++ b/contrib/terraform-testing-infrastructure/modules/config-files/main.tf @@ -56,6 +56,7 @@ locals { java_11_home = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-11-openjdk-amd64" : "/usr/lib/jvm/java-11-openjdk" java_17_home = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-17-openjdk-amd64" : "/usr/lib/jvm/java-17-openjdk" + java_21_home = var.os_distro == "ubuntu" ? "/usr/lib/jvm/java-21-openjdk-amd64" : "/usr/lib/jvm/java-21-openjdk" accumulo_root_pw = coalesce(var.accumulo_root_password, random_string.accumulo_root_password.result) template_vars = { @@ -64,6 +65,7 @@ locals { worker_ips = var.worker_ips java_11_home = local.java_11_home java_17_home = local.java_17_home + java_21_home = local.java_21_home accumulo_branch_name = var.accumulo_branch_name accumulo_dir = var.accumulo_dir accumulo_repo = var.accumulo_repo diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl index 1eda8f9b..8d9c0510 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/accumulo-properties.tftpl @@ -41,7 +41,7 @@ general.opentelemetry.enabled=true # Micrometer settings general.micrometer.enabled=true general.micrometer.jvm.metrics.enabled=true -general.micrometer.factory=org.apache.accumulo.test.metrics.TestStatsDRegistryFactory +general.micrometer.factory=org.apache.accumulo.core.spi.metrics.AccumuloMonitorMeterRegistryFactory,org.apache.accumulo.test.metrics.TestStatsDRegistryFactory # Per-Table Encryption instance.crypto.opts.factory=org.apache.accumulo.core.spi.crypto.PerTableCryptoServiceFactory diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl index e347c75d..91dc12fa 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/cluster.yaml.tftpl @@ -18,7 +18,9 @@ # manager: - - ${manager_ip} + servers_per_host: 1 + hosts: + - ${manager_ip} monitor: - ${manager_ip} diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl index 56894562..73707f45 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-datanode.service.tftpl @@ -22,7 +22,7 @@ Description=Hadoop DataNode start/stop After=remote-fs.target [Service] -Environment=JAVA_HOME=${java_11_home} +Environment=JAVA_HOME=${java_17_home} Environment=HADOOP_HOME=${software_root}/hadoop/hadoop-${hadoop_version} Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs User=hadoop diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl index 6d4dafe2..bdd441cc 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop-namenode.service.tftpl @@ -22,7 +22,7 @@ Description=Hadoop NameNode start/stop After=remote-fs.target [Service] -Environment=JAVA_HOME=${java_11_home} +Environment=JAVA_HOME=${java_17_home} Environment=HADOOP_HOME=${software_root}/hadoop/hadoop-${hadoop_version} Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs User=hadoop diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl index d07b2e6d..b03a5258 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl @@ -10,7 +10,7 @@ export ACCUMULO_LOG_DIR=${accumulo_dir}/logs export M2_HOME=${software_root}/apache-maven/apache-maven-${maven_version} export PDSH_RCMD_TYPE=ssh -export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar -Dotel.traces.exporter=otlp -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.otlp.endpoint=http://${manager_ip}:4317 -Dotel.exporter.otlp.protocol=grpc -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125" +export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar -Dotel.traces.exporter=otlp -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.otlp.endpoint=http://${manager_ip}:4317 -Dotel.exporter.otlp.protocol=grpc -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125" # User specific environment and startup programs PATH=$PATH:$HOME/.local/bin:$HOME/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ACCUMULO_HOME/bin:$M2_HOME/bin diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl index 4032048b..dd3aed49 100755 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/initialize_accumulo.sh.tftpl @@ -40,8 +40,8 @@ pdsh -S -g worker sudo systemctl start telegraf # # Initialize Accumulo with the supplied instance name and root user password # -accumulo init --instance-name "$1" --password "$2" -accumulo init --add-resource-groups accumulo_meta,user_small,user_large +accumulo inst init --instance-name "$1" --password "$2" +accumulo inst init --add-resource-groups accumulo_meta,user_small,user_large # # Start jaegertracing container diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl index bff423d3..9dc697e4 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl @@ -20,7 +20,8 @@ set -eo pipefail -export JAVA_HOME=${java_17_home} +# Use Java 21 for building +export JAVA_HOME=${java_21_home} SOURCES_DIR="${software_root}/sources" @@ -88,28 +89,13 @@ if [ -f $ACCUMULO_SRC ]; then else # # Download, build, and install Accumulo - echo "Cloning Accumulo-Access repo from https://github.com/apache/accumulo-access.git" - cd $SOURCES_DIR - rm -rf accumulo-access-repo - git clone https://github.com/apache/accumulo-access.git accumulo-access-repo - cd accumulo-access-repo - git checkout main - ${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -ntp clean install -PskipQA - echo "Binary tarball not found, cloning Accumulo repo from ${accumulo_repo}" cd $SOURCES_DIR rm -rf accumulo-repo git clone ${accumulo_repo} accumulo-repo cd accumulo-repo git checkout ${accumulo_branch_name} - # Java 17 required for Accumulo build - OLDPATH="$PATH" - export JAVA_HOME=${java_17_home} - export PATH=$JAVA_HOME/bin:$PATH ${software_root}/apache-maven/apache-maven-${maven_version}/bin/mvn -Dapache.snapshots=true -ntp clean install -PskipQA - # Reset JAVA HOME to Java 17 for the remainder of the script - export JAVA_HOME=${java_17_home} - export PATH=$OLDPATH mkdir -p ${software_root}/accumulo tar zxf assemble/target/accumulo-${accumulo_version}-bin.tar.gz -C ${software_root}/accumulo fi @@ -124,8 +110,8 @@ openssl rand -out ${software_root}/accumulo/accumulo-${accumulo_version}/conf/pe # # OpenTelemetry dependencies # -if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar ]; then - wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/2.26.1/opentelemetry-javaagent-2.26.1.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.26.1.jar +if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar ]; then + wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/2.31.1/opentelemetry-javaagent-2.31.1.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-2.31.1.jar fi # # Micrometer dependencies @@ -137,8 +123,8 @@ if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/accumulo-te echo "accumulo-test-${accumulo_version}.jar not found, metrics won't work..." fi fi -if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.14.5.jar ]; then - wget https://search.maven.org/remotecontent?filepath=io/micrometer/micrometer-registry-statsd/1.14.5/micrometer-registry-statsd-1.14.5.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.14.5.jar +if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar ]; then + wget https://search.maven.org/remotecontent?filepath=io/micrometer/micrometer-registry-statsd/1.16.7/micrometer-registry-statsd-1.16.7.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.16.7.jar fi # diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl index b9318069..52b8de1f 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-nodemanager.service.tftpl @@ -22,7 +22,7 @@ Description=Yarn NodeManager start/stop After=hadoop-datanode.service [Service] -Environment=JAVA_HOME=${java_11_home} +Environment=JAVA_HOME=${java_17_home} Environment=YARN_HOME=${software_root}/hadoop/hadoop-${hadoop_version} Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs User=hadoop diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl index fa7647e9..0efe7f3f 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/yarn-resourcemanager.service.tftpl @@ -22,7 +22,7 @@ Description=Yarn ResourceManager start/stop After=hadoop-namenode.service [Service] -Environment=JAVA_HOME=${java_11_home} +Environment=JAVA_HOME=${java_17_home} Environment=YARN_HOME=${software_root}/hadoop/hadoop-${hadoop_version} Environment=HADOOP_LOG_DIR=${hadoop_dir}/logs User=hadoop diff --git a/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile b/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile index b5386ac3..a6b9b685 100644 --- a/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile +++ b/contrib/terraform-testing-infrastructure/timely-grafana/Dockerfile @@ -21,12 +21,12 @@ FROM redhat/ubi8 as builder COPY build_output/maven /opt/apache-maven COPY build_output/repo /opt/apache-maven COPY settings.xml /tmp/settings.xml -ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk +ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk ENV TIMELY_VERSION="3.0.0-SNAPSHOT" -ARG HADOOP_VERSION="3.3.6" +ARG HADOOP_VERSION="3.5.0" RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ - dnf install -y java-17-openjdk-devel git wget collectd collectd-java && \ + dnf install -y java-21-openjdk-devel git wget collectd collectd-java && \ dnf clean all && \ rm -rf /var/cache/dnf @@ -38,16 +38,25 @@ RUN cd /opt && \ /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml clean install RUN cd /opt && \ - git clone https://github.com/NationalSecurityAgency/datawave-in-memory-accumulo.git datawave-in-memory-accumulo && \ - cd datawave-in-memory-accumulo && \ - git checkout accumulo4 && \ + git clone https://github.com/NationalSecurityAgency/datawave.git datawave && \ + cd datawave && \ + git checkout c1fd5d0a787bfa73301a586d0e68598426a32ed7 && \ + cd contrib/datawave-utils/read-properties && \ + /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install && \ + git checkout 7fa6fc2261cc79bce9b468d6ae8b248e44f8ff4f && \ + cd ../code-style && \ + /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install && \ + git checkout accumulo4-2026 && \ + cd ../../.. && \ + /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install -N && \ + cd core/in-memory-accumulo && \ /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean install RUN cd /opt && \ git clone https://github.com/NationalSecurityAgency/timely.git timely && \ cd timely && \ git checkout accumulo4 && \ - /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean package -Pcollectd -DskipTests + /opt/apache-maven/apache-maven-*/bin/mvn -s /tmp/settings.xml -Dversion.hadoop=${HADOOP_VERSION} clean package -Pcollectd -DskipTests -Denforcer.skip=true -Dspotbugs.skip=true RUN mkdir /tmp/timely-server && \ tar zxf /opt/timely/server/target/timely-server-${TIMELY_VERSION}-dist.tar.gz --strip-components=1 -C /tmp/timely-server && \ diff --git a/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh b/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh index 6291972b..1fd6f2c2 100755 --- a/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh +++ b/contrib/terraform-testing-infrastructure/timely-grafana/build-image.sh @@ -18,7 +18,7 @@ # under the License. # -HADOOP_VERSION=${1:-"3.3.6"} +HADOOP_VERSION=${1:-"3.5.0"} MVN_REPO=$2 MVN_DIR=$3