From 5f27c2b01353c2056fddf1d084a098b07815f0e7 Mon Sep 17 00:00:00 2001 From: AlejandroFabianCampos Date: Tue, 29 Sep 2026 12:28:38 -0300 Subject: [PATCH] feat: streams and key-value store support for 0.4.0 --- CHANGELOG.md | 17 +- README.md | 2 +- docs/data-sources/kv_list.md | 35 + docs/data-sources/kv_value.md | 35 + docs/data-sources/stream.md | 251 +++++++ docs/index.md | 2 +- docs/resources/kv_list.md | 52 ++ docs/resources/kv_list_items.md | 53 ++ docs/resources/kv_value.md | 56 ++ docs/resources/stream.md | 399 +++++++++++ .../quicknode_kv_list/data-source.tf | 8 + .../quicknode_kv_value/data-source.tf | 8 + .../quicknode_stream/data-source.tf | 13 + examples/provider/provider.tf | 2 +- .../resources/quicknode_kv_list/import.sh | 2 + .../resources/quicknode_kv_list/resource.tf | 10 + .../quicknode_kv_list_items/import.sh | 4 + .../quicknode_kv_list_items/resource.tf | 9 + .../resources/quicknode_kv_value/import.sh | 2 + .../resources/quicknode_kv_value/resource.tf | 14 + examples/resources/quicknode_stream/import.sh | 3 + .../resources/quicknode_stream/resource.tf | 112 +++ internal/client/client.go | 13 +- internal/client/kv.go | 131 ++++ internal/client/kv_test.go | 189 +++++ internal/client/rest.go | 86 +++ internal/client/streams.go | 525 ++++++++++++++ internal/client/streams_test.go | 201 ++++++ internal/client/transport.go | 4 + internal/provider/kv.go | 109 +++ internal/provider/kv_acceptance_test.go | 326 +++++++++ internal/provider/kv_list_data_source.go | 72 ++ internal/provider/kv_list_items_resource.go | 206 ++++++ internal/provider/kv_list_resource.go | 178 +++++ internal/provider/kv_value_data_source.go | 65 ++ internal/provider/kv_value_resource.go | 150 ++++ internal/provider/provider.go | 8 + internal/provider/stream_acceptance_test.go | 237 +++++++ internal/provider/stream_data_source.go | 195 ++++++ internal/provider/stream_destination.go | 589 ++++++++++++++++ internal/provider/stream_resource.go | 650 ++++++++++++++++++ internal/provider/user_agent.go | 33 + internal/provider/user_agent_test.go | 20 + 43 files changed, 5070 insertions(+), 6 deletions(-) create mode 100644 docs/data-sources/kv_list.md create mode 100644 docs/data-sources/kv_value.md create mode 100644 docs/data-sources/stream.md create mode 100644 docs/resources/kv_list.md create mode 100644 docs/resources/kv_list_items.md create mode 100644 docs/resources/kv_value.md create mode 100644 docs/resources/stream.md create mode 100644 examples/data-sources/quicknode_kv_list/data-source.tf create mode 100644 examples/data-sources/quicknode_kv_value/data-source.tf create mode 100644 examples/data-sources/quicknode_stream/data-source.tf create mode 100644 examples/resources/quicknode_kv_list/import.sh create mode 100644 examples/resources/quicknode_kv_list/resource.tf create mode 100644 examples/resources/quicknode_kv_list_items/import.sh create mode 100644 examples/resources/quicknode_kv_list_items/resource.tf create mode 100644 examples/resources/quicknode_kv_value/import.sh create mode 100644 examples/resources/quicknode_kv_value/resource.tf create mode 100644 examples/resources/quicknode_stream/import.sh create mode 100644 examples/resources/quicknode_stream/resource.tf create mode 100644 internal/client/kv.go create mode 100644 internal/client/kv_test.go create mode 100644 internal/client/rest.go create mode 100644 internal/client/streams.go create mode 100644 internal/client/streams_test.go create mode 100644 internal/provider/kv.go create mode 100644 internal/provider/kv_acceptance_test.go create mode 100644 internal/provider/kv_list_data_source.go create mode 100644 internal/provider/kv_list_items_resource.go create mode 100644 internal/provider/kv_list_resource.go create mode 100644 internal/provider/kv_value_data_source.go create mode 100644 internal/provider/kv_value_resource.go create mode 100644 internal/provider/stream_acceptance_test.go create mode 100644 internal/provider/stream_data_source.go create mode 100644 internal/provider/stream_destination.go create mode 100644 internal/provider/stream_resource.go create mode 100644 internal/provider/user_agent.go create mode 100644 internal/provider/user_agent_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 255fdae..02cef65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,21 @@ # Changelog -## 0.3.0 (Unreleased) +## 0.4.0 + +FEATURES: + +* **New Resource:** `quicknode_stream`: webhook, S3, Azure, PostgreSQL and Kafka + destinations, extra destinations, and filters as plain source. +* **New Resource:** `quicknode_kv_list` +* **New Resource:** `quicknode_kv_list_items` +* **New Resource:** `quicknode_kv_value` +* **New Data Source:** `quicknode_stream` +* **New Data Source:** `quicknode_kv_list` +* **New Data Source:** `quicknode_kv_value` +* Requests send a `quicknode-terraform/` User-Agent with the platform and + Terraform version. + +## 0.3.0 (September 25, 2026) BREAKING CHANGES: diff --git a/README.md b/README.md index 4ccb707..fd2cbeb 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ terraform { required_providers { quicknode = { source = "quicknode/quicknode" - version = "~> 0.3.0" + version = "~> 0.4.0" } } } diff --git a/docs/data-sources/kv_list.md b/docs/data-sources/kv_list.md new file mode 100644 index 0000000..a9f826b --- /dev/null +++ b/docs/data-sources/kv_list.md @@ -0,0 +1,35 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "quicknode_kv_list Data Source - quicknode" +subcategory: "" +description: |- + Items in a Quicknode Key-Value Store list. A missing list reads as empty. +--- + +# quicknode_kv_list (Data Source) + +Items in a Quicknode Key-Value Store list. A missing list reads as empty. + +## Example Usage + +```terraform +# Read a list that a Streams filter maintains. +data "quicknode_kv_list" "tracked_wallets" { + key = "tracked-wallets" +} + +output "tracked_wallet_count" { + value = length(data.quicknode_kv_list.tracked_wallets.items) +} +``` + + +## Schema + +### Required + +- `key` (String) Name of the list. + +### Read-Only + +- `items` (Set of String) Items in the list. diff --git a/docs/data-sources/kv_value.md b/docs/data-sources/kv_value.md new file mode 100644 index 0000000..a184d2b --- /dev/null +++ b/docs/data-sources/kv_value.md @@ -0,0 +1,35 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "quicknode_kv_value Data Source - quicknode" +subcategory: "" +description: |- + A value in the Quicknode Key-Value Store. Reading a missing key is an error. +--- + +# quicknode_kv_value (Data Source) + +A value in the Quicknode Key-Value Store. Reading a missing key is an error. + +## Example Usage + +```terraform +# Read a value that a Streams filter writes, such as the last block processed. +data "quicknode_kv_value" "last_block" { + key = "last-processed-block" +} + +output "last_processed_block" { + value = tonumber(data.quicknode_kv_value.last_block.value) +} +``` + + +## Schema + +### Required + +- `key` (String) Name of the value. + +### Read-Only + +- `value` (String) The stored value. diff --git a/docs/data-sources/stream.md b/docs/data-sources/stream.md new file mode 100644 index 0000000..b2ef4c7 --- /dev/null +++ b/docs/data-sources/stream.md @@ -0,0 +1,251 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "quicknode_stream Data Source - quicknode" +subcategory: "" +description: |- + An existing stream, looked up by id. Destination credentials are stored in Terraform state, so keep state encrypted and remote. +--- + +# quicknode_stream (Data Source) + +An existing stream, looked up by `id`. Destination credentials are stored in Terraform state, so keep state encrypted and remote. + +## Example Usage + +```terraform +# Look up a stream created outside Terraform. +data "quicknode_stream" "transfers" { + id = "0b4a6f8e-3c1d-4e2a-9f7b-5d6c8e1a2b3c" +} + +# The last block the stream delivered. It moves while the stream runs. +output "transfers_sequence" { + value = data.quicknode_stream.transfers.sequence +} + +output "transfers_state" { + value = data.quicknode_stream.transfers.status +} +``` + + +## Schema + +### Required + +- `id` (String) Stream id. + +### Read-Only + +- `dataset` (String) What each batch carries. +- `dataset_batch_size` (Number) Blocks per batch. +- `destination` (Attributes) Where the stream delivers. One type is set; the others are null. (see [below for nested schema](#nestedatt--destination)) +- `elastic_batch_enabled` (Boolean) Whether batches grow while the stream catches up. +- `end_range` (Number) Last block the stream delivers, or null if it runs until paused. +- `extra_destinations` (Attributes List) More destinations that receive every batch, or null if there are none. (see [below for nested schema](#nestedatt--extra_destinations)) +- `filter_function` (String) Source code of the filter, or null if the stream has none. +- `filter_language` (String) `javascript` or `go`. +- `fix_block_reorgs` (Boolean) Whether corrected blocks are delivered when the network reorganizes. +- `keep_distance_from_tip` (Number) How many blocks the stream stays behind the newest block. +- `name` (String) Name shown in the dashboard. +- `network` (String) Network slug. +- `notification_email` (String) Address notified when the stream is terminated, or null. +- `region` (String) Region the stream runs in. +- `restream_batch_on_reorg` (Boolean) Whether the whole batch is delivered again when a block in it is reorganized. +- `sequence` (Number) Last block delivered. It moves while the stream runs. +- `start_range` (Number) First block the stream delivers. +- `status` (String) `active`, `paused`, `terminated` or `completed`. + + +### Nested Schema for `destination` + +Read-Only: + +- `azure` (Attributes) Settings when the destination is `azure`, otherwise null. (see [below for nested schema](#nestedatt--destination--azure)) +- `kafka` (Attributes) Settings when the destination is `kafka`, otherwise null. (see [below for nested schema](#nestedatt--destination--kafka)) +- `postgres` (Attributes) Settings when the destination is `postgres`, otherwise null. (see [below for nested schema](#nestedatt--destination--postgres)) +- `s3` (Attributes) Settings when the destination is `s3`, otherwise null. (see [below for nested schema](#nestedatt--destination--s3)) +- `webhook` (Attributes) Settings when the destination is `webhook`, otherwise null. (see [below for nested schema](#nestedatt--destination--webhook)) + + +### Nested Schema for `destination.azure` + +Read-Only: + +- `blob_prefix` (String) See `destination.azure.blob_prefix` on the `quicknode_stream` resource. +- `compression` (String) See `destination.azure.compression` on the `quicknode_stream` resource. +- `container` (String) See `destination.azure.container` on the `quicknode_stream` resource. +- `file_type` (String) See `destination.azure.file_type` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.azure.max_retry` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.azure.retry_interval_sec` on the `quicknode_stream` resource. +- `sas_token` (String, Sensitive) See `destination.azure.sas_token` on the `quicknode_stream` resource. +- `storage_account` (String) See `destination.azure.storage_account` on the `quicknode_stream` resource. + + + +### Nested Schema for `destination.kafka` + +Read-Only: + +- `batch_size` (Number) See `destination.kafka.batch_size` on the `quicknode_stream` resource. +- `bootstrap_servers` (String) See `destination.kafka.bootstrap_servers` on the `quicknode_stream` resource. +- `compression` (String) See `destination.kafka.compression` on the `quicknode_stream` resource. +- `linger_ms` (Number) See `destination.kafka.linger_ms` on the `quicknode_stream` resource. +- `max_message_bytes` (Number) See `destination.kafka.max_message_bytes` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.kafka.max_retry` on the `quicknode_stream` resource. +- `mechanisms` (String) See `destination.kafka.mechanisms` on the `quicknode_stream` resource. +- `password` (String, Sensitive) See `destination.kafka.password` on the `quicknode_stream` resource. +- `protocol` (String) See `destination.kafka.protocol` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.kafka.retry_interval_sec` on the `quicknode_stream` resource. +- `ssl_ca_pem` (String) See `destination.kafka.ssl_ca_pem` on the `quicknode_stream` resource. +- `ssl_certificate_pem` (String) See `destination.kafka.ssl_certificate_pem` on the `quicknode_stream` resource. +- `ssl_key_pem` (String, Sensitive) See `destination.kafka.ssl_key_pem` on the `quicknode_stream` resource. +- `timeout_sec` (Number) See `destination.kafka.timeout_sec` on the `quicknode_stream` resource. +- `topic_name` (String) See `destination.kafka.topic_name` on the `quicknode_stream` resource. +- `username` (String) See `destination.kafka.username` on the `quicknode_stream` resource. + + + +### Nested Schema for `destination.postgres` + +Read-Only: + +- `database` (String) See `destination.postgres.database` on the `quicknode_stream` resource. +- `host` (String) See `destination.postgres.host` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.postgres.max_retry` on the `quicknode_stream` resource. +- `password` (String, Sensitive) See `destination.postgres.password` on the `quicknode_stream` resource. +- `port` (Number) See `destination.postgres.port` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.postgres.retry_interval_sec` on the `quicknode_stream` resource. +- `sslmode` (String) See `destination.postgres.sslmode` on the `quicknode_stream` resource. +- `table_name` (String) See `destination.postgres.table_name` on the `quicknode_stream` resource. +- `username` (String) See `destination.postgres.username` on the `quicknode_stream` resource. + + + +### Nested Schema for `destination.s3` + +Read-Only: + +- `access_key` (String) See `destination.s3.access_key` on the `quicknode_stream` resource. +- `bucket` (String) See `destination.s3.bucket` on the `quicknode_stream` resource. +- `compression` (String) See `destination.s3.compression` on the `quicknode_stream` resource. +- `endpoint` (String) See `destination.s3.endpoint` on the `quicknode_stream` resource. +- `file_type` (String) See `destination.s3.file_type` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.s3.max_retry` on the `quicknode_stream` resource. +- `object_prefix` (String) See `destination.s3.object_prefix` on the `quicknode_stream` resource. +- `region` (String) See `destination.s3.region` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.s3.retry_interval_sec` on the `quicknode_stream` resource. +- `secret_key` (String, Sensitive) See `destination.s3.secret_key` on the `quicknode_stream` resource. +- `use_ssl` (Boolean) See `destination.s3.use_ssl` on the `quicknode_stream` resource. + + + +### Nested Schema for `destination.webhook` + +Read-Only: + +- `compression` (String) See `destination.webhook.compression` on the `quicknode_stream` resource. +- `headers` (Map of String) See `destination.webhook.headers` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.webhook.max_retry` on the `quicknode_stream` resource. +- `post_timeout_sec` (Number) See `destination.webhook.post_timeout_sec` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.webhook.retry_interval_sec` on the `quicknode_stream` resource. +- `security_token` (String, Sensitive) See `destination.webhook.security_token` on the `quicknode_stream` resource. +- `url` (String) See `destination.webhook.url` on the `quicknode_stream` resource. + + + + +### Nested Schema for `extra_destinations` + +Read-Only: + +- `azure` (Attributes) Settings when the destination is `azure`, otherwise null. (see [below for nested schema](#nestedatt--extra_destinations--azure)) +- `kafka` (Attributes) Settings when the destination is `kafka`, otherwise null. (see [below for nested schema](#nestedatt--extra_destinations--kafka)) +- `postgres` (Attributes) Settings when the destination is `postgres`, otherwise null. (see [below for nested schema](#nestedatt--extra_destinations--postgres)) +- `s3` (Attributes) Settings when the destination is `s3`, otherwise null. (see [below for nested schema](#nestedatt--extra_destinations--s3)) +- `webhook` (Attributes) Settings when the destination is `webhook`, otherwise null. (see [below for nested schema](#nestedatt--extra_destinations--webhook)) + + +### Nested Schema for `extra_destinations.azure` + +Read-Only: + +- `blob_prefix` (String) See `destination.azure.blob_prefix` on the `quicknode_stream` resource. +- `compression` (String) See `destination.azure.compression` on the `quicknode_stream` resource. +- `container` (String) See `destination.azure.container` on the `quicknode_stream` resource. +- `file_type` (String) See `destination.azure.file_type` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.azure.max_retry` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.azure.retry_interval_sec` on the `quicknode_stream` resource. +- `sas_token` (String, Sensitive) See `destination.azure.sas_token` on the `quicknode_stream` resource. +- `storage_account` (String) See `destination.azure.storage_account` on the `quicknode_stream` resource. + + + +### Nested Schema for `extra_destinations.kafka` + +Read-Only: + +- `batch_size` (Number) See `destination.kafka.batch_size` on the `quicknode_stream` resource. +- `bootstrap_servers` (String) See `destination.kafka.bootstrap_servers` on the `quicknode_stream` resource. +- `compression` (String) See `destination.kafka.compression` on the `quicknode_stream` resource. +- `linger_ms` (Number) See `destination.kafka.linger_ms` on the `quicknode_stream` resource. +- `max_message_bytes` (Number) See `destination.kafka.max_message_bytes` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.kafka.max_retry` on the `quicknode_stream` resource. +- `mechanisms` (String) See `destination.kafka.mechanisms` on the `quicknode_stream` resource. +- `password` (String, Sensitive) See `destination.kafka.password` on the `quicknode_stream` resource. +- `protocol` (String) See `destination.kafka.protocol` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.kafka.retry_interval_sec` on the `quicknode_stream` resource. +- `ssl_ca_pem` (String) See `destination.kafka.ssl_ca_pem` on the `quicknode_stream` resource. +- `ssl_certificate_pem` (String) See `destination.kafka.ssl_certificate_pem` on the `quicknode_stream` resource. +- `ssl_key_pem` (String, Sensitive) See `destination.kafka.ssl_key_pem` on the `quicknode_stream` resource. +- `timeout_sec` (Number) See `destination.kafka.timeout_sec` on the `quicknode_stream` resource. +- `topic_name` (String) See `destination.kafka.topic_name` on the `quicknode_stream` resource. +- `username` (String) See `destination.kafka.username` on the `quicknode_stream` resource. + + + +### Nested Schema for `extra_destinations.postgres` + +Read-Only: + +- `database` (String) See `destination.postgres.database` on the `quicknode_stream` resource. +- `host` (String) See `destination.postgres.host` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.postgres.max_retry` on the `quicknode_stream` resource. +- `password` (String, Sensitive) See `destination.postgres.password` on the `quicknode_stream` resource. +- `port` (Number) See `destination.postgres.port` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.postgres.retry_interval_sec` on the `quicknode_stream` resource. +- `sslmode` (String) See `destination.postgres.sslmode` on the `quicknode_stream` resource. +- `table_name` (String) See `destination.postgres.table_name` on the `quicknode_stream` resource. +- `username` (String) See `destination.postgres.username` on the `quicknode_stream` resource. + + + +### Nested Schema for `extra_destinations.s3` + +Read-Only: + +- `access_key` (String) See `destination.s3.access_key` on the `quicknode_stream` resource. +- `bucket` (String) See `destination.s3.bucket` on the `quicknode_stream` resource. +- `compression` (String) See `destination.s3.compression` on the `quicknode_stream` resource. +- `endpoint` (String) See `destination.s3.endpoint` on the `quicknode_stream` resource. +- `file_type` (String) See `destination.s3.file_type` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.s3.max_retry` on the `quicknode_stream` resource. +- `object_prefix` (String) See `destination.s3.object_prefix` on the `quicknode_stream` resource. +- `region` (String) See `destination.s3.region` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.s3.retry_interval_sec` on the `quicknode_stream` resource. +- `secret_key` (String, Sensitive) See `destination.s3.secret_key` on the `quicknode_stream` resource. +- `use_ssl` (Boolean) See `destination.s3.use_ssl` on the `quicknode_stream` resource. + + + +### Nested Schema for `extra_destinations.webhook` + +Read-Only: + +- `compression` (String) See `destination.webhook.compression` on the `quicknode_stream` resource. +- `headers` (Map of String) See `destination.webhook.headers` on the `quicknode_stream` resource. +- `max_retry` (Number) See `destination.webhook.max_retry` on the `quicknode_stream` resource. +- `post_timeout_sec` (Number) See `destination.webhook.post_timeout_sec` on the `quicknode_stream` resource. +- `retry_interval_sec` (Number) See `destination.webhook.retry_interval_sec` on the `quicknode_stream` resource. +- `security_token` (String, Sensitive) See `destination.webhook.security_token` on the `quicknode_stream` resource. +- `url` (String) See `destination.webhook.url` on the `quicknode_stream` resource. diff --git a/docs/index.md b/docs/index.md index e4cb67d..c019188 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,7 +26,7 @@ terraform { required_providers { quicknode = { source = "quicknode/quicknode" - version = "~> 0.3.0" + version = "~> 0.4.0" } } } diff --git a/docs/resources/kv_list.md b/docs/resources/kv_list.md new file mode 100644 index 0000000..5f2f4fe --- /dev/null +++ b/docs/resources/kv_list.md @@ -0,0 +1,52 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "quicknode_kv_list Resource - quicknode" +subcategory: "" +description: |- + A list in the Quicknode Key-Value Store. Streams filters can read and update it, for example a set of wallet addresses to watch. + This resource owns the whole list: items added outside Terraform are removed on the next apply. For a list that a filter or another tool also writes to, use quicknode_kv_list_items. Don't manage the same list with both. +--- + +# quicknode_kv_list (Resource) + +A list in the Quicknode Key-Value Store. Streams filters can read and update it, for example a set of wallet addresses to watch. + +This resource owns the whole list: items added outside Terraform are removed on the next apply. For a list that a filter or another tool also writes to, use `quicknode_kv_list_items`. Don't manage the same list with both. + +## Example Usage + +```terraform +variable "watched_wallets" { + type = set(string) +} + +# Terraform owns the whole list. A Streams filter can read it with +# qnLib.qnContainsListItems("watched-wallets", [...]). +resource "quicknode_kv_list" "watched_wallets" { + key = "watched-wallets" + items = var.watched_wallets +} +``` + + +## Schema + +### Required + +- `items` (Set of String) Items in the list. Items are case-sensitive, so `0xABC` and `0xabc` are different items. Changes are sent as additions and removals, up to 1500 items per request. +- `key` (String) Name of the list, as filters refer to it. Up to 255 letters, digits, spaces and `.` `:` `_` `-` `$`. Changing it replaces the resource. + +### Read-Only + +- `id` (String) Same as `key`. + +## Import + +Import is supported using the following syntax: + +The [`terraform import` command](https://developer.hashicorp.com/terraform/cli/commands/import) can be used, for example: + +```shell +# Lists are imported by their key. +terraform import quicknode_kv_list.watched_wallets watched-wallets +``` diff --git a/docs/resources/kv_list_items.md b/docs/resources/kv_list_items.md new file mode 100644 index 0000000..d6f5ead --- /dev/null +++ b/docs/resources/kv_list_items.md @@ -0,0 +1,53 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "quicknode_kv_list_items Resource - quicknode" +subcategory: "" +description: |- + Items in a Quicknode Key-Value Store list. Terraform manages only these items and leaves the rest of the list alone, for lists that Streams filters or other tools also write to. Adding items to a missing list creates it. + Create fails if any of the items are already in the list, so destroy never removes items Terraform didn't add. To take over existing items, import the resource and apply. Don't use it with a quicknode_kv_list for the same key, since that resource removes items it doesn't know about. +--- + +# quicknode_kv_list_items (Resource) + +Items in a Quicknode Key-Value Store list. Terraform manages only these items and leaves the rest of the list alone, for lists that Streams filters or other tools also write to. Adding items to a missing list creates it. + +Create fails if any of the items are already in the list, so destroy never removes items Terraform didn't add. To take over existing items, import the resource and apply. Don't use it with a `quicknode_kv_list` for the same key, since that resource removes items it doesn't know about. + +## Example Usage + +```terraform +# A list that a Streams filter also adds to. Terraform tracks only these items +# and leaves the filter's items in place. +resource "quicknode_kv_list_items" "treasury" { + list_key = "tracked-wallets" + items = [ + "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045", + "0x71C7656EC7ab88b098defB751B7401B5f6d8976F", + ] +} +``` + + +## Schema + +### Required + +- `items` (Set of String) Items this resource adds to the list. Items are case-sensitive, so `0xABC` and `0xabc` are different items. Changes are sent as additions and removals, up to 1500 items per request. Items removed outside Terraform are added back on the next apply. +- `list_key` (String) Name of the list the items belong to. Up to 255 letters, digits, spaces and `.` `:` `_` `-` `$`. Changing it replaces the resource. + +### Read-Only + +- `id` (String) Same as `list_key`. + +## Import + +Import is supported using the following syntax: + +The [`terraform import` command](https://developer.hashicorp.com/terraform/cli/commands/import) can be used, for example: + +```shell +# Import by list key to take over items already in the list. The import starts +# with no items; the next apply takes over the configured ones and leaves the +# rest alone. +terraform import quicknode_kv_list_items.treasury tracked-wallets +``` diff --git a/docs/resources/kv_value.md b/docs/resources/kv_value.md new file mode 100644 index 0000000..72b7669 --- /dev/null +++ b/docs/resources/kv_value.md @@ -0,0 +1,56 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "quicknode_kv_value Resource - quicknode" +subcategory: "" +description: |- + A value in the Quicknode Key-Value Store. Streams filters can read and update it, for example a threshold or the last block processed. + A filter that writes the value shows up as a change, and the next apply sets it back. Values and lists are separate, so a value and a list can share a key. +--- + +# quicknode_kv_value (Resource) + +A value in the Quicknode Key-Value Store. Streams filters can read and update it, for example a threshold or the last block processed. + +A filter that writes the value shows up as a change, and the next apply sets it back. Values and lists are separate, so a value and a list can share a key. + +## Example Usage + +```terraform +# A setting a Streams filter reads with qnLib.qnGetValue("alert-threshold"). +resource "quicknode_kv_value" "alert_threshold" { + key = "alert-threshold" + value = "1000000" +} + +# Values are strings; use jsonencode for structured settings. +resource "quicknode_kv_value" "alert_config" { + key = "alert-config" + value = jsonencode({ + min_value_usd = 50000 + tokens = ["USDC", "USDT"] + }) +} +``` + + +## Schema + +### Required + +- `key` (String) Name of the value, as filters refer to it. Up to 255 letters, digits, spaces and `.` `:` `_` `-` `$`. Changing it replaces the resource. +- `value` (String) The value, as a string. Use `jsonencode` for structured data. Terraform state holds it in plain text. + +### Read-Only + +- `id` (String) Same as `key`. + +## Import + +Import is supported using the following syntax: + +The [`terraform import` command](https://developer.hashicorp.com/terraform/cli/commands/import) can be used, for example: + +```shell +# Values are imported by their key. +terraform import quicknode_kv_value.alert_threshold alert-threshold +``` diff --git a/docs/resources/stream.md b/docs/resources/stream.md new file mode 100644 index 0000000..86bf040 --- /dev/null +++ b/docs/resources/stream.md @@ -0,0 +1,399 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "quicknode_stream Resource - quicknode" +subcategory: "" +description: |- + A Quicknode Stream. It delivers blockchain data from a network to one or more destinations. + Quicknode can stop a stream by itself: terminated when a destination keeps failing or the account's plan no longer allows it, and completed after it delivers end_range. state reports which. For a terminated stream the plan changes status back to the configured value, and applying it resumes or pauses the stream. A completed stream can't be changed or resumed, so any change replaces it. +--- + +# quicknode_stream (Resource) + +A Quicknode Stream. It delivers blockchain data from a network to one or more destinations. + +Quicknode can stop a stream by itself: `terminated` when a destination keeps failing or the account's plan no longer allows it, and `completed` after it delivers `end_range`. `state` reports which. For a terminated stream the plan changes `status` back to the configured value, and applying it resumes or pauses the stream. A completed stream can't be changed or resumed, so any change replaces it. + +## Example Usage + +```terraform +variable "archive_secret_key" { + type = string + sensitive = true +} + +variable "archive_sas_token" { + type = string + sensitive = true +} + +variable "database_password" { + type = string + sensitive = true +} + +variable "kafka_password" { + type = string + sensitive = true +} + +resource "quicknode_stream" "transfers" { + name = "usdc-transfers" + network = "ethereum-mainnet" + dataset = "receipts" + region = "usa_east" + status = "active" + + # The filter is plain source; the provider encodes it for the API. + filter_function = file("${path.module}/filter.js") + + # Stay behind the newest block so reorganized blocks are never delivered. + keep_distance_from_tip = 3 + + destination = { + webhook = { + url = "https://ingest.example.com/quicknode" + headers = { "X-Source" = "quicknode-streams" } + } + } + + # Every batch is also delivered to each of these. The stream moves on only + # after every destination accepts a batch. + extra_destinations = [ + { + s3 = { + bucket = "chain-archive" + region = "us-east-1" + object_prefix = "usdc-transfers/" + compression = "gzip" + access_key = "AKIAIOSFODNN7EXAMPLE" + secret_key = var.archive_secret_key + } + }, + { + azure = { + storage_account = "chainarchive" + container = "usdc-transfers" + file_type = ".parquet" + sas_token = var.archive_sas_token + } + }, + { + postgres = { + host = "db.example.com" + database = "chain" + table_name = "usdc_transfers" + username = "streams" + password = var.database_password + } + }, + { + kafka = { + bootstrap_servers = "broker-1.example.com:9092,broker-2.example.com:9092" + topic_name = "usdc-transfers" + protocol = "sasl_ssl" + mechanisms = "SCRAM-SHA-512" + username = "streams" + password = var.kafka_password + } + }, + ] +} + +# Quicknode signs each webhook request with this token, so the receiver can +# check it came from Quicknode. +output "webhook_security_token" { + value = quicknode_stream.transfers.destination.webhook.security_token + sensitive = true +} + +# A backfill of a fixed block range. It stops as completed after end_range, +# and any later change replaces it. +resource "quicknode_stream" "backfill" { + name = "usdc-transfers-backfill" + network = "ethereum-mainnet" + dataset = "receipts" + region = "usa_east" + status = "active" + start_range = 21000000 + end_range = 21100000 + + dataset_batch_size = 10 + + filter_function = file("${path.module}/filter.js") + + destination = { + webhook = { + url = "https://ingest.example.com/quicknode/backfill" + compression = "gzip" + } + } +} +``` + + +## Schema + +### Required + +- `dataset` (String) What each batch carries, for example `block`, `block_with_receipts`, `receipts`, `logs`, `transactions` or `trace_blocks`. The datasets available depend on the network. Changing it replaces the stream. +- `destination` (Attributes) Where the stream delivers. Set exactly one type. Changing the type replaces the stream. (see [below for nested schema](#nestedatt--destination)) +- `name` (String) Name shown in the dashboard. Names are not unique. +- `network` (String) Network slug, for example `ethereum-mainnet` or `solana-devnet`. Changing it replaces the stream. +- `region` (String) Region the stream runs in: `usa_east`, `europe_central` or `asia_east`. Changing it replaces the stream. +- `status` (String) `active` or `paused`. Required, so importing a stream never starts or stops it. + +### Optional + +- `dataset_batch_size` (Number) Blocks per batch. Some fast networks require a larger minimum. Defaults to `1`. +- `elastic_batch_enabled` (Boolean) Grow batches while the stream catches up to the newest block. Only allowed without `end_range`. Defaults to `false`. +- `end_range` (Number) Last block to deliver. When set, the stream stops as `completed` after delivering it. When omitted, the stream runs until it is paused or deleted. +- `extra_destinations` (Attributes List) More destinations that receive every batch, each set like `destination`. Types can repeat. The stream moves on only after every destination accepts a batch, and stops if one keeps failing. The limit depends on the account's plan. (see [below for nested schema](#nestedatt--extra_destinations)) +- `filter_function` (String) Source code of the filter that shapes each batch, as plain text. Read it from a file with `file("filter.js")`. Quicknode runs the filter against `start_range` when the stream is created and rejects the stream if it fails. +- `filter_language` (String) Language of `filter_function`: `javascript` or `go`. Defaults to `javascript`. +- `fix_block_reorgs` (Boolean) Deliver corrected blocks when the network reorganizes. Depends on the account's plan. Defaults to `false`. +- `keep_distance_from_tip` (Number) Stay this many blocks behind the newest block, so reorganized blocks are never delivered. Depends on the account's plan. +- `notification_email` (String) Address notified when the stream is terminated. +- `restream_batch_on_reorg` (Boolean) Deliver the whole batch again when a block in it is reorganized. +- `resume_from` (String) Where a paused or terminated stream continues when `status` changes to `active`: `last` continues after the last block delivered, `latest` skips ahead to the newest block. `latest` moves `start_range`, so it cannot be combined with a configured `start_range`. Defaults to `last`. +- `start_range` (Number) First block to deliver. When omitted, the stream starts at the newest block and this attribute reports which one. Changing it moves the stream to that block, which can deliver blocks again or skip them. + +### Read-Only + +- `id` (String) Stream id. +- `state` (String) The stream's actual status: `active`, `paused`, `terminated` or `completed`. + + +### Nested Schema for `destination` + +Optional: + +- `azure` (Attributes) Write each batch as a blob to Azure Storage. (see [below for nested schema](#nestedatt--destination--azure)) +- `kafka` (Attributes) Produce each batch as messages to a Kafka topic. (see [below for nested schema](#nestedatt--destination--kafka)) +- `postgres` (Attributes) Insert each batch into a PostgreSQL table. (see [below for nested schema](#nestedatt--destination--postgres)) +- `s3` (Attributes) Write each batch as an object to Amazon S3 or an S3-compatible store. (see [below for nested schema](#nestedatt--destination--s3)) +- `webhook` (Attributes) Deliver each batch as an HTTP POST. (see [below for nested schema](#nestedatt--destination--webhook)) + + +### Nested Schema for `destination.azure` + +Required: + +- `container` (String) Container name. +- `sas_token` (String, Sensitive) Shared access signature token with write access to the container. +- `storage_account` (String) Storage account name. + +Optional: + +- `blob_prefix` (String) Prefix added to each blob name. Defaults to ``. +- `compression` (String) `none` or `gzip`. Defaults to `none`. +- `file_type` (String) `.json` or `.parquet`. Defaults to `.json`. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. + + + +### Nested Schema for `destination.kafka` + +Required: + +- `bootstrap_servers` (String) Comma-separated `host:port` list of brokers. +- `topic_name` (String) Topic the messages are produced to. + +Optional: + +- `batch_size` (Number) Producer batch size in bytes. Defaults to `16384`. +- `compression` (String) `none`, `gzip`, `snappy`, `lz4` or `zstd`. Defaults to `zstd`. +- `linger_ms` (Number) Milliseconds the producer waits to fill a batch. Defaults to `5`. +- `max_message_bytes` (Number) Largest message the producer sends, in bytes. Defaults to `1048576`. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `mechanisms` (String) SASL mechanism: `PLAIN`, `GSSAPI`, `SCRAM-SHA-256`, `SCRAM-SHA-512` or `OAUTHBEARER`. +- `password` (String, Sensitive) SASL password. +- `protocol` (String) `plaintext`, `ssl`, `sasl_ssl` or `sasl_plaintext`. Defaults to `plaintext`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `ssl_ca_pem` (String) PEM-encoded CA certificate that signed the brokers' certificates. +- `ssl_certificate_pem` (String) PEM-encoded client certificate. +- `ssl_key_pem` (String, Sensitive) PEM-encoded client private key. +- `timeout_sec` (Number) Seconds to wait for the brokers to acknowledge. Defaults to `30`. +- `username` (String) SASL user. + + + +### Nested Schema for `destination.postgres` + +Required: + +- `database` (String) Database name. +- `host` (String) Database host. +- `password` (String, Sensitive) Database password. +- `table_name` (String) Table the rows are written to. +- `username` (String) Database user. + +Optional: + +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `port` (Number) Database port. Defaults to `5432`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `sslmode` (String) `require` or `disable`. Defaults to `require`. + + + +### Nested Schema for `destination.s3` + +Required: + +- `access_key` (String) Access key id. +- `bucket` (String) Bucket name. +- `secret_key` (String, Sensitive) Secret access key. + +Optional: + +- `compression` (String) `none` or `gzip`. Defaults to `none`. +- `endpoint` (String) S3 endpoint host. Defaults to `s3.amazonaws.com`. +- `file_type` (String) `.json` or `.parquet`. Defaults to `.json`. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `object_prefix` (String) Prefix added to each object name. It is joined to the file name with no separator, so end it with `/` to write into a folder. Defaults to ``. +- `region` (String) Bucket region, for example `us-east-1`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `use_ssl` (Boolean) Connect to the endpoint over TLS. Defaults to `true`. + + + +### Nested Schema for `destination.webhook` + +Required: + +- `url` (String) URL the batches are posted to. + +Optional: + +- `compression` (String) `none` or `gzip`. Defaults to `none`. +- `headers` (Map of String) Extra request headers. Omit the attribute rather than setting an empty map. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `post_timeout_sec` (Number) Seconds to wait for the receiver to respond. Defaults to `30`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `security_token` (String, Sensitive) Token used to sign each request, so the receiver can verify it came from Quicknode. When omitted on the primary destination, Quicknode generates one and this attribute reports it. + + + + +### Nested Schema for `extra_destinations` + +Optional: + +- `azure` (Attributes) Write each batch as a blob to Azure Storage. (see [below for nested schema](#nestedatt--extra_destinations--azure)) +- `kafka` (Attributes) Produce each batch as messages to a Kafka topic. (see [below for nested schema](#nestedatt--extra_destinations--kafka)) +- `postgres` (Attributes) Insert each batch into a PostgreSQL table. (see [below for nested schema](#nestedatt--extra_destinations--postgres)) +- `s3` (Attributes) Write each batch as an object to Amazon S3 or an S3-compatible store. (see [below for nested schema](#nestedatt--extra_destinations--s3)) +- `webhook` (Attributes) Deliver each batch as an HTTP POST. (see [below for nested schema](#nestedatt--extra_destinations--webhook)) + + +### Nested Schema for `extra_destinations.azure` + +Required: + +- `container` (String) Container name. +- `sas_token` (String, Sensitive) Shared access signature token with write access to the container. +- `storage_account` (String) Storage account name. + +Optional: + +- `blob_prefix` (String) Prefix added to each blob name. Defaults to ``. +- `compression` (String) `none` or `gzip`. Defaults to `none`. +- `file_type` (String) `.json` or `.parquet`. Defaults to `.json`. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. + + + +### Nested Schema for `extra_destinations.kafka` + +Required: + +- `bootstrap_servers` (String) Comma-separated `host:port` list of brokers. +- `topic_name` (String) Topic the messages are produced to. + +Optional: + +- `batch_size` (Number) Producer batch size in bytes. Defaults to `16384`. +- `compression` (String) `none`, `gzip`, `snappy`, `lz4` or `zstd`. Defaults to `zstd`. +- `linger_ms` (Number) Milliseconds the producer waits to fill a batch. Defaults to `5`. +- `max_message_bytes` (Number) Largest message the producer sends, in bytes. Defaults to `1048576`. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `mechanisms` (String) SASL mechanism: `PLAIN`, `GSSAPI`, `SCRAM-SHA-256`, `SCRAM-SHA-512` or `OAUTHBEARER`. +- `password` (String, Sensitive) SASL password. +- `protocol` (String) `plaintext`, `ssl`, `sasl_ssl` or `sasl_plaintext`. Defaults to `plaintext`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `ssl_ca_pem` (String) PEM-encoded CA certificate that signed the brokers' certificates. +- `ssl_certificate_pem` (String) PEM-encoded client certificate. +- `ssl_key_pem` (String, Sensitive) PEM-encoded client private key. +- `timeout_sec` (Number) Seconds to wait for the brokers to acknowledge. Defaults to `30`. +- `username` (String) SASL user. + + + +### Nested Schema for `extra_destinations.postgres` + +Required: + +- `database` (String) Database name. +- `host` (String) Database host. +- `password` (String, Sensitive) Database password. +- `table_name` (String) Table the rows are written to. +- `username` (String) Database user. + +Optional: + +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `port` (Number) Database port. Defaults to `5432`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `sslmode` (String) `require` or `disable`. Defaults to `require`. + + + +### Nested Schema for `extra_destinations.s3` + +Required: + +- `access_key` (String) Access key id. +- `bucket` (String) Bucket name. +- `secret_key` (String, Sensitive) Secret access key. + +Optional: + +- `compression` (String) `none` or `gzip`. Defaults to `none`. +- `endpoint` (String) S3 endpoint host. Defaults to `s3.amazonaws.com`. +- `file_type` (String) `.json` or `.parquet`. Defaults to `.json`. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `object_prefix` (String) Prefix added to each object name. It is joined to the file name with no separator, so end it with `/` to write into a folder. Defaults to ``. +- `region` (String) Bucket region, for example `us-east-1`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `use_ssl` (Boolean) Connect to the endpoint over TLS. Defaults to `true`. + + + +### Nested Schema for `extra_destinations.webhook` + +Required: + +- `url` (String) URL the batches are posted to. + +Optional: + +- `compression` (String) `none` or `gzip`. Defaults to `none`. +- `headers` (Map of String) Extra request headers. Omit the attribute rather than setting an empty map. +- `max_retry` (Number) Delivery attempts before the stream is terminated. Defaults to `3`. +- `post_timeout_sec` (Number) Seconds to wait for the receiver to respond. Defaults to `30`. +- `retry_interval_sec` (Number) Seconds between delivery attempts. Defaults to `1`. +- `security_token` (String, Sensitive) Token used to sign each request, so the receiver can verify it came from Quicknode. When omitted on the primary destination, Quicknode generates one and this attribute reports it. + +## Import + +Import is supported using the following syntax: + +The [`terraform import` command](https://developer.hashicorp.com/terraform/cli/commands/import) can be used, for example: + +```shell +# Streams are imported by id. Importing doesn't start or pause the stream, so +# set status to match its current state. +terraform import quicknode_stream.transfers 0b4a6f8e-3c1d-4e2a-9f7b-5d6c8e1a2b3c +``` diff --git a/examples/data-sources/quicknode_kv_list/data-source.tf b/examples/data-sources/quicknode_kv_list/data-source.tf new file mode 100644 index 0000000..0458956 --- /dev/null +++ b/examples/data-sources/quicknode_kv_list/data-source.tf @@ -0,0 +1,8 @@ +# Read a list that a Streams filter maintains. +data "quicknode_kv_list" "tracked_wallets" { + key = "tracked-wallets" +} + +output "tracked_wallet_count" { + value = length(data.quicknode_kv_list.tracked_wallets.items) +} diff --git a/examples/data-sources/quicknode_kv_value/data-source.tf b/examples/data-sources/quicknode_kv_value/data-source.tf new file mode 100644 index 0000000..3a319e6 --- /dev/null +++ b/examples/data-sources/quicknode_kv_value/data-source.tf @@ -0,0 +1,8 @@ +# Read a value that a Streams filter writes, such as the last block processed. +data "quicknode_kv_value" "last_block" { + key = "last-processed-block" +} + +output "last_processed_block" { + value = tonumber(data.quicknode_kv_value.last_block.value) +} diff --git a/examples/data-sources/quicknode_stream/data-source.tf b/examples/data-sources/quicknode_stream/data-source.tf new file mode 100644 index 0000000..cb52802 --- /dev/null +++ b/examples/data-sources/quicknode_stream/data-source.tf @@ -0,0 +1,13 @@ +# Look up a stream created outside Terraform. +data "quicknode_stream" "transfers" { + id = "0b4a6f8e-3c1d-4e2a-9f7b-5d6c8e1a2b3c" +} + +# The last block the stream delivered. It moves while the stream runs. +output "transfers_sequence" { + value = data.quicknode_stream.transfers.sequence +} + +output "transfers_state" { + value = data.quicknode_stream.transfers.status +} diff --git a/examples/provider/provider.tf b/examples/provider/provider.tf index 89d00d1..62ca7e7 100644 --- a/examples/provider/provider.tf +++ b/examples/provider/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { quicknode = { source = "quicknode/quicknode" - version = "~> 0.3.0" + version = "~> 0.4.0" } } } diff --git a/examples/resources/quicknode_kv_list/import.sh b/examples/resources/quicknode_kv_list/import.sh new file mode 100644 index 0000000..7a4a76a --- /dev/null +++ b/examples/resources/quicknode_kv_list/import.sh @@ -0,0 +1,2 @@ +# Lists are imported by their key. +terraform import quicknode_kv_list.watched_wallets watched-wallets diff --git a/examples/resources/quicknode_kv_list/resource.tf b/examples/resources/quicknode_kv_list/resource.tf new file mode 100644 index 0000000..9a35fc9 --- /dev/null +++ b/examples/resources/quicknode_kv_list/resource.tf @@ -0,0 +1,10 @@ +variable "watched_wallets" { + type = set(string) +} + +# Terraform owns the whole list. A Streams filter can read it with +# qnLib.qnContainsListItems("watched-wallets", [...]). +resource "quicknode_kv_list" "watched_wallets" { + key = "watched-wallets" + items = var.watched_wallets +} diff --git a/examples/resources/quicknode_kv_list_items/import.sh b/examples/resources/quicknode_kv_list_items/import.sh new file mode 100644 index 0000000..2a344e1 --- /dev/null +++ b/examples/resources/quicknode_kv_list_items/import.sh @@ -0,0 +1,4 @@ +# Import by list key to take over items already in the list. The import starts +# with no items; the next apply takes over the configured ones and leaves the +# rest alone. +terraform import quicknode_kv_list_items.treasury tracked-wallets diff --git a/examples/resources/quicknode_kv_list_items/resource.tf b/examples/resources/quicknode_kv_list_items/resource.tf new file mode 100644 index 0000000..ab492ba --- /dev/null +++ b/examples/resources/quicknode_kv_list_items/resource.tf @@ -0,0 +1,9 @@ +# A list that a Streams filter also adds to. Terraform tracks only these items +# and leaves the filter's items in place. +resource "quicknode_kv_list_items" "treasury" { + list_key = "tracked-wallets" + items = [ + "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045", + "0x71C7656EC7ab88b098defB751B7401B5f6d8976F", + ] +} diff --git a/examples/resources/quicknode_kv_value/import.sh b/examples/resources/quicknode_kv_value/import.sh new file mode 100644 index 0000000..e426dc1 --- /dev/null +++ b/examples/resources/quicknode_kv_value/import.sh @@ -0,0 +1,2 @@ +# Values are imported by their key. +terraform import quicknode_kv_value.alert_threshold alert-threshold diff --git a/examples/resources/quicknode_kv_value/resource.tf b/examples/resources/quicknode_kv_value/resource.tf new file mode 100644 index 0000000..755ed5a --- /dev/null +++ b/examples/resources/quicknode_kv_value/resource.tf @@ -0,0 +1,14 @@ +# A setting a Streams filter reads with qnLib.qnGetValue("alert-threshold"). +resource "quicknode_kv_value" "alert_threshold" { + key = "alert-threshold" + value = "1000000" +} + +# Values are strings; use jsonencode for structured settings. +resource "quicknode_kv_value" "alert_config" { + key = "alert-config" + value = jsonencode({ + min_value_usd = 50000 + tokens = ["USDC", "USDT"] + }) +} diff --git a/examples/resources/quicknode_stream/import.sh b/examples/resources/quicknode_stream/import.sh new file mode 100644 index 0000000..9e0748a --- /dev/null +++ b/examples/resources/quicknode_stream/import.sh @@ -0,0 +1,3 @@ +# Streams are imported by id. Importing doesn't start or pause the stream, so +# set status to match its current state. +terraform import quicknode_stream.transfers 0b4a6f8e-3c1d-4e2a-9f7b-5d6c8e1a2b3c diff --git a/examples/resources/quicknode_stream/resource.tf b/examples/resources/quicknode_stream/resource.tf new file mode 100644 index 0000000..f92c7bb --- /dev/null +++ b/examples/resources/quicknode_stream/resource.tf @@ -0,0 +1,112 @@ +variable "archive_secret_key" { + type = string + sensitive = true +} + +variable "archive_sas_token" { + type = string + sensitive = true +} + +variable "database_password" { + type = string + sensitive = true +} + +variable "kafka_password" { + type = string + sensitive = true +} + +resource "quicknode_stream" "transfers" { + name = "usdc-transfers" + network = "ethereum-mainnet" + dataset = "receipts" + region = "usa_east" + status = "active" + + # The filter is plain source; the provider encodes it for the API. + filter_function = file("${path.module}/filter.js") + + # Stay behind the newest block so reorganized blocks are never delivered. + keep_distance_from_tip = 3 + + destination = { + webhook = { + url = "https://ingest.example.com/quicknode" + headers = { "X-Source" = "quicknode-streams" } + } + } + + # Every batch is also delivered to each of these. The stream moves on only + # after every destination accepts a batch. + extra_destinations = [ + { + s3 = { + bucket = "chain-archive" + region = "us-east-1" + object_prefix = "usdc-transfers/" + compression = "gzip" + access_key = "AKIAIOSFODNN7EXAMPLE" + secret_key = var.archive_secret_key + } + }, + { + azure = { + storage_account = "chainarchive" + container = "usdc-transfers" + file_type = ".parquet" + sas_token = var.archive_sas_token + } + }, + { + postgres = { + host = "db.example.com" + database = "chain" + table_name = "usdc_transfers" + username = "streams" + password = var.database_password + } + }, + { + kafka = { + bootstrap_servers = "broker-1.example.com:9092,broker-2.example.com:9092" + topic_name = "usdc-transfers" + protocol = "sasl_ssl" + mechanisms = "SCRAM-SHA-512" + username = "streams" + password = var.kafka_password + } + }, + ] +} + +# Quicknode signs each webhook request with this token, so the receiver can +# check it came from Quicknode. +output "webhook_security_token" { + value = quicknode_stream.transfers.destination.webhook.security_token + sensitive = true +} + +# A backfill of a fixed block range. It stops as completed after end_range, +# and any later change replaces it. +resource "quicknode_stream" "backfill" { + name = "usdc-transfers-backfill" + network = "ethereum-mainnet" + dataset = "receipts" + region = "usa_east" + status = "active" + start_range = 21000000 + end_range = 21100000 + + dataset_batch_size = 10 + + filter_function = file("${path.module}/filter.js") + + destination = { + webhook = { + url = "https://ingest.example.com/quicknode/backfill" + compression = "gzip" + } + } +} diff --git a/internal/client/client.go b/internal/client/client.go index 7bf92cd..4395421 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -21,7 +21,9 @@ const DefaultBaseURL = "https://api.quicknode.com" const URLTokenPlaceholder = "REPLACE_WITH_TOKEN" type Client struct { - api *admin.ClientWithResponses + api *admin.ClientWithResponses + http *http.Client + baseURL string } type Option func(*options) @@ -31,6 +33,7 @@ type options struct { httpClient *http.Client maxRetries int requestsPerSecond int + userAgent string } func WithBaseURL(baseURL string) Option { @@ -45,6 +48,11 @@ func WithHTTPClient(httpClient *http.Client) Option { return func(o *options) { o.httpClient = httpClient } } +// WithUserAgent sets the User-Agent header on every request. +func WithUserAgent(userAgent string) Option { + return func(o *options) { o.userAgent = userAgent } +} + func WithMaxRetries(maxRetries int) Option { return func(o *options) { o.maxRetries = maxRetries } } @@ -77,6 +85,7 @@ func New(apiKey string, opts ...Option) (*Client, error) { httpClient.Transport = &retryTransport{ base: httpClient.Transport, apiKey: apiKey, + userAgent: settings.userAgent, maxRetries: settings.maxRetries, baseDelay: defaultBaseDelay, maxDelay: defaultMaxDelay, @@ -90,7 +99,7 @@ func New(apiKey string, opts ...Option) (*Client, error) { if err != nil { return nil, err } - return &Client{api: api}, nil + return &Client{api: api, http: httpClient, baseURL: strings.TrimSuffix(settings.baseURL, "/")}, nil } type Network struct { diff --git a/internal/client/kv.go b/internal/client/kv.go new file mode 100644 index 0000000..2305af6 --- /dev/null +++ b/internal/client/kv.go @@ -0,0 +1,131 @@ +package client + +import ( + "context" + "net/http" + "net/url" + "regexp" + "strconv" +) + +const ( + kvListsPath = "/kv/rest/v1/lists" + kvValuesPath = "/kv/rest/v1/values" + kvPageSize = 1000 +) + +// KVBatchLimit is the most items one list write may add and remove together. +// Larger changes are split into several writes, so a failure part way through +// leaves some of them applied. +const KVBatchLimit = 1500 + +// KVKeyPattern matches the list and value keys the Key-Value API accepts. +var KVKeyPattern = regexp.MustCompile(`^[A-Za-z0-9 ._:$-]+$`) + +type kvEnvelope[T any] struct { + Data T `json:"data"` + Cursor string `json:"cursor"` +} + +type kvListPage struct { + Items []string `json:"items"` +} + +type kvValue struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type kvListCreate struct { + Key string `json:"key"` + Items []string `json:"items"` +} + +type kvListPatch struct { + AddItems []string `json:"addItems"` + RemoveItems []string `json:"removeItems"` +} + +func kvListPath(key string) string { + return kvListsPath + "/" + url.PathEscape(key) +} + +func kvValuePath(key string) string { + return kvValuesPath + "/" + url.PathEscape(key) +} + +// GetKVList returns every item in the list, in the API's sorted order. A list +// that does not exist reads the same as an empty one, so both return no items +// and no error. +func (c *Client) GetKVList(ctx context.Context, key string) ([]string, error) { + var items []string + cursor := "" + for { + query := url.Values{"limit": {strconv.Itoa(kvPageSize)}} + if cursor != "" { + query.Set("cursor", cursor) + } + var page kvEnvelope[kvListPage] + if err := c.doJSON(ctx, "get key-value list", http.MethodGet, kvListPath(key)+"?"+query.Encode(), nil, &page); err != nil { + return nil, err + } + items = append(items, page.Data.Items...) + if page.Cursor == "" || len(page.Data.Items) == 0 { + return items, nil + } + cursor = page.Cursor + } +} + +// CreateKVList writes a new list. Creating a list whose key already exists adds +// the items to it rather than replacing it, so callers check first. +func (c *Client) CreateKVList(ctx context.Context, key string, items []string) error { + first := items + if len(first) > KVBatchLimit { + first = items[:KVBatchLimit] + } + if err := c.doJSON(ctx, "create key-value list", http.MethodPost, kvListsPath, kvListCreate{Key: key, Items: first}, nil); err != nil { + return err + } + return c.UpdateKVList(ctx, key, items[len(first):], nil) +} + +// UpdateKVList adds and removes items in batches of at most KVBatchLimit. +func (c *Client) UpdateKVList(ctx context.Context, key string, add, remove []string) error { + for len(add) > 0 || len(remove) > 0 { + removeCount := min(KVBatchLimit, len(remove)) + addCount := min(KVBatchLimit-removeCount, len(add)) + batch := kvListPatch{ + AddItems: append([]string{}, add[:addCount]...), + RemoveItems: append([]string{}, remove[:removeCount]...), + } + add, remove = add[addCount:], remove[removeCount:] + if err := c.doJSON(ctx, "update key-value list", http.MethodPatch, kvListPath(key), batch, nil); err != nil { + return err + } + } + return nil +} + +func (c *Client) DeleteKVList(ctx context.Context, key string) error { + return c.doJSON(ctx, "delete key-value list", http.MethodDelete, kvListPath(key), nil, nil) +} + +// GetKVValue returns an error that satisfies IsNotFound when the key has no +// value. +func (c *Client) GetKVValue(ctx context.Context, key string) (string, error) { + var result kvEnvelope[kvValue] + if err := c.doJSON(ctx, "get key-value value", http.MethodGet, kvValuePath(key), nil, &result); err != nil { + return "", err + } + return result.Data.Value, nil +} + +// SetKVValue stores the value, replacing any value the key already has. +func (c *Client) SetKVValue(ctx context.Context, key, value string) error { + return c.doJSON(ctx, "set key-value value", http.MethodPost, kvValuesPath, kvValue{Key: key, Value: value}, nil) +} + +func (c *Client) DeleteKVValue(ctx context.Context, key string) error { + return c.doJSON(ctx, "delete key-value value", http.MethodDelete, kvValuePath(key), nil, nil) +} diff --git a/internal/client/kv_test.go b/internal/client/kv_test.go new file mode 100644 index 0000000..13711ee --- /dev/null +++ b/internal/client/kv_test.go @@ -0,0 +1,189 @@ +package client + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "testing" +) + +// fakeKVList stands in for one list on the Key-Value API. It enforces the +// batch limit, pages GET responses and records every request path. +type fakeKVList struct { + items []string + pageSize int + requests []string +} + +func newFakeKVClient(t *testing.T, list *fakeKVList) *Client { + t.Helper() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + list.requests = append(list.requests, r.Method+" "+r.URL.EscapedPath()) + w.Header().Set("Content-Type", "application/json") + + var body struct { + Items []string `json:"items"` + AddItems []string `json:"addItems"` + RemoveItems []string `json:"removeItems"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + + switch r.Method { + case http.MethodPost, http.MethodPatch: + if len(body.Items)+len(body.AddItems)+len(body.RemoveItems) > KVBatchLimit { + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprint(w, `{"statusCode":400,"message":"too many items"}`) + return + } + for _, item := range append(body.Items, body.AddItems...) { + if !slices.Contains(list.items, item) { + list.items = append(list.items, item) + } + } + list.items = slices.DeleteFunc(list.items, func(item string) bool { return slices.Contains(body.RemoveItems, item) }) + slices.Sort(list.items) + _, _ = fmt.Fprint(w, `{"code":200,"msg":"ok","data":null}`) + default: + start, _ := strconv.Atoi(r.URL.Query().Get("cursor")) + end := min(start+list.pageSize, len(list.items)) + cursor := "" + if end < len(list.items) { + cursor = strconv.Itoa(end) + } + page, _ := json.Marshal(list.items[start:end]) + if len(list.items) == 0 { + page = []byte("null") + } + _, _ = fmt.Fprintf(w, `{"code":200,"msg":"","data":{"items":%s},"cursor":%q}`, page, cursor) + } + })) + t.Cleanup(server.Close) + + quicknode, err := New("test-key", WithBaseURL(server.URL), WithMaxRetries(0)) + if err != nil { + t.Fatalf("New: %v", err) + } + return quicknode +} + +func numberedItems(prefix string, count int) []string { + items := make([]string, count) + for index := range items { + items[index] = fmt.Sprintf("%s-%05d", prefix, index) + } + return items +} + +func TestCreateKVListSplitsLargeListsIntoBatches(t *testing.T) { + list := &fakeKVList{pageSize: 1000} + quicknode := newFakeKVClient(t, list) + + if err := quicknode.CreateKVList(context.Background(), "wallets", numberedItems("item", 3200)); err != nil { + t.Fatalf("CreateKVList: %v", err) + } + if len(list.items) != 3200 { + t.Fatalf("list holds %d items, want 3200", len(list.items)) + } + want := []string{"POST /kv/rest/v1/lists", "PATCH /kv/rest/v1/lists/wallets", "PATCH /kv/rest/v1/lists/wallets"} + if !slices.Equal(list.requests, want) { + t.Fatalf("requests = %v, want %v", list.requests, want) + } +} + +func TestUpdateKVListSharesTheBatchLimitBetweenAddsAndRemoves(t *testing.T) { + list := &fakeKVList{pageSize: 1000, items: numberedItems("old", 1000)} + quicknode := newFakeKVClient(t, list) + + if err := quicknode.UpdateKVList(context.Background(), "wallets", numberedItems("new", 1000), numberedItems("old", 1000)); err != nil { + t.Fatalf("UpdateKVList: %v", err) + } + if !slices.Equal(list.items, numberedItems("new", 1000)) { + t.Fatalf("list holds %d items starting %q, want the 1000 new ones", len(list.items), list.items[0]) + } + if len(list.requests) != 2 { + t.Fatalf("made %d requests, want 2: %v", len(list.requests), list.requests) + } +} + +func TestGetKVListFollowsTheCursor(t *testing.T) { + list := &fakeKVList{pageSize: 2, items: []string{"a", "b", "c", "d", "e"}} + quicknode := newFakeKVClient(t, list) + + items, err := quicknode.GetKVList(context.Background(), "wallets") + if err != nil { + t.Fatalf("GetKVList: %v", err) + } + if !slices.Equal(items, list.items) { + t.Fatalf("items = %v, want %v", items, list.items) + } + if len(list.requests) != 3 { + t.Fatalf("made %d requests, want 3", len(list.requests)) + } +} + +func TestGetKVListReadsAMissingListAsEmpty(t *testing.T) { + quicknode := newFakeKVClient(t, &fakeKVList{pageSize: 1000}) + + items, err := quicknode.GetKVList(context.Background(), "missing") + if err != nil { + t.Fatalf("GetKVList: %v", err) + } + if len(items) != 0 { + t.Fatalf("items = %v, want none", items) + } +} + +func TestGetKVValueReportsAMissingKeyAsNotFound(t *testing.T) { + quicknode, recorded := newRecordingClient(t, http.StatusNotFound, `{"statusCode":404,"message":"Key not found"}`) + + _, err := quicknode.GetKVValue(context.Background(), "threshold") + if !IsNotFound(err) { + t.Fatalf("err = %v, want not found", err) + } + if recorded.path != "/kv/rest/v1/values/threshold" { + t.Fatalf("path = %q", recorded.path) + } +} + +func TestSetKVValueSendsKeyAndValue(t *testing.T) { + quicknode, recorded := newRecordingClient(t, http.StatusCreated, `{"code":200,"msg":"Key value stored","data":null}`) + + if err := quicknode.SetKVValue(context.Background(), "threshold", "42"); err != nil { + t.Fatalf("SetKVValue: %v", err) + } + if recorded.method != http.MethodPost || recorded.body["key"] != "threshold" || recorded.body["value"] != "42" { + t.Fatalf("request = %s %v", recorded.method, recorded.body) + } +} + +func TestEveryRequestCarriesTheUserAgent(t *testing.T) { + var seen []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Header.Get("User-Agent")) + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprint(w, `{"code":200,"msg":"","data":{"key":"threshold","value":"1"}}`) + })) + t.Cleanup(server.Close) + + quicknode, err := New("test-key", WithBaseURL(server.URL), WithUserAgent("quicknode-terraform/0.3.0 (linux-x86_64; terraform-1.9.8)")) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, err := quicknode.GetKVValue(context.Background(), "threshold"); err != nil { + t.Fatalf("GetKVValue: %v", err) + } + _, _ = quicknode.ListChains(context.Background()) + for _, userAgent := range seen { + if userAgent != "quicknode-terraform/0.3.0 (linux-x86_64; terraform-1.9.8)" { + t.Fatalf("User-Agent = %q", userAgent) + } + } + if len(seen) != 2 { + t.Fatalf("server saw %d requests, want 2", len(seen)) + } +} diff --git a/internal/client/rest.go b/internal/client/rest.go new file mode 100644 index 0000000..9472fd6 --- /dev/null +++ b/internal/client/rest.go @@ -0,0 +1,86 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +// restErrorBody is the failure shape of the Streams and Key-Value REST APIs. +// A validation failure reports message as a list of strings rather than one. +type restErrorBody struct { + Message json.RawMessage `json:"message"` + Msg string `json:"msg"` +} + +func (c *Client) doJSON(ctx context.Context, operation, method, path string, body, out any) error { + var payload io.Reader + var raw []byte + if body != nil { + var err error + raw, err = json.Marshal(body) + if err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + payload = bytes.NewReader(raw) + } + + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, payload) + if err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + req.GetBody = func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(raw)), nil + } + } + req.Header.Set("Accept", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return restError(operation, resp.StatusCode, respBody) + } + if out == nil || len(respBody) == 0 { + return nil + } + if err := json.Unmarshal(respBody, out); err != nil { + return fmt.Errorf("%s: decoding response: %w", operation, err) + } + return nil +} + +func restError(operation string, status int, body []byte) error { + var parsed restErrorBody + if err := json.Unmarshal(body, &parsed); err != nil { + return statusError(operation, status, body) + } + + message := parsed.Msg + var single string + var many []string + switch { + case json.Unmarshal(parsed.Message, &single) == nil && single != "": + message = single + case json.Unmarshal(parsed.Message, &many) == nil && len(many) > 0: + message = strings.Join(many, "; ") + } + if message == "" { + return statusError(operation, status, body) + } + return &Error{Operation: operation, Status: status, Message: message} +} diff --git a/internal/client/streams.go b/internal/client/streams.go new file mode 100644 index 0000000..03a0e70 --- /dev/null +++ b/internal/client/streams.go @@ -0,0 +1,525 @@ +package client + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" +) + +const streamsPath = "/streams/rest/v1/streams" + +const ( + StreamStatusActive = "active" + StreamStatusPaused = "paused" + StreamStatusTerminated = "terminated" + StreamStatusCompleted = "completed" +) + +const ( + DestinationWebhook = "webhook" + DestinationS3 = "s3" + DestinationAzure = "azure" + DestinationPostgres = "postgres" + DestinationKafka = "kafka" +) + +// StreamRangeUnset is what the API reports for an end_range that runs forever, +// and what it accepts as a start_range meaning the latest block. +const StreamRangeUnset int64 = -1 + +type Stream struct { + ID string + Name string + Status string + Network string + Dataset string + Region string + + // FilterFunction is the filter's source code. The API carries it base64 + // encoded; the client encodes and decodes it. + FilterFunction string + FilterLanguage string + + StartRange int64 + EndRange int64 + DatasetBatchSize int64 + ElasticBatchEnabled bool + RestreamBatchOnReorg bool + FixBlockReorgs int64 + KeepDistanceFromTip int64 + NotificationEmail string + + // Sequence is the last block delivered. It moves on its own while the + // stream runs. + Sequence int64 + + Destination StreamDestination + ExtraDestinations []StreamDestination +} + +type StreamInput struct { + Name string + Network string + Dataset string + Region string + Status string + FilterFunction string + FilterLanguage string + + StartRange *int64 + EndRange *int64 + DatasetBatchSize int64 + ElasticBatchEnabled bool + RestreamBatchOnReorg *bool + FixBlockReorgs *int64 + KeepDistanceFromTip *int64 + NotificationEmail *string + + Destination StreamDestination + ExtraDestinations []StreamDestination +} + +// StreamUpdate sends only the fields that are set. Network, dataset and region +// cannot change after creation, and status changes go through PauseStream and +// ActivateStream, which run the plan checks a PATCH skips. +type StreamUpdate struct { + Name *string + FilterFunction *string + FilterLanguage *string + + StartRange *int64 + EndRange *int64 + DatasetBatchSize *int64 + ElasticBatchEnabled *bool + RestreamBatchOnReorg *bool + FixBlockReorgs *int64 + KeepDistanceFromTip *int64 + NotificationEmail *string + + Destination *StreamDestination + ExtraDestinations *[]StreamDestination +} + +// StreamDestination holds exactly one destination type. A type the API reports +// but the client does not model, such as one configured in the dashboard, +// leaves every field nil and is named by Type. +type StreamDestination struct { + Webhook *WebhookDestination + S3 *S3Destination + Azure *AzureDestination + Postgres *PostgresDestination + Kafka *KafkaDestination + + unknownType string +} + +type WebhookDestination struct { + URL string `json:"url"` + Compression string `json:"compression"` + Headers HeaderMap `json:"headers"` + MaxRetry int64 `json:"max_retry"` + RetryIntervalSec int64 `json:"retry_interval_sec"` + PostTimeoutSec int64 `json:"post_timeout_sec"` + SecurityToken string `json:"security_token,omitempty"` + MTLS *bool `json:"mtls,omitempty"` + SSLCAPEM string `json:"ssl_ca_pem,omitempty"` +} + +type S3Destination struct { + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` + Region string `json:"region,omitempty"` + ObjectPrefix string `json:"object_prefix"` + FileType string `json:"file_type"` + FileCompressionType string `json:"file_compression_type"` + UseSSL bool `json:"use_ssl"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + MaxRetry int64 `json:"max_retry"` + RetryIntervalSec int64 `json:"retry_interval_sec"` +} + +type AzureDestination struct { + StorageAccount string `json:"storage_account"` + Container string `json:"container"` + BlobPrefix string `json:"blob_prefix"` + FileType string `json:"file_type"` + FileCompressionType string `json:"file_compression_type"` + SASToken string `json:"sas_token,omitempty"` + MaxRetry int64 `json:"max_retry"` + RetryIntervalSec int64 `json:"retry_interval_sec"` +} + +type PostgresDestination struct { + Host string `json:"host"` + Port int64 `json:"port"` + Database string `json:"database"` + TableName string `json:"table_name"` + Username string `json:"username"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + MaxRetry int64 `json:"max_retry"` + RetryIntervalSec int64 `json:"retry_interval_sec"` +} + +type KafkaDestination struct { + BootstrapServers string `json:"bootstrap_servers"` + TopicName string `json:"topic_name"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Mechanisms string `json:"mechanisms,omitempty"` + Protocol string `json:"protocol,omitempty"` + CompressionType string `json:"compression_type"` + BatchSize int64 `json:"batch_size"` + LingerMs int64 `json:"linger_ms"` + MaxMessageBytes int64 `json:"max_message_bytes"` + TimeoutSec int64 `json:"timeout_sec"` + MaxRetry int64 `json:"max_retry"` + RetryIntervalSec int64 `json:"retry_interval_sec"` + SSLCAPEM string `json:"ssl_ca_pem,omitempty"` + SSLCertificatePEM string `json:"ssl_certificate_pem,omitempty"` + SSLKeyPEM string `json:"ssl_key_pem,omitempty"` +} + +// HeaderMap is a webhook's extra request headers. The API stores header values +// as strings or numbers, so decoding accepts both, and it rejects a missing +// headers object, so a nil map encodes as an empty one. +type HeaderMap map[string]string + +func (h HeaderMap) MarshalJSON() ([]byte, error) { + if h == nil { + return []byte("{}"), nil + } + return json.Marshal(map[string]string(h)) +} + +func (h *HeaderMap) UnmarshalJSON(raw []byte) error { + var values map[string]any + if err := json.Unmarshal(raw, &values); err != nil { + return err + } + decoded := make(HeaderMap, len(values)) + for key, value := range values { + switch typed := value.(type) { + case string: + decoded[key] = typed + case float64: + decoded[key] = strconv.FormatFloat(typed, 'f', -1, 64) + case bool: + decoded[key] = strconv.FormatBool(typed) + case nil: + decoded[key] = "" + default: + return fmt.Errorf("header %q has unsupported value %v", key, value) + } + } + *h = decoded + return nil +} + +func (d StreamDestination) Type() string { + switch { + case d.Webhook != nil: + return DestinationWebhook + case d.S3 != nil: + return DestinationS3 + case d.Azure != nil: + return DestinationAzure + case d.Postgres != nil: + return DestinationPostgres + case d.Kafka != nil: + return DestinationKafka + default: + return d.unknownType + } +} + +func (d StreamDestination) attributes() (any, error) { + switch { + case d.Webhook != nil: + return d.Webhook, nil + case d.S3 != nil: + return d.S3, nil + case d.Azure != nil: + return d.Azure, nil + case d.Postgres != nil: + return d.Postgres, nil + case d.Kafka != nil: + return d.Kafka, nil + default: + return nil, fmt.Errorf("destination type %q is not supported", d.unknownType) + } +} + +type destinationWire struct { + Destination string `json:"destination"` + Attributes json.RawMessage `json:"destination_attributes"` +} + +func (d StreamDestination) MarshalJSON() ([]byte, error) { + attributes, err := d.attributes() + if err != nil { + return nil, err + } + raw, err := json.Marshal(attributes) + if err != nil { + return nil, err + } + return json.Marshal(destinationWire{Destination: d.Type(), Attributes: raw}) +} + +func (d *StreamDestination) UnmarshalJSON(raw []byte) error { + var wire destinationWire + if err := json.Unmarshal(raw, &wire); err != nil { + return err + } + decoded, err := decodeDestination(wire.Destination, wire.Attributes) + if err != nil { + return err + } + *d = decoded + return nil +} + +func decodeDestination(kind string, raw json.RawMessage) (StreamDestination, error) { + var destination StreamDestination + var target any + switch kind { + case DestinationWebhook: + destination.Webhook = &WebhookDestination{} + target = destination.Webhook + case DestinationS3: + destination.S3 = &S3Destination{} + target = destination.S3 + case DestinationAzure: + destination.Azure = &AzureDestination{} + target = destination.Azure + case DestinationPostgres: + destination.Postgres = &PostgresDestination{} + target = destination.Postgres + case DestinationKafka: + destination.Kafka = &KafkaDestination{} + target = destination.Kafka + default: + return StreamDestination{unknownType: kind}, nil + } + if len(raw) == 0 || string(raw) == "null" { + return destination, nil + } + if err := json.Unmarshal(raw, target); err != nil { + return StreamDestination{}, fmt.Errorf("decoding %s destination: %w", kind, err) + } + return destination, nil +} + +type streamWire struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Network string `json:"network"` + Dataset string `json:"dataset"` + Region string `json:"region"` + FilterFunction string `json:"filter_function"` + FilterLanguage string `json:"filter_language"` + StartRange int64 `json:"start_range"` + EndRange int64 `json:"end_range"` + DatasetBatchSize int64 `json:"dataset_batch_size"` + ElasticBatchEnabled bool `json:"elastic_batch_enabled"` + RestreamBatchOnReorg bool `json:"restream_batch_on_reorg"` + FixBlockReorgs int64 `json:"fix_block_reorgs"` + KeepDistanceFromTip int64 `json:"keep_distance_from_tip"` + NotificationEmail string `json:"notification_email"` + Sequence int64 `json:"sequence"` + Destination string `json:"destination"` + DestinationAttributes json.RawMessage `json:"destination_attributes"` + ExtraDestinations []StreamDestination `json:"extra_destinations"` +} + +func (w streamWire) toStream() (*Stream, error) { + filter, err := decodeFilter(w.FilterFunction) + if err != nil { + return nil, err + } + destination, err := decodeDestination(w.Destination, w.DestinationAttributes) + if err != nil { + return nil, err + } + extras := w.ExtraDestinations + if extras == nil { + extras = []StreamDestination{} + } + return &Stream{ + ID: w.ID, + Name: w.Name, + Status: w.Status, + Network: w.Network, + Dataset: w.Dataset, + Region: w.Region, + FilterFunction: filter, + FilterLanguage: w.FilterLanguage, + StartRange: w.StartRange, + EndRange: w.EndRange, + DatasetBatchSize: w.DatasetBatchSize, + ElasticBatchEnabled: w.ElasticBatchEnabled, + RestreamBatchOnReorg: w.RestreamBatchOnReorg, + FixBlockReorgs: w.FixBlockReorgs, + KeepDistanceFromTip: w.KeepDistanceFromTip, + NotificationEmail: w.NotificationEmail, + Sequence: w.Sequence, + Destination: destination, + ExtraDestinations: extras, + }, nil +} + +type streamCreateWire struct { + Name string `json:"name"` + Network string `json:"network"` + Dataset string `json:"dataset"` + Region string `json:"region"` + Status string `json:"status"` + FilterFunction string `json:"filter_function,omitempty"` + FilterLanguage string `json:"filter_language,omitempty"` + StartRange *int64 `json:"start_range,omitempty"` + EndRange *int64 `json:"end_range,omitempty"` + DatasetBatchSize int64 `json:"dataset_batch_size"` + ElasticBatchEnabled bool `json:"elastic_batch_enabled"` + RestreamBatchOnReorg *bool `json:"restream_batch_on_reorg,omitempty"` + FixBlockReorgs *int64 `json:"fix_block_reorgs,omitempty"` + KeepDistanceFromTip *int64 `json:"keep_distance_from_tip,omitempty"` + NotificationEmail *string `json:"notification_email,omitempty"` + Destination string `json:"destination"` + DestinationAttributes any `json:"destination_attributes"` + ExtraDestinations []StreamDestination `json:"extra_destinations,omitempty"` +} + +type streamUpdateWire struct { + Name *string `json:"name,omitempty"` + FilterFunction *string `json:"filter_function,omitempty"` + FilterLanguage *string `json:"filter_language,omitempty"` + StartRange *int64 `json:"start_range,omitempty"` + EndRange *int64 `json:"end_range,omitempty"` + DatasetBatchSize *int64 `json:"dataset_batch_size,omitempty"` + ElasticBatchEnabled *bool `json:"elastic_batch_enabled,omitempty"` + RestreamBatchOnReorg *bool `json:"restream_batch_on_reorg,omitempty"` + FixBlockReorgs *int64 `json:"fix_block_reorgs,omitempty"` + KeepDistanceFromTip *int64 `json:"keep_distance_from_tip,omitempty"` + NotificationEmail *string `json:"notification_email,omitempty"` + Destination *string `json:"destination,omitempty"` + DestinationAttributes any `json:"destination_attributes,omitempty"` + ExtraDestinations *[]StreamDestination `json:"extra_destinations,omitempty"` +} + +func encodeFilter(source string) string { + if source == "" { + return "" + } + return base64.StdEncoding.EncodeToString([]byte(source)) +} + +func decodeFilter(encoded string) (string, error) { + if encoded == "" { + return "", nil + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return "", fmt.Errorf("decoding filter_function: %w", err) + } + return string(decoded), nil +} + +func streamPath(id string) string { + return streamsPath + "/" + url.PathEscape(id) +} + +func (c *Client) CreateStream(ctx context.Context, input StreamInput) (*Stream, error) { + attributes, err := input.Destination.attributes() + if err != nil { + return nil, fmt.Errorf("create stream: %w", err) + } + + body := streamCreateWire{ + Name: input.Name, + Network: input.Network, + Dataset: input.Dataset, + Region: input.Region, + Status: input.Status, + FilterFunction: encodeFilter(input.FilterFunction), + FilterLanguage: input.FilterLanguage, + StartRange: input.StartRange, + EndRange: input.EndRange, + DatasetBatchSize: input.DatasetBatchSize, + ElasticBatchEnabled: input.ElasticBatchEnabled, + RestreamBatchOnReorg: input.RestreamBatchOnReorg, + FixBlockReorgs: input.FixBlockReorgs, + KeepDistanceFromTip: input.KeepDistanceFromTip, + NotificationEmail: input.NotificationEmail, + Destination: input.Destination.Type(), + DestinationAttributes: attributes, + ExtraDestinations: input.ExtraDestinations, + } + + var wire streamWire + if err := c.doJSON(ctx, "create stream", http.MethodPost, streamsPath, body, &wire); err != nil { + return nil, err + } + return wire.toStream() +} + +func (c *Client) GetStream(ctx context.Context, id string) (*Stream, error) { + var wire streamWire + if err := c.doJSON(ctx, "get stream", http.MethodGet, streamPath(id), nil, &wire); err != nil { + return nil, err + } + return wire.toStream() +} + +func (c *Client) UpdateStream(ctx context.Context, id string, update StreamUpdate) (*Stream, error) { + body := streamUpdateWire{ + Name: update.Name, + FilterLanguage: update.FilterLanguage, + StartRange: update.StartRange, + EndRange: update.EndRange, + DatasetBatchSize: update.DatasetBatchSize, + ElasticBatchEnabled: update.ElasticBatchEnabled, + RestreamBatchOnReorg: update.RestreamBatchOnReorg, + FixBlockReorgs: update.FixBlockReorgs, + KeepDistanceFromTip: update.KeepDistanceFromTip, + NotificationEmail: update.NotificationEmail, + ExtraDestinations: update.ExtraDestinations, + } + if update.FilterFunction != nil { + encoded := encodeFilter(*update.FilterFunction) + body.FilterFunction = &encoded + } + if update.Destination != nil { + attributes, err := update.Destination.attributes() + if err != nil { + return nil, fmt.Errorf("update stream: %w", err) + } + kind := update.Destination.Type() + body.Destination = &kind + body.DestinationAttributes = attributes + } + + var wire streamWire + if err := c.doJSON(ctx, "update stream", http.MethodPatch, streamPath(id), body, &wire); err != nil { + return nil, err + } + return wire.toStream() +} + +func (c *Client) DeleteStream(ctx context.Context, id string) error { + return c.doJSON(ctx, "delete stream", http.MethodDelete, streamPath(id), nil, nil) +} + +func (c *Client) ActivateStream(ctx context.Context, id string) error { + return c.doJSON(ctx, "activate stream", http.MethodPost, streamPath(id)+"/activate", nil, nil) +} + +func (c *Client) PauseStream(ctx context.Context, id string) error { + return c.doJSON(ctx, "pause stream", http.MethodPost, streamPath(id)+"/pause", nil, nil) +} diff --git a/internal/client/streams_test.go b/internal/client/streams_test.go new file mode 100644 index 0000000..ef86117 --- /dev/null +++ b/internal/client/streams_test.go @@ -0,0 +1,201 @@ +package client + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// streamBody is a live GET response with the secrets replaced. The primary +// destination carries the server's version marker, the extra one does not. +const streamBody = `{"id":"aca58bf5","name":"tftrial","status":"paused","created_at":"2026-09-28T22:31:43Z","current_hash":"", +"dataset":"block","dataset_batch_size":1,"destination":"webhook", +"destination_attributes":{"compression":"none","headers":{"X-Probe":"1","X-Count":2},"max_retry":3,"post_timeout_sec":10,"retry_interval_sec":1,"security_token":"token","url":"https://example.com/a","version":"v5"}, +"end_range":-1,"filter_function":"ZnVuY3Rpb24gbWFpbihzdHJlYW0pIHsgcmV0dXJuIHN0cmVhbTsgfQ==","filter_language":"javascript", +"fix_block_reorgs":0,"keep_distance_from_tip":0,"network":"ethereum-sepolia","notification_email":"","region":"usa_east", +"sequence":0,"start_range":11803526,"updated_at":"2026-09-28T22:31:43Z","elastic_batch_enabled":false,"restream_batch_on_reorg":false, +"extra_destinations":[{"destination":"webhook","destination_attributes":{"compression":"none","headers":{},"max_retry":1,"post_timeout_sec":10,"retry_interval_sec":1,"url":"https://example.com/b"}}]}` + +type recordedRequest struct { + method string + path string + body map[string]any +} + +func newRecordingClient(t *testing.T, status int, response string) (*Client, *recordedRequest) { + t.Helper() + + recorded := &recordedRequest{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recorded.method = r.Method + recorded.path = r.URL.Path + raw, _ := io.ReadAll(r.Body) + if len(raw) > 0 { + _ = json.Unmarshal(raw, &recorded.body) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(response)) + })) + t.Cleanup(server.Close) + + quicknode, err := New("test-key", WithBaseURL(server.URL), WithMaxRetries(0)) + if err != nil { + t.Fatalf("New: %v", err) + } + return quicknode, recorded +} + +func TestGetStreamDecodesFilterAndDestinations(t *testing.T) { + quicknode, _ := newRecordingClient(t, http.StatusOK, streamBody) + + stream, err := quicknode.GetStream(context.Background(), "aca58bf5") + if err != nil { + t.Fatalf("GetStream: %v", err) + } + + if stream.FilterFunction != "function main(stream) { return stream; }" { + t.Errorf("filter was not decoded: %q", stream.FilterFunction) + } + if stream.StartRange != 11803526 || stream.EndRange != StreamRangeUnset { + t.Errorf("ranges = %d..%d", stream.StartRange, stream.EndRange) + } + + webhook := stream.Destination.Webhook + if webhook == nil { + t.Fatalf("primary destination type = %q, want webhook", stream.Destination.Type()) + } + if webhook.URL != "https://example.com/a" || webhook.SecurityToken != "token" { + t.Errorf("webhook = %+v", *webhook) + } + if webhook.Headers["X-Count"] != "2" { + t.Errorf("a numeric header value was not kept as a string: %v", webhook.Headers) + } + + if len(stream.ExtraDestinations) != 1 || stream.ExtraDestinations[0].Webhook == nil { + t.Fatalf("extra destinations = %+v", stream.ExtraDestinations) + } + if stream.ExtraDestinations[0].Webhook.URL != "https://example.com/b" { + t.Errorf("extra webhook = %+v", *stream.ExtraDestinations[0].Webhook) + } +} + +func TestGetStreamReportsUnmodeledDestination(t *testing.T) { + body := `{"id":"1","destination":"snowflake","destination_attributes":{"account":"x"},"extra_destinations":null}` + quicknode, _ := newRecordingClient(t, http.StatusOK, body) + + stream, err := quicknode.GetStream(context.Background(), "1") + if err != nil { + t.Fatalf("GetStream: %v", err) + } + if stream.Destination.Type() != "snowflake" { + t.Errorf("type = %q, want snowflake", stream.Destination.Type()) + } + if stream.ExtraDestinations == nil { + t.Error("a null extra_destinations should decode as an empty list") + } +} + +func TestCreateStreamEncodesFilterAndDestination(t *testing.T) { + quicknode, recorded := newRecordingClient(t, http.StatusCreated, streamBody) + + endRange := int64(100) + _, err := quicknode.CreateStream(context.Background(), StreamInput{ + Name: "tftrial", + Network: "ethereum-sepolia", + Dataset: "block", + Region: "usa_east", + Status: StreamStatusPaused, + FilterFunction: "function main(stream) { return stream; }", + EndRange: &endRange, + DatasetBatchSize: 1, + Destination: StreamDestination{Webhook: &WebhookDestination{ + URL: "https://example.com/a", Compression: "none", MaxRetry: 3, RetryIntervalSec: 1, PostTimeoutSec: 10, + }}, + ExtraDestinations: []StreamDestination{{S3: &S3Destination{Bucket: "archive"}}}, + }) + if err != nil { + t.Fatalf("CreateStream: %v", err) + } + + if recorded.method != http.MethodPost || recorded.path != "/streams/rest/v1/streams" { + t.Errorf("request = %s %s", recorded.method, recorded.path) + } + body := recorded.body + if body["filter_function"] != "ZnVuY3Rpb24gbWFpbihzdHJlYW0pIHsgcmV0dXJuIHN0cmVhbTsgfQ==" { + t.Errorf("filter_function = %v", body["filter_function"]) + } + if _, sent := body["start_range"]; sent { + t.Error("an unset start_range must be omitted so the server picks the latest block") + } + if body["end_range"] != float64(100) { + t.Errorf("end_range = %v", body["end_range"]) + } + if body["destination"] != "webhook" { + t.Errorf("destination = %v", body["destination"]) + } + + attributes := body["destination_attributes"].(map[string]any) + if _, sent := attributes["security_token"]; sent { + t.Error("an unset security_token must be omitted so the server generates one") + } + if headers, ok := attributes["headers"].(map[string]any); !ok || len(headers) != 0 { + t.Errorf("headers should be sent as an empty object, got %v", attributes["headers"]) + } + + extras := body["extra_destinations"].([]any) + extra := extras[0].(map[string]any) + if extra["destination"] != "s3" { + t.Errorf("extra destination = %v", extra["destination"]) + } +} + +func TestUpdateStreamSendsOnlySetFields(t *testing.T) { + quicknode, recorded := newRecordingClient(t, http.StatusOK, streamBody) + + name := "renamed" + noExtras := []StreamDestination{} + _, err := quicknode.UpdateStream(context.Background(), "aca58bf5", StreamUpdate{ + Name: &name, + ExtraDestinations: &noExtras, + }) + if err != nil { + t.Fatalf("UpdateStream: %v", err) + } + + if recorded.method != http.MethodPatch || recorded.path != "/streams/rest/v1/streams/aca58bf5" { + t.Errorf("request = %s %s", recorded.method, recorded.path) + } + if len(recorded.body) != 2 { + t.Errorf("body should carry only name and extra_destinations, got %v", recorded.body) + } + if extras, ok := recorded.body["extra_destinations"].([]any); !ok || len(extras) != 0 { + t.Errorf("clearing extra destinations should send an empty list, got %v", recorded.body["extra_destinations"]) + } +} + +func TestStreamErrorsCarryAPIMessage(t *testing.T) { + quicknode, _ := newRecordingClient(t, http.StatusNotFound, + `{"statusCode":404,"timestamp":"2026-09-28T00:00:00Z","path":"/streams/rest/v1/streams/x","message":"STREAM_NOT_FOUND"}`) + + _, err := quicknode.GetStream(context.Background(), "x") + if !IsNotFound(err) { + t.Fatalf("want a not-found error, got %v", err) + } + + quicknode, _ = newRecordingClient(t, http.StatusBadRequest, + `{"statusCode":400,"message":["name should not be empty","region must be one of the following values"]}`) + + _, err = quicknode.GetStream(context.Background(), "x") + var apiErr *Error + if err == nil || !errors.As(err, &apiErr) { + t.Fatalf("want an API error, got %v", err) + } + if apiErr.Message != "name should not be empty; region must be one of the following values" { + t.Errorf("message = %q", apiErr.Message) + } +} diff --git a/internal/client/transport.go b/internal/client/transport.go index 09ee0a3..2139de7 100644 --- a/internal/client/transport.go +++ b/internal/client/transport.go @@ -24,6 +24,7 @@ const ( type retryTransport struct { base http.RoundTripper apiKey string + userAgent string maxRetries int baseDelay time.Duration maxDelay time.Duration @@ -68,6 +69,9 @@ func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) { if t.apiKey != "" { clone.Header.Set("x-api-key", t.apiKey) } + if t.userAgent != "" { + clone.Header.Set("User-Agent", t.userAgent) + } if req.GetBody != nil { body, err := req.GetBody() if err != nil { diff --git a/internal/provider/kv.go b/internal/provider/kv.go new file mode 100644 index 0000000..0109403 --- /dev/null +++ b/internal/provider/kv.go @@ -0,0 +1,109 @@ +package provider + +import ( + "context" + "fmt" + "slices" + + "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/datasource" + datasourceschema "github.com/hashicorp/terraform-plugin-framework/datasource/schema" + "github.com/hashicorp/terraform-plugin-framework/diag" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +func kvKeyValidators() []validator.String { + return []validator.String{ + stringvalidator.LengthBetween(1, 255), + stringvalidator.RegexMatches(client.KVKeyPattern, "may only contain letters, digits, spaces and . : _ - $"), + } +} + +func kvKeyAttribute(description string) schema.StringAttribute { + return schema.StringAttribute{ + Required: true, + MarkdownDescription: description + " Up to 255 letters, digits, spaces and `.` `:` `_` `-` `$`. Changing it replaces the resource.", + PlanModifiers: []planmodifier.String{stringplanmodifier.RequiresReplace()}, + Validators: kvKeyValidators(), + } +} + +func kvIDAttribute(description string) schema.StringAttribute { + return schema.StringAttribute{ + Computed: true, + MarkdownDescription: description, + PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, + } +} + +func stringSet(items []string) (types.Set, diag.Diagnostics) { + values := make([]attr.Value, 0, len(items)) + for _, item := range items { + values = append(values, types.StringValue(item)) + } + return types.SetValue(types.StringType, values) +} + +func setStrings(ctx context.Context, set types.Set) ([]string, diag.Diagnostics) { + var items []string + diags := set.ElementsAs(ctx, &items, false) + slices.Sort(items) + return items, diags +} + +// setDifference returns the items in from that are not in without. +func setDifference(from, without []string) []string { + excluded := make(map[string]bool, len(without)) + for _, item := range without { + excluded[item] = true + } + var difference []string + for _, item := range from { + if !excluded[item] { + difference = append(difference, item) + } + } + return difference +} + +func kvDataSourceKey(description string) datasourceschema.StringAttribute { + return datasourceschema.StringAttribute{ + Required: true, + MarkdownDescription: description, + Validators: kvKeyValidators(), + } +} + +func kvDataSourceClient(req datasource.ConfigureRequest, resp *datasource.ConfigureResponse, name string) *client.Client { + if req.ProviderData == nil { + return nil + } + data, ok := req.ProviderData.(providerData) + if !ok { + resp.Diagnostics.AddError("Unexpected provider data", fmt.Sprintf("The %s data source expected providerData, got %T.", name, req.ProviderData)) + return nil + } + return data.Client +} + +// setIntersection returns the items in from that are also in within. +func setIntersection(from, within []string) []string { + included := make(map[string]bool, len(within)) + for _, item := range within { + included[item] = true + } + var intersection []string + for _, item := range from { + if included[item] { + intersection = append(intersection, item) + } + } + return intersection +} diff --git a/internal/provider/kv_acceptance_test.go b/internal/provider/kv_acceptance_test.go new file mode 100644 index 0000000..2c6b40f --- /dev/null +++ b/internal/provider/kv_acceptance_test.go @@ -0,0 +1,326 @@ +package provider_test + +import ( + "context" + "fmt" + "os" + "regexp" + "slices" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/plancheck" + "github.com/hashicorp/terraform-plugin-testing/terraform" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +// Key-value keys are account wide, so every test uses a random one. +func kvAcceptanceKey() string { + return acctest.RandomWithPrefix("tfacc") +} + +func testAccClient(t *testing.T) *client.Client { + t.Helper() + quicknode, err := client.New(os.Getenv("QUICKNODE_API_KEY")) + if err != nil { + t.Fatalf("could not build a client: %v", err) + } + return quicknode +} + +func testAccCheckKVDestroyed(state *terraform.State) error { + quicknode, err := client.New(os.Getenv("QUICKNODE_API_KEY")) + if err != nil { + return fmt.Errorf("could not build a client to confirm the destroy: %w", err) + } + ctx := context.Background() + + for name, resourceState := range state.RootModule().Resources { + attributes := resourceState.Primary.Attributes + switch resourceState.Type { + case "quicknode_kv_list": + items, err := quicknode.GetKVList(ctx, attributes["key"]) + if err != nil { + return fmt.Errorf("%s: could not confirm list %s was destroyed: %w", name, attributes["key"], err) + } + if len(items) > 0 { + return fmt.Errorf("%s: list %s still has %d items after destroy", name, attributes["key"], len(items)) + } + case "quicknode_kv_list_items": + items, err := quicknode.GetKVList(ctx, attributes["list_key"]) + if err != nil { + return fmt.Errorf("%s: could not confirm the items were removed from %s: %w", name, attributes["list_key"], err) + } + for key, item := range attributes { + if strings.HasPrefix(key, "items.") && key != "items.#" && slices.Contains(items, item) { + return fmt.Errorf("%s: list %s still holds %s after destroy", name, attributes["list_key"], item) + } + } + case "quicknode_kv_value": + _, err := quicknode.GetKVValue(ctx, attributes["key"]) + if client.IsNotFound(err) { + continue + } + if err != nil { + return fmt.Errorf("%s: could not confirm value %s was destroyed: %w", name, attributes["key"], err) + } + return fmt.Errorf("%s: value %s still exists after destroy", name, attributes["key"]) + } + } + return nil +} + +func TestAccKVList_lifecycle(t *testing.T) { + key := kvAcceptanceKey() + withItems := func(items string) string { + return fmt.Sprintf(` +resource "quicknode_kv_list" "test" { + key = %q + items = %s +} + +data "quicknode_kv_list" "test" { + key = quicknode_kv_list.test.key + depends_on = [quicknode_kv_list.test] +} +`, key, items) + } + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckKVDestroyed, + Steps: []resource.TestStep{ + { + Config: withItems(`["0xAbC", "0xabc", "b"]`), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("quicknode_kv_list.test", "id", key), + resource.TestCheckResourceAttr("quicknode_kv_list.test", "items.#", "3"), + resource.TestCheckTypeSetElemAttr("quicknode_kv_list.test", "items.*", "0xAbC"), + resource.TestCheckResourceAttr("data.quicknode_kv_list.test", "items.#", "3"), + ), + }, + { + ResourceName: "quicknode_kv_list.test", + ImportState: true, + ImportStateId: key, + ImportStateVerify: true, + }, + { + Config: withItems(`["0xabc", "c", "d"]`), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("quicknode_kv_list.test", "items.#", "3"), + resource.TestCheckTypeSetElemAttr("quicknode_kv_list.test", "items.*", "c"), + resource.TestCheckResourceAttr("data.quicknode_kv_list.test", "items.#", "3"), + resource.TestCheckTypeSetElemAttr("data.quicknode_kv_list.test", "items.*", "d"), + ), + }, + }, + }) +} + +// TestAccKVList_refusesAnExistingList checks that creating a list never merges +// into one Terraform does not own. +func TestAccKVList_refusesAnExistingList(t *testing.T) { + key := kvAcceptanceKey() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckKVDestroyed, + Steps: []resource.TestStep{ + { + PreConfig: func() { + quicknode := testAccClient(t) + if err := quicknode.CreateKVList(context.Background(), key, []string{"outside"}); err != nil { + t.Fatalf("could not create the existing list: %v", err) + } + t.Cleanup(func() { _ = quicknode.DeleteKVList(context.Background(), key) }) + }, + Config: fmt.Sprintf(` +resource "quicknode_kv_list" "test" { + key = %q + items = ["inside"] +} +`, key), + ExpectError: regexp.MustCompile(`Key-value list already exists`), + }, + }, + }) +} + +func kvListItemsConfig(key, items string) string { + return fmt.Sprintf(` +resource "quicknode_kv_list_items" "test" { + list_key = %q + items = %s +} + +data "quicknode_kv_list" "test" { + key = quicknode_kv_list_items.test.list_key + depends_on = [quicknode_kv_list_items.test] +} +`, key, items) +} + +func TestAccKVListItems_leavesOtherItemsAlone(t *testing.T) { + key := kvAcceptanceKey() + t.Cleanup(func() { _ = testAccClient(t).DeleteKVList(context.Background(), key) }) + const wallet = "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045" + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckKVDestroyed, + Steps: []resource.TestStep{ + { + Config: kvListItemsConfig(key, fmt.Sprintf(`[%q, "tokens/usdc"]`, wallet)), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("quicknode_kv_list_items.test", "id", key), + resource.TestCheckResourceAttr("quicknode_kv_list_items.test", "items.#", "2"), + func(*terraform.State) error { + return testAccClient(t).UpdateKVList(context.Background(), key, []string{"added-outside"}, nil) + }, + ), + }, + { + Config: kvListItemsConfig(key, fmt.Sprintf(`[%q, "0xnew"]`, wallet)), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction("quicknode_kv_list_items.test", plancheck.ResourceActionUpdate), + }, + }, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("quicknode_kv_list_items.test", "items.#", "2"), + resource.TestCheckResourceAttr("data.quicknode_kv_list.test", "items.#", "3"), + resource.TestCheckTypeSetElemAttr("data.quicknode_kv_list.test", "items.*", "added-outside"), + resource.TestCheckTypeSetElemAttr("data.quicknode_kv_list.test", "items.*", "0xnew"), + ), + }, + { + PreConfig: func() { + if err := testAccClient(t).UpdateKVList(context.Background(), key, nil, []string{wallet}); err != nil { + t.Fatalf("could not remove the item outside Terraform: %v", err) + } + }, + Config: kvListItemsConfig(key, fmt.Sprintf(`[%q, "0xnew"]`, wallet)), + Check: resource.TestCheckTypeSetElemAttr("data.quicknode_kv_list.test", "items.*", wallet), + }, + { + Config: `# empty`, + Check: func(*terraform.State) error { + items, err := testAccClient(t).GetKVList(context.Background(), key) + if err != nil { + return err + } + if len(items) != 1 || items[0] != "added-outside" { + return fmt.Errorf("list holds %v after destroy, want only the item added outside Terraform", items) + } + return nil + }, + }, + }, + }) +} + +// TestAccKVListItems_takesOverExistingItemsByImport checks that create refuses +// items already in the list, and that importing then applying takes over only +// the configured ones. +func TestAccKVListItems_takesOverExistingItemsByImport(t *testing.T) { + key := kvAcceptanceKey() + t.Cleanup(func() { _ = testAccClient(t).DeleteKVList(context.Background(), key) }) + config := fmt.Sprintf(` +resource "quicknode_kv_list_items" "test" { + list_key = %q + items = ["owned"] +} +`, key) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckKVDestroyed, + Steps: []resource.TestStep{ + { + PreConfig: func() { + if err := testAccClient(t).CreateKVList(context.Background(), key, []string{"owned", "kept"}); err != nil { + t.Fatalf("could not create the existing list: %v", err) + } + }, + Config: config, + ExpectError: regexp.MustCompile(`Items are already in the list`), + }, + { + Config: config, + ResourceName: "quicknode_kv_list_items.test", + ImportState: true, + ImportStateId: key, + ImportStatePersist: true, + }, + { + Config: config, + Check: resource.TestCheckResourceAttr("quicknode_kv_list_items.test", "items.#", "1"), + }, + { + Config: `# empty`, + Check: func(*terraform.State) error { + items, err := testAccClient(t).GetKVList(context.Background(), key) + if err != nil { + return err + } + if len(items) != 1 || items[0] != "kept" { + return fmt.Errorf("list holds %v after destroy, want only the item Terraform never owned", items) + } + return nil + }, + }, + }, + }) +} + +func TestAccKVValue_lifecycle(t *testing.T) { + key := kvAcceptanceKey() + withValue := func(value string) string { + return fmt.Sprintf(` +resource "quicknode_kv_value" "test" { + key = %q + value = %s +} + +data "quicknode_kv_value" "test" { + key = quicknode_kv_value.test.key + depends_on = [quicknode_kv_value.test] +} +`, key, value) + } + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckKVDestroyed, + Steps: []resource.TestStep{ + { + Config: withValue(`"100"`), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("quicknode_kv_value.test", "value", "100"), + resource.TestCheckResourceAttr("data.quicknode_kv_value.test", "value", "100"), + ), + }, + { + ResourceName: "quicknode_kv_value.test", + ImportState: true, + ImportStateId: key, + ImportStateVerify: true, + }, + { + Config: withValue(`jsonencode({ min = 1, max = 5 })`), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("quicknode_kv_value.test", "value", `{"max":5,"min":1}`), + resource.TestCheckResourceAttr("data.quicknode_kv_value.test", "value", `{"max":5,"min":1}`), + ), + }, + }, + }) +} diff --git a/internal/provider/kv_list_data_source.go b/internal/provider/kv_list_data_source.go new file mode 100644 index 0000000..e114f9f --- /dev/null +++ b/internal/provider/kv_list_data_source.go @@ -0,0 +1,72 @@ +package provider + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-framework/datasource" + "github.com/hashicorp/terraform-plugin-framework/datasource/schema" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +var _ datasource.DataSourceWithConfigure = (*kvListDataSource)(nil) + +type kvListDataSource struct { + client *client.Client +} + +type kvListDataSourceModel struct { + Key types.String `tfsdk:"key"` + Items types.Set `tfsdk:"items"` +} + +func NewKVListDataSource() datasource.DataSource { + return &kvListDataSource{} +} + +func (d *kvListDataSource) Metadata(_ context.Context, req datasource.MetadataRequest, resp *datasource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_kv_list" +} + +func (d *kvListDataSource) Schema(_ context.Context, _ datasource.SchemaRequest, resp *datasource.SchemaResponse) { + resp.Schema = schema.Schema{ + MarkdownDescription: "Items in a Quicknode Key-Value Store list. A missing list reads as empty.", + Attributes: map[string]schema.Attribute{ + "key": kvDataSourceKey("Name of the list."), + "items": schema.SetAttribute{ + Computed: true, + ElementType: types.StringType, + MarkdownDescription: "Items in the list.", + }, + }, + } +} + +func (d *kvListDataSource) Configure(_ context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) { + if configured := kvDataSourceClient(req, resp, "key-value list"); configured != nil { + d.client = configured + } +} + +func (d *kvListDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) { + var config kvListDataSourceModel + resp.Diagnostics.Append(req.Config.Get(ctx, &config)...) + if resp.Diagnostics.HasError() { + return + } + + items, err := d.client.GetKVList(ctx, config.Key.ValueString()) + if err != nil { + resp.Diagnostics.AddError("Could not read the key-value list", err.Error()) + return + } + + set, diags := stringSet(items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + config.Items = set + resp.Diagnostics.Append(resp.State.Set(ctx, &config)...) +} diff --git a/internal/provider/kv_list_items_resource.go b/internal/provider/kv_list_items_resource.go new file mode 100644 index 0000000..1934631 --- /dev/null +++ b/internal/provider/kv_list_items_resource.go @@ -0,0 +1,206 @@ +package provider + +import ( + "context" + "fmt" + "strings" + + "github.com/hashicorp/terraform-plugin-framework-validators/setvalidator" + "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" + "github.com/hashicorp/terraform-plugin-framework/path" + "github.com/hashicorp/terraform-plugin-framework/resource" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +var _ resource.Resource = (*kvListItemsResource)(nil) +var _ resource.ResourceWithConfigure = (*kvListItemsResource)(nil) +var _ resource.ResourceWithImportState = (*kvListItemsResource)(nil) + +const kvListItemsConflictPreview = 5 + +type kvListItemsResource struct { + client *client.Client +} + +type kvListItemsResourceModel struct { + ID types.String `tfsdk:"id"` + ListKey types.String `tfsdk:"list_key"` + Items types.Set `tfsdk:"items"` +} + +func NewKVListItemsResource() resource.Resource { + return &kvListItemsResource{} +} + +func (r *kvListItemsResource) Metadata(_ context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_kv_list_items" +} + +func (r *kvListItemsResource) Schema(_ context.Context, _ resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = schema.Schema{ + MarkdownDescription: "Items in a Quicknode Key-Value Store list. Terraform manages only these items and leaves the rest of the list alone, " + + "for lists that Streams filters or other tools also write to. Adding items to a missing list creates it.\n\n" + + "Create fails if any of the items are already in the list, so destroy never removes items Terraform didn't add. " + + "To take over existing items, import the resource and apply. " + + "Don't use it with a `quicknode_kv_list` for the same key, since that resource removes items it doesn't know about.", + Attributes: map[string]schema.Attribute{ + "id": kvIDAttribute("Same as `list_key`."), + "list_key": kvKeyAttribute("Name of the list the items belong to."), + "items": schema.SetAttribute{ + Required: true, + ElementType: types.StringType, + MarkdownDescription: "Items this resource adds to the list. Items are case-sensitive, so `0xABC` and `0xabc` are different items. " + + "Changes are sent as additions and removals, up to 1500 items per request. Items removed outside Terraform are added back on the next apply.", + Validators: []validator.Set{ + setvalidator.SizeAtLeast(1), + setvalidator.ValueStringsAre(stringvalidator.LengthAtLeast(1)), + }, + }, + }, + } +} + +func (r *kvListItemsResource) Configure(_ context.Context, req resource.ConfigureRequest, resp *resource.ConfigureResponse) { + if req.ProviderData == nil { + return + } + data, ok := req.ProviderData.(providerData) + if !ok { + resp.Diagnostics.AddError("Unexpected provider data", fmt.Sprintf("The key-value list items resource expected providerData, got %T.", req.ProviderData)) + return + } + r.client = data.Client +} + +func (r *kvListItemsResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var plan kvListItemsResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + if resp.Diagnostics.HasError() { + return + } + listKey := plan.ListKey.ValueString() + + items, diags := setStrings(ctx, plan.Items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + existing, err := r.client.GetKVList(ctx, listKey) + if err != nil { + resp.Diagnostics.AddError("Could not read the key-value list", err.Error()) + return + } + if conflicts := setIntersection(items, existing); len(conflicts) > 0 { + preview := conflicts[:min(len(conflicts), kvListItemsConflictPreview)] + resp.Diagnostics.AddAttributeError(path.Root("items"), "Items are already in the list", + fmt.Sprintf("List %q already holds %d of these items, including %s. To manage them, import this resource with the list key %q and apply.", + listKey, len(conflicts), strings.Join(quoteAll(preview), ", "), listKey)) + return + } + + if err := r.client.UpdateKVList(ctx, listKey, items, nil); err != nil { + resp.Diagnostics.AddError("Could not add the items to the key-value list", err.Error()) + return + } + + plan.ID = types.StringValue(listKey) + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +// Read keeps only the items this resource added that are still in the list, +// so items other writers add never show up as a change. +func (r *kvListItemsResource) Read(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { + var state kvListItemsResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + managed, diags := setStrings(ctx, state.Items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + existing, err := r.client.GetKVList(ctx, state.ListKey.ValueString()) + if err != nil { + resp.Diagnostics.AddError("Could not read the key-value list", err.Error()) + return + } + + set, diags := stringSet(setIntersection(managed, existing)) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + state.ID = state.ListKey + state.Items = set + resp.Diagnostics.Append(resp.State.Set(ctx, &state)...) +} + +func (r *kvListItemsResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) { + var plan, state kvListItemsResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + planned, diags := setStrings(ctx, plan.Items) + resp.Diagnostics.Append(diags...) + current, diags := setStrings(ctx, state.Items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + add := setDifference(planned, current) + remove := setDifference(current, planned) + if err := r.client.UpdateKVList(ctx, state.ListKey.ValueString(), add, remove); err != nil { + resp.Diagnostics.AddError("Could not update the key-value list", err.Error()) + return + } + + plan.ID = state.ID + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +func (r *kvListItemsResource) Delete(ctx context.Context, req resource.DeleteRequest, resp *resource.DeleteResponse) { + var state kvListItemsResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + items, diags := setStrings(ctx, state.Items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + if err := r.client.UpdateKVList(ctx, state.ListKey.ValueString(), nil, items); err != nil { + resp.Diagnostics.AddError("Could not remove the items from the key-value list", err.Error()) + } +} + +// ImportState takes the list key and starts with no items, since the list +// alone cannot say which items this resource should own. The next apply takes +// over the configured items, including any that are already in the list. +func (r *kvListItemsResource) ImportState(ctx context.Context, req resource.ImportStateRequest, resp *resource.ImportStateResponse) { + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("id"), req.ID)...) + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("list_key"), req.ID)...) + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("items"), types.SetValueMust(types.StringType, nil))...) +} + +func quoteAll(items []string) []string { + quoted := make([]string, len(items)) + for index, item := range items { + quoted[index] = fmt.Sprintf("%q", item) + } + return quoted +} diff --git a/internal/provider/kv_list_resource.go b/internal/provider/kv_list_resource.go new file mode 100644 index 0000000..5cabd5b --- /dev/null +++ b/internal/provider/kv_list_resource.go @@ -0,0 +1,178 @@ +package provider + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-plugin-framework-validators/setvalidator" + "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" + "github.com/hashicorp/terraform-plugin-framework/path" + "github.com/hashicorp/terraform-plugin-framework/resource" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +var _ resource.Resource = (*kvListResource)(nil) +var _ resource.ResourceWithConfigure = (*kvListResource)(nil) +var _ resource.ResourceWithImportState = (*kvListResource)(nil) + +type kvListResource struct { + client *client.Client +} + +type kvListResourceModel struct { + ID types.String `tfsdk:"id"` + Key types.String `tfsdk:"key"` + Items types.Set `tfsdk:"items"` +} + +func NewKVListResource() resource.Resource { + return &kvListResource{} +} + +func (r *kvListResource) Metadata(_ context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_kv_list" +} + +func (r *kvListResource) Schema(_ context.Context, _ resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = schema.Schema{ + MarkdownDescription: "A list in the Quicknode Key-Value Store. Streams filters can read and update it, for example a set of wallet addresses to watch.\n\n" + + "This resource owns the whole list: items added outside Terraform are removed on the next apply. " + + "For a list that a filter or another tool also writes to, use `quicknode_kv_list_items`. Don't manage the same list with both.", + Attributes: map[string]schema.Attribute{ + "id": kvIDAttribute("Same as `key`."), + "key": kvKeyAttribute("Name of the list, as filters refer to it."), + "items": schema.SetAttribute{ + Required: true, + ElementType: types.StringType, + MarkdownDescription: "Items in the list. Items are case-sensitive, so `0xABC` and `0xabc` are different items. " + + "Changes are sent as additions and removals, up to 1500 items per request.", + Validators: []validator.Set{ + setvalidator.SizeAtLeast(1), + setvalidator.ValueStringsAre(stringvalidator.LengthAtLeast(1)), + }, + }, + }, + } +} + +func (r *kvListResource) Configure(_ context.Context, req resource.ConfigureRequest, resp *resource.ConfigureResponse) { + if req.ProviderData == nil { + return + } + data, ok := req.ProviderData.(providerData) + if !ok { + resp.Diagnostics.AddError("Unexpected provider data", fmt.Sprintf("The key-value list resource expected providerData, got %T.", req.ProviderData)) + return + } + r.client = data.Client +} + +func (r *kvListResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var plan kvListResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + if resp.Diagnostics.HasError() { + return + } + key := plan.Key.ValueString() + + items, diags := setStrings(ctx, plan.Items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + existing, err := r.client.GetKVList(ctx, key) + if err != nil { + resp.Diagnostics.AddError("Could not check for an existing key-value list", err.Error()) + return + } + if len(existing) > 0 { + resp.Diagnostics.AddAttributeError(path.Root("key"), "Key-value list already exists", + fmt.Sprintf("A list named %q already exists with %d items. Import it with `terraform import` to manage it, or choose another key.", key, len(existing))) + return + } + + if err := r.client.CreateKVList(ctx, key, items); err != nil { + resp.Diagnostics.AddError("Could not create the key-value list", err.Error()) + return + } + + plan.ID = types.StringValue(key) + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +func (r *kvListResource) Read(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { + var state kvListResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + items, err := r.client.GetKVList(ctx, state.Key.ValueString()) + if err != nil { + resp.Diagnostics.AddError("Could not read the key-value list", err.Error()) + return + } + if len(items) == 0 { + resp.State.RemoveResource(ctx) + return + } + + set, diags := stringSet(items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + state.ID = state.Key + state.Items = set + resp.Diagnostics.Append(resp.State.Set(ctx, &state)...) +} + +func (r *kvListResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) { + var plan, state kvListResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + planned, diags := setStrings(ctx, plan.Items) + resp.Diagnostics.Append(diags...) + current, diags := setStrings(ctx, state.Items) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + add := setDifference(planned, current) + remove := setDifference(current, planned) + if err := r.client.UpdateKVList(ctx, state.Key.ValueString(), add, remove); err != nil { + resp.Diagnostics.AddError("Could not update the key-value list", err.Error()) + return + } + + plan.ID = state.ID + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +func (r *kvListResource) Delete(ctx context.Context, req resource.DeleteRequest, resp *resource.DeleteResponse) { + var state kvListResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + if err := r.client.DeleteKVList(ctx, state.Key.ValueString()); err != nil { + resp.Diagnostics.AddError("Could not delete the key-value list", err.Error()) + } +} + +// ImportState takes the list's key. +func (r *kvListResource) ImportState(ctx context.Context, req resource.ImportStateRequest, resp *resource.ImportStateResponse) { + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("id"), req.ID)...) + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("key"), req.ID)...) +} diff --git a/internal/provider/kv_value_data_source.go b/internal/provider/kv_value_data_source.go new file mode 100644 index 0000000..5c440db --- /dev/null +++ b/internal/provider/kv_value_data_source.go @@ -0,0 +1,65 @@ +package provider + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-framework/datasource" + "github.com/hashicorp/terraform-plugin-framework/datasource/schema" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +var _ datasource.DataSourceWithConfigure = (*kvValueDataSource)(nil) + +type kvValueDataSource struct { + client *client.Client +} + +type kvValueDataSourceModel struct { + Key types.String `tfsdk:"key"` + Value types.String `tfsdk:"value"` +} + +func NewKVValueDataSource() datasource.DataSource { + return &kvValueDataSource{} +} + +func (d *kvValueDataSource) Metadata(_ context.Context, req datasource.MetadataRequest, resp *datasource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_kv_value" +} + +func (d *kvValueDataSource) Schema(_ context.Context, _ datasource.SchemaRequest, resp *datasource.SchemaResponse) { + resp.Schema = schema.Schema{ + MarkdownDescription: "A value in the Quicknode Key-Value Store. Reading a missing key is an error.", + Attributes: map[string]schema.Attribute{ + "key": kvDataSourceKey("Name of the value."), + "value": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "The stored value.", + }, + }, + } +} + +func (d *kvValueDataSource) Configure(_ context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) { + if configured := kvDataSourceClient(req, resp, "key-value value"); configured != nil { + d.client = configured + } +} + +func (d *kvValueDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) { + var config kvValueDataSourceModel + resp.Diagnostics.Append(req.Config.Get(ctx, &config)...) + if resp.Diagnostics.HasError() { + return + } + + value, err := d.client.GetKVValue(ctx, config.Key.ValueString()) + if err != nil { + resp.Diagnostics.AddError("Could not read the key-value value", err.Error()) + return + } + config.Value = types.StringValue(value) + resp.Diagnostics.Append(resp.State.Set(ctx, &config)...) +} diff --git a/internal/provider/kv_value_resource.go b/internal/provider/kv_value_resource.go new file mode 100644 index 0000000..7b820c5 --- /dev/null +++ b/internal/provider/kv_value_resource.go @@ -0,0 +1,150 @@ +package provider + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" + "github.com/hashicorp/terraform-plugin-framework/path" + "github.com/hashicorp/terraform-plugin-framework/resource" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +var _ resource.Resource = (*kvValueResource)(nil) +var _ resource.ResourceWithConfigure = (*kvValueResource)(nil) +var _ resource.ResourceWithImportState = (*kvValueResource)(nil) + +type kvValueResource struct { + client *client.Client +} + +type kvValueResourceModel struct { + ID types.String `tfsdk:"id"` + Key types.String `tfsdk:"key"` + Value types.String `tfsdk:"value"` +} + +func NewKVValueResource() resource.Resource { + return &kvValueResource{} +} + +func (r *kvValueResource) Metadata(_ context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_kv_value" +} + +func (r *kvValueResource) Schema(_ context.Context, _ resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = schema.Schema{ + MarkdownDescription: "A value in the Quicknode Key-Value Store. Streams filters can read and update it, for example a threshold or the last block processed.\n\n" + + "A filter that writes the value shows up as a change, and the next apply sets it back. Values and lists are separate, so a value and a list can share a key.", + Attributes: map[string]schema.Attribute{ + "id": kvIDAttribute("Same as `key`."), + "key": kvKeyAttribute("Name of the value, as filters refer to it."), + "value": schema.StringAttribute{ + Required: true, + MarkdownDescription: "The value, as a string. Use `jsonencode` for structured data. Terraform state holds it in plain text.", + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + }, + }, + } +} + +func (r *kvValueResource) Configure(_ context.Context, req resource.ConfigureRequest, resp *resource.ConfigureResponse) { + if req.ProviderData == nil { + return + } + data, ok := req.ProviderData.(providerData) + if !ok { + resp.Diagnostics.AddError("Unexpected provider data", fmt.Sprintf("The key-value value resource expected providerData, got %T.", req.ProviderData)) + return + } + r.client = data.Client +} + +func (r *kvValueResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var plan kvValueResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + if resp.Diagnostics.HasError() { + return + } + key := plan.Key.ValueString() + + _, err := r.client.GetKVValue(ctx, key) + if err == nil { + resp.Diagnostics.AddAttributeError(path.Root("key"), "Key-value value already exists", + fmt.Sprintf("A value named %q already exists. Import it with `terraform import` to manage it, or choose another key.", key)) + return + } + if !client.IsNotFound(err) { + resp.Diagnostics.AddError("Could not check for an existing key-value value", err.Error()) + return + } + + if err := r.client.SetKVValue(ctx, key, plan.Value.ValueString()); err != nil { + resp.Diagnostics.AddError("Could not create the key-value value", err.Error()) + return + } + + plan.ID = types.StringValue(key) + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +func (r *kvValueResource) Read(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { + var state kvValueResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + value, err := r.client.GetKVValue(ctx, state.Key.ValueString()) + if client.IsNotFound(err) { + resp.State.RemoveResource(ctx) + return + } + if err != nil { + resp.Diagnostics.AddError("Could not read the key-value value", err.Error()) + return + } + + state.ID = state.Key + state.Value = types.StringValue(value) + resp.Diagnostics.Append(resp.State.Set(ctx, &state)...) +} + +func (r *kvValueResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) { + var plan, state kvValueResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + if err := r.client.SetKVValue(ctx, state.Key.ValueString(), plan.Value.ValueString()); err != nil { + resp.Diagnostics.AddError("Could not update the key-value value", err.Error()) + return + } + + plan.ID = state.ID + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +func (r *kvValueResource) Delete(ctx context.Context, req resource.DeleteRequest, resp *resource.DeleteResponse) { + var state kvValueResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + if err := r.client.DeleteKVValue(ctx, state.Key.ValueString()); err != nil { + resp.Diagnostics.AddError("Could not delete the key-value value", err.Error()) + } +} + +// ImportState takes the value's key. +func (r *kvValueResource) ImportState(ctx context.Context, req resource.ImportStateRequest, resp *resource.ImportStateResponse) { + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("id"), req.ID)...) + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("key"), req.ID)...) +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 0f8963a..be6fd0d 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -105,6 +105,7 @@ func (p *quicknodeProvider) Configure(ctx context.Context, req provider.Configur quicknode, err := client.New(apiKey, client.WithBaseURL(config.BaseURL.ValueString()), client.WithRequestsPerSecond(int(config.RequestsPerSecond.ValueInt64())), + client.WithUserAgent(userAgent(p.version, req.TerraformVersion)), ) if err != nil { resp.Diagnostics.AddError("Could not build the Quicknode API client", err.Error()) @@ -137,6 +138,10 @@ func (p *quicknodeProvider) Resources(_ context.Context) []func() resource.Resou NewRequestFilterResource, NewRateLimitsResource, NewMethodRateLimitResource, + NewStreamResource, + NewKVListResource, + NewKVListItemsResource, + NewKVValueResource, } return append(resources, securityEntryResources()...) } @@ -147,5 +152,8 @@ func (p *quicknodeProvider) DataSources(_ context.Context) []func() datasource.D NewEndpointDataSource, NewEndpointsDataSource, NewEndpointURLsDataSource, + NewStreamDataSource, + NewKVListDataSource, + NewKVValueDataSource, } } diff --git a/internal/provider/stream_acceptance_test.go b/internal/provider/stream_acceptance_test.go new file mode 100644 index 0000000..f21fd8c --- /dev/null +++ b/internal/provider/stream_acceptance_test.go @@ -0,0 +1,237 @@ +package provider_test + +import ( + "context" + "fmt" + "os" + "regexp" + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/plancheck" + "github.com/hashicorp/terraform-plugin-testing/terraform" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +// The streams in these tests stay paused, so nothing is ever delivered to the +// example.com receiver. Quicknode does not contact a webhook when a stream is +// created, only when it delivers. +const streamAcceptanceReceiver = "https://example.com/quicknode-tfacc" + +const streamAcceptanceFilter = `function main(stream) { + return { number: stream.data[0].number }; +} +` + +func testAccCheckStreamsDestroyed(state *terraform.State) error { + quicknode, err := client.New(os.Getenv("QUICKNODE_API_KEY")) + if err != nil { + return fmt.Errorf("could not build a client to confirm the destroy: %w", err) + } + + for name, resourceState := range state.RootModule().Resources { + if resourceState.Type != "quicknode_stream" { + continue + } + id := resourceState.Primary.ID + stream, err := quicknode.GetStream(context.Background(), id) + if client.IsNotFound(err) { + continue + } + if err != nil { + return fmt.Errorf("%s: could not confirm stream %s was destroyed: %w", name, id, err) + } + return fmt.Errorf("%s: stream %s still exists after destroy, with status %q", name, id, stream.Status) + } + return nil +} + +// streamConfig renders a Sepolia stream. settings is added to the resource +// body and destination replaces the webhook destination when set. +func streamConfig(name, status, settings, destination string) string { + if destination == "" { + destination = fmt.Sprintf(`{ + webhook = { + url = %q + headers = { "X-Tfacc" = "true" } + } + }`, streamAcceptanceReceiver) + } + return fmt.Sprintf(` +resource "quicknode_stream" "test" { + name = %q + network = %q + dataset = "block" + region = "usa_east" + status = %q + filter_function = %q +%s + destination = %s +} +`, name, acceptanceNetwork, status, streamAcceptanceFilter, settings, destination) +} + +func TestAccStream_lifecycle(t *testing.T) { + extraDestination := fmt.Sprintf(` + extra_destinations = [{ + webhook = { + url = %q + compression = "gzip" + } + }] +`, streamAcceptanceReceiver+"/extra") + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckStreamsDestroyed, + Steps: []resource.TestStep{ + { + Config: streamConfig("tfacc-stream", "paused", "", ""), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttrSet("quicknode_stream.test", "id"), + resource.TestCheckResourceAttr("quicknode_stream.test", "state", "paused"), + resource.TestCheckResourceAttr("quicknode_stream.test", "filter_function", streamAcceptanceFilter), + resource.TestCheckResourceAttrSet("quicknode_stream.test", "start_range"), + resource.TestCheckNoResourceAttr("quicknode_stream.test", "end_range"), + resource.TestMatchResourceAttr("quicknode_stream.test", "destination.webhook.security_token", regexp.MustCompile(`.+`)), + resource.TestCheckResourceAttr("quicknode_stream.test", "destination.webhook.post_timeout_sec", "30"), + resource.TestCheckResourceAttr("quicknode_stream.test", "destination.webhook.max_retry", "3"), + resource.TestCheckNoResourceAttr("quicknode_stream.test", "extra_destinations"), + ), + }, + { + ResourceName: "quicknode_stream.test", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: streamConfig("tfacc-stream-renamed", "paused", extraDestination, ""), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction("quicknode_stream.test", plancheck.ResourceActionUpdate), + }, + }, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("quicknode_stream.test", "name", "tfacc-stream-renamed"), + resource.TestCheckResourceAttr("quicknode_stream.test", "extra_destinations.#", "1"), + resource.TestCheckResourceAttr("quicknode_stream.test", "extra_destinations.0.webhook.compression", "gzip"), + resource.TestCheckNoResourceAttr("quicknode_stream.test", "extra_destinations.0.webhook.security_token"), + ), + }, + { + Config: streamConfig("tfacc-stream-renamed", "paused", "", ""), + Check: resource.TestCheckNoResourceAttr("quicknode_stream.test", "extra_destinations"), + }, + { + Config: streamConfig("tfacc-stream-renamed", "paused", " extra_destinations = []\n", ""), + Check: resource.TestCheckResourceAttr("quicknode_stream.test", "extra_destinations.#", "0"), + }, + { + Config: streamConfig("tfacc-stream-renamed", "paused", "", ""), + Check: resource.TestCheckNoResourceAttr("quicknode_stream.test", "extra_destinations"), + }, + }, + }) +} + +func TestAccStreamDataSource_byID(t *testing.T) { + config := streamConfig("tfacc-stream-data-source", "paused", "", "") + ` +data "quicknode_stream" "test" { + id = quicknode_stream.test.id +} +` + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckStreamsDestroyed, + Steps: []resource.TestStep{ + { + Config: config, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttrPair("data.quicknode_stream.test", "name", "quicknode_stream.test", "name"), + resource.TestCheckResourceAttrPair("data.quicknode_stream.test", "status", "quicknode_stream.test", "state"), + resource.TestCheckResourceAttrPair("data.quicknode_stream.test", "start_range", "quicknode_stream.test", "start_range"), + resource.TestCheckResourceAttrPair("data.quicknode_stream.test", "filter_function", "quicknode_stream.test", "filter_function"), + resource.TestCheckResourceAttrPair( + "data.quicknode_stream.test", "destination.webhook.security_token", + "quicknode_stream.test", "destination.webhook.security_token"), + resource.TestCheckResourceAttr("data.quicknode_stream.test", "destination.webhook.headers.X-Tfacc", "true"), + resource.TestCheckNoResourceAttr("data.quicknode_stream.test", "destination.s3"), + resource.TestCheckResourceAttrSet("data.quicknode_stream.test", "sequence"), + ), + }, + }, + }) +} + +// TestAccStream_planRules checks the rules the provider enforces at plan time. +// None of the plan-only steps apply, so the S3 credentials are never used. +func TestAccStream_planRules(t *testing.T) { + const s3Destination = `{ + s3 = { + bucket = "tfacc-never-created" + region = "us-east-1" + access_key = "AKIAEXAMPLE" + secret_key = "not-a-real-secret" + } + }` + const conditionalDestination = `{ + webhook = var.use_postgres ? null : { url = "https://example.com/quicknode-tfacc" } + postgres = var.use_postgres ? { + host = "db.example.com" + database = "tfacc" + table_name = "blocks" + username = "tfacc" + password = "not-a-real-password" + } : null + }` + const useThePostgresDestination = ` +variable "use_postgres" { + type = bool + default = true +} +` + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: protoV6ProviderFactories, + CheckDestroy: testAccCheckStreamsDestroyed, + Steps: []resource.TestStep{ + { + Config: streamConfig("tfacc-stream-rules", "paused", " elastic_batch_enabled = true\n end_range = 100", ""), + PlanOnly: true, + ExpectError: regexp.MustCompile(`Elastic batching needs an open-ended stream`), + }, + { + Config: streamConfig("tfacc-stream-rules", "paused", "", ""), + }, + { + Config: streamConfig("tfacc-stream-rules", "active", " resume_from = \"latest\"\n start_range = 1", ""), + PlanOnly: true, + ExpectError: regexp.MustCompile(`resume_from = "latest" conflicts with start_range`), + }, + { + Config: streamConfig("tfacc-stream-rules", "paused", "", s3Destination), + PlanOnly: true, + ExpectNonEmptyPlan: true, + ConfigPlanChecks: resource.ConfigPlanChecks{ + PostApplyPreRefresh: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction("quicknode_stream.test", plancheck.ResourceActionDestroyBeforeCreate), + }, + }, + }, + { + Config: useThePostgresDestination + streamConfig("tfacc-stream-rules", "paused", "", conditionalDestination), + PlanOnly: true, + ExpectNonEmptyPlan: true, + ConfigPlanChecks: resource.ConfigPlanChecks{ + PostApplyPreRefresh: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction("quicknode_stream.test", plancheck.ResourceActionDestroyBeforeCreate), + }, + }, + }, + }, + }) +} diff --git a/internal/provider/stream_data_source.go b/internal/provider/stream_data_source.go new file mode 100644 index 0000000..d90e872 --- /dev/null +++ b/internal/provider/stream_data_source.go @@ -0,0 +1,195 @@ +package provider + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/datasource" + "github.com/hashicorp/terraform-plugin-framework/datasource/schema" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +var _ datasource.DataSource = (*streamDataSource)(nil) +var _ datasource.DataSourceWithConfigure = (*streamDataSource)(nil) + +var streamSecretAttributes = map[string]bool{ + "security_token": true, + "secret_key": true, + "sas_token": true, + "password": true, + "ssl_key_pem": true, +} + +type streamDataSource struct { + client *client.Client +} + +type streamDataSourceModel struct { + ID types.String `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Network types.String `tfsdk:"network"` + Dataset types.String `tfsdk:"dataset"` + Region types.String `tfsdk:"region"` + Status types.String `tfsdk:"status"` + FilterFunction types.String `tfsdk:"filter_function"` + FilterLanguage types.String `tfsdk:"filter_language"` + StartRange types.Int64 `tfsdk:"start_range"` + EndRange types.Int64 `tfsdk:"end_range"` + Sequence types.Int64 `tfsdk:"sequence"` + DatasetBatchSize types.Int64 `tfsdk:"dataset_batch_size"` + ElasticBatchEnabled types.Bool `tfsdk:"elastic_batch_enabled"` + FixBlockReorgs types.Bool `tfsdk:"fix_block_reorgs"` + KeepDistanceFromTip types.Int64 `tfsdk:"keep_distance_from_tip"` + RestreamBatchOnReorg types.Bool `tfsdk:"restream_batch_on_reorg"` + NotificationEmail types.String `tfsdk:"notification_email"` + Destination types.Object `tfsdk:"destination"` + ExtraDestinations types.List `tfsdk:"extra_destinations"` +} + +func NewStreamDataSource() datasource.DataSource { + return &streamDataSource{} +} + +func (d *streamDataSource) Metadata(_ context.Context, req datasource.MetadataRequest, resp *datasource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_stream" +} + +func (d *streamDataSource) Schema(_ context.Context, _ datasource.SchemaRequest, resp *datasource.SchemaResponse) { + computedString := func(description string) schema.StringAttribute { + return schema.StringAttribute{Computed: true, MarkdownDescription: description} + } + computedInt := func(description string) schema.Int64Attribute { + return schema.Int64Attribute{Computed: true, MarkdownDescription: description} + } + computedBool := func(description string) schema.BoolAttribute { + return schema.BoolAttribute{Computed: true, MarkdownDescription: description} + } + resp.Schema = schema.Schema{ + MarkdownDescription: "An existing stream, looked up by `id`. " + + "Destination credentials are stored in Terraform state, so keep state encrypted and remote.", + Attributes: map[string]schema.Attribute{ + "id": schema.StringAttribute{ + Required: true, + MarkdownDescription: "Stream id.", + }, + "name": computedString("Name shown in the dashboard."), + "network": computedString("Network slug."), + "dataset": computedString("What each batch carries."), + "region": computedString("Region the stream runs in."), + "status": computedString("`active`, `paused`, `terminated` or `completed`."), + "filter_function": computedString("Source code of the filter, or null if the stream has none."), + "filter_language": computedString("`javascript` or `go`."), + "start_range": computedInt("First block the stream delivers."), + "end_range": computedInt("Last block the stream delivers, or null if it runs until paused."), + "sequence": computedInt("Last block delivered. It moves while the stream runs."), + "dataset_batch_size": computedInt("Blocks per batch."), + "elastic_batch_enabled": computedBool("Whether batches grow while the stream catches up."), + "fix_block_reorgs": computedBool("Whether corrected blocks are delivered when the network reorganizes."), + "keep_distance_from_tip": computedInt("How many blocks the stream stays behind the newest block."), + "restream_batch_on_reorg": computedBool("Whether the whole batch is delivered again when a block in it is reorganized."), + "notification_email": computedString("Address notified when the stream is terminated, or null."), + "destination": schema.SingleNestedAttribute{ + Computed: true, + MarkdownDescription: "Where the stream delivers. One type is set; the others are null.", + Attributes: computedDestinationAttributes(), + }, + "extra_destinations": schema.ListNestedAttribute{ + Computed: true, + MarkdownDescription: "More destinations that receive every batch, or null if there are none.", + NestedObject: schema.NestedAttributeObject{Attributes: computedDestinationAttributes()}, + }, + }, + } +} + +// computedDestinationAttributes mirrors the resource's destination types as +// read-only attributes, built from the same attribute types so the two cannot +// drift apart. +func computedDestinationAttributes() map[string]schema.Attribute { + attributes := map[string]schema.Attribute{} + for name, destinationType := range streamDestinationAttrTypes { + children := map[string]schema.Attribute{} + for childName, childType := range destinationType.(types.ObjectType).AttrTypes { + description := fmt.Sprintf("See `destination.%s.%s` on the `quicknode_stream` resource.", name, childName) + children[childName] = computedAttribute(childType, streamSecretAttributes[childName], description) + } + attributes[name] = schema.SingleNestedAttribute{ + Computed: true, + MarkdownDescription: fmt.Sprintf("Settings when the destination is `%s`, otherwise null.", name), + Attributes: children, + } + } + return attributes +} + +func computedAttribute(attrType attr.Type, sensitive bool, description string) schema.Attribute { + switch attrType { + case types.Int64Type: + return schema.Int64Attribute{Computed: true, MarkdownDescription: description} + case types.BoolType: + return schema.BoolAttribute{Computed: true, MarkdownDescription: description} + case types.StringType: + return schema.StringAttribute{Computed: true, Sensitive: sensitive, MarkdownDescription: description} + default: + return schema.MapAttribute{Computed: true, ElementType: types.StringType, MarkdownDescription: description} + } +} + +func (d *streamDataSource) Configure(_ context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) { + if req.ProviderData == nil { + return + } + data, ok := req.ProviderData.(providerData) + if !ok { + resp.Diagnostics.AddError("Unexpected provider data", fmt.Sprintf("The stream data source expected providerData, got %T.", req.ProviderData)) + return + } + d.client = data.Client +} + +func (d *streamDataSource) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) { + var config streamDataSourceModel + resp.Diagnostics.Append(req.Config.Get(ctx, &config)...) + if resp.Diagnostics.HasError() { + return + } + + stream, err := d.client.GetStream(ctx, config.ID.ValueString()) + if err != nil { + resp.Diagnostics.AddError("Could not read the Quicknode stream", err.Error()) + return + } + + config.Name = types.StringValue(stream.Name) + config.Network = types.StringValue(stream.Network) + config.Dataset = types.StringValue(stream.Dataset) + config.Region = types.StringValue(stream.Region) + config.Status = types.StringValue(stream.Status) + config.FilterFunction = stringOrNull(stream.FilterFunction) + config.FilterLanguage = types.StringValue(stream.FilterLanguage) + config.StartRange = types.Int64Value(stream.StartRange) + config.EndRange = rangeOrNull(stream.EndRange) + config.Sequence = types.Int64Value(stream.Sequence) + config.DatasetBatchSize = types.Int64Value(stream.DatasetBatchSize) + config.ElasticBatchEnabled = types.BoolValue(stream.ElasticBatchEnabled) + config.FixBlockReorgs = types.BoolValue(stream.FixBlockReorgs != 0) + config.KeepDistanceFromTip = types.Int64Value(stream.KeepDistanceFromTip) + config.RestreamBatchOnReorg = types.BoolValue(stream.RestreamBatchOnReorg) + config.NotificationEmail = stringOrNull(stream.NotificationEmail) + + destination, diags := destinationObject(stream.Destination) + resp.Diagnostics.Append(diags...) + config.Destination = destination + + extras, diags := destinationsList(stream.ExtraDestinations) + resp.Diagnostics.Append(diags...) + config.ExtraDestinations = extras + + if resp.Diagnostics.HasError() { + return + } + resp.Diagnostics.Append(resp.State.Set(ctx, &config)...) +} diff --git a/internal/provider/stream_destination.go b/internal/provider/stream_destination.go new file mode 100644 index 0000000..04b3d89 --- /dev/null +++ b/internal/provider/stream_destination.go @@ -0,0 +1,589 @@ +package provider + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-plugin-framework-validators/int64validator" + "github.com/hashicorp/terraform-plugin-framework-validators/mapvalidator" + "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/diag" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/booldefault" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/int64default" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringdefault" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" + "github.com/hashicorp/terraform-plugin-framework/types" + "github.com/hashicorp/terraform-plugin-framework/types/basetypes" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +var webhookDestinationAttrTypes = map[string]attr.Type{ + "url": types.StringType, + "compression": types.StringType, + "headers": types.MapType{ElemType: types.StringType}, + "security_token": types.StringType, + "max_retry": types.Int64Type, + "retry_interval_sec": types.Int64Type, + "post_timeout_sec": types.Int64Type, +} + +var s3DestinationAttrTypes = map[string]attr.Type{ + "endpoint": types.StringType, + "bucket": types.StringType, + "region": types.StringType, + "object_prefix": types.StringType, + "file_type": types.StringType, + "compression": types.StringType, + "use_ssl": types.BoolType, + "access_key": types.StringType, + "secret_key": types.StringType, + "max_retry": types.Int64Type, + "retry_interval_sec": types.Int64Type, +} + +var azureDestinationAttrTypes = map[string]attr.Type{ + "storage_account": types.StringType, + "container": types.StringType, + "blob_prefix": types.StringType, + "file_type": types.StringType, + "compression": types.StringType, + "sas_token": types.StringType, + "max_retry": types.Int64Type, + "retry_interval_sec": types.Int64Type, +} + +var postgresDestinationAttrTypes = map[string]attr.Type{ + "host": types.StringType, + "port": types.Int64Type, + "database": types.StringType, + "table_name": types.StringType, + "username": types.StringType, + "password": types.StringType, + "sslmode": types.StringType, + "max_retry": types.Int64Type, + "retry_interval_sec": types.Int64Type, +} + +var kafkaDestinationAttrTypes = map[string]attr.Type{ + "bootstrap_servers": types.StringType, + "topic_name": types.StringType, + "username": types.StringType, + "password": types.StringType, + "mechanisms": types.StringType, + "protocol": types.StringType, + "compression": types.StringType, + "batch_size": types.Int64Type, + "linger_ms": types.Int64Type, + "max_message_bytes": types.Int64Type, + "timeout_sec": types.Int64Type, + "ssl_ca_pem": types.StringType, + "ssl_certificate_pem": types.StringType, + "ssl_key_pem": types.StringType, + "max_retry": types.Int64Type, + "retry_interval_sec": types.Int64Type, +} + +var streamDestinationAttrTypes = map[string]attr.Type{ + client.DestinationWebhook: types.ObjectType{AttrTypes: webhookDestinationAttrTypes}, + client.DestinationS3: types.ObjectType{AttrTypes: s3DestinationAttrTypes}, + client.DestinationAzure: types.ObjectType{AttrTypes: azureDestinationAttrTypes}, + client.DestinationPostgres: types.ObjectType{AttrTypes: postgresDestinationAttrTypes}, + client.DestinationKafka: types.ObjectType{AttrTypes: kafkaDestinationAttrTypes}, +} + +type streamDestinationModel struct { + Webhook types.Object `tfsdk:"webhook"` + S3 types.Object `tfsdk:"s3"` + Azure types.Object `tfsdk:"azure"` + Postgres types.Object `tfsdk:"postgres"` + Kafka types.Object `tfsdk:"kafka"` +} + +type webhookDestinationModel struct { + URL types.String `tfsdk:"url"` + Compression types.String `tfsdk:"compression"` + Headers types.Map `tfsdk:"headers"` + SecurityToken types.String `tfsdk:"security_token"` + MaxRetry types.Int64 `tfsdk:"max_retry"` + RetryIntervalSec types.Int64 `tfsdk:"retry_interval_sec"` + PostTimeoutSec types.Int64 `tfsdk:"post_timeout_sec"` +} + +type s3DestinationModel struct { + Endpoint types.String `tfsdk:"endpoint"` + Bucket types.String `tfsdk:"bucket"` + Region types.String `tfsdk:"region"` + ObjectPrefix types.String `tfsdk:"object_prefix"` + FileType types.String `tfsdk:"file_type"` + Compression types.String `tfsdk:"compression"` + UseSSL types.Bool `tfsdk:"use_ssl"` + AccessKey types.String `tfsdk:"access_key"` + SecretKey types.String `tfsdk:"secret_key"` + MaxRetry types.Int64 `tfsdk:"max_retry"` + RetryIntervalSec types.Int64 `tfsdk:"retry_interval_sec"` +} + +type azureDestinationModel struct { + StorageAccount types.String `tfsdk:"storage_account"` + Container types.String `tfsdk:"container"` + BlobPrefix types.String `tfsdk:"blob_prefix"` + FileType types.String `tfsdk:"file_type"` + Compression types.String `tfsdk:"compression"` + SASToken types.String `tfsdk:"sas_token"` + MaxRetry types.Int64 `tfsdk:"max_retry"` + RetryIntervalSec types.Int64 `tfsdk:"retry_interval_sec"` +} + +type postgresDestinationModel struct { + Host types.String `tfsdk:"host"` + Port types.Int64 `tfsdk:"port"` + Database types.String `tfsdk:"database"` + TableName types.String `tfsdk:"table_name"` + Username types.String `tfsdk:"username"` + Password types.String `tfsdk:"password"` + SSLMode types.String `tfsdk:"sslmode"` + MaxRetry types.Int64 `tfsdk:"max_retry"` + RetryIntervalSec types.Int64 `tfsdk:"retry_interval_sec"` +} + +type kafkaDestinationModel struct { + BootstrapServers types.String `tfsdk:"bootstrap_servers"` + TopicName types.String `tfsdk:"topic_name"` + Username types.String `tfsdk:"username"` + Password types.String `tfsdk:"password"` + Mechanisms types.String `tfsdk:"mechanisms"` + Protocol types.String `tfsdk:"protocol"` + Compression types.String `tfsdk:"compression"` + BatchSize types.Int64 `tfsdk:"batch_size"` + LingerMs types.Int64 `tfsdk:"linger_ms"` + MaxMessageBytes types.Int64 `tfsdk:"max_message_bytes"` + TimeoutSec types.Int64 `tfsdk:"timeout_sec"` + SSLCAPEM types.String `tfsdk:"ssl_ca_pem"` + SSLCertificatePEM types.String `tfsdk:"ssl_certificate_pem"` + SSLKeyPEM types.String `tfsdk:"ssl_key_pem"` + MaxRetry types.Int64 `tfsdk:"max_retry"` + RetryIntervalSec types.Int64 `tfsdk:"retry_interval_sec"` +} + +func requiredString(description string) schema.StringAttribute { + return schema.StringAttribute{ + Required: true, + MarkdownDescription: description, + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + } +} + +func optionalString(description string) schema.StringAttribute { + return schema.StringAttribute{ + Optional: true, + MarkdownDescription: description, + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + } +} + +func secretString(description string) schema.StringAttribute { + return schema.StringAttribute{ + Optional: true, + Sensitive: true, + MarkdownDescription: description, + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + } +} + +func requiredSecret(description string) schema.StringAttribute { + return schema.StringAttribute{ + Required: true, + Sensitive: true, + MarkdownDescription: description, + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + } +} + +func defaultedString(description, value string, allowed ...string) schema.StringAttribute { + attribute := schema.StringAttribute{ + Optional: true, + Computed: true, + Default: stringdefault.StaticString(value), + MarkdownDescription: fmt.Sprintf("%s Defaults to `%s`.", description, value), + } + if len(allowed) > 0 { + attribute.Validators = []validator.String{stringvalidator.OneOf(allowed...)} + } + return attribute +} + +func defaultedInt(description string, value, minimum int64) schema.Int64Attribute { + return schema.Int64Attribute{ + Optional: true, + Computed: true, + Default: int64default.StaticInt64(value), + MarkdownDescription: fmt.Sprintf("%s Defaults to `%d`.", description, value), + Validators: []validator.Int64{int64validator.AtLeast(minimum)}, + } +} + +func retryAttributes(attributes map[string]schema.Attribute) map[string]schema.Attribute { + attributes["max_retry"] = defaultedInt("Delivery attempts before the stream is terminated.", 3, 1) + attributes["retry_interval_sec"] = defaultedInt("Seconds between delivery attempts.", 1, 1) + return attributes +} + +// streamDestinationAttributes is shared by destination and each entry of +// extra_destinations. Exactly one type may be set; the resource checks that in +// ValidateConfig. +func streamDestinationAttributes() map[string]schema.Attribute { + return map[string]schema.Attribute{ + client.DestinationWebhook: schema.SingleNestedAttribute{ + Optional: true, + MarkdownDescription: "Deliver each batch as an HTTP POST.", + Attributes: retryAttributes(map[string]schema.Attribute{ + "url": requiredString("URL the batches are posted to."), + "compression": defaultedString("`none` or `gzip`.", "none", "none", "gzip"), + "headers": schema.MapAttribute{ + Optional: true, + ElementType: types.StringType, + MarkdownDescription: "Extra request headers. Omit the attribute rather than setting an empty map.", + Validators: []validator.Map{mapvalidator.SizeAtLeast(1)}, + }, + "security_token": schema.StringAttribute{ + Optional: true, + Computed: true, + Sensitive: true, + MarkdownDescription: "Token used to sign each request, so the receiver can verify it came from Quicknode. " + + "When omitted on the primary destination, Quicknode generates one and this attribute reports it.", + PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, + }, + "post_timeout_sec": defaultedInt("Seconds to wait for the receiver to respond.", 30, 1), + }), + }, + client.DestinationS3: schema.SingleNestedAttribute{ + Optional: true, + MarkdownDescription: "Write each batch as an object to Amazon S3 or an S3-compatible store.", + Attributes: retryAttributes(map[string]schema.Attribute{ + "endpoint": defaultedString("S3 endpoint host.", "s3.amazonaws.com"), + "bucket": requiredString("Bucket name."), + "region": optionalString("Bucket region, for example `us-east-1`."), + "object_prefix": defaultedString("Prefix added to each object name. It is joined to the file name with no separator, so end it with `/` to write into a folder.", ""), + "file_type": defaultedString("`.json` or `.parquet`.", ".json", ".json", ".parquet"), + "compression": defaultedString("`none` or `gzip`.", "none", "none", "gzip"), + "use_ssl": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(true), + MarkdownDescription: "Connect to the endpoint over TLS. Defaults to `true`.", + }, + "access_key": requiredString("Access key id."), + "secret_key": requiredSecret("Secret access key."), + }), + }, + client.DestinationAzure: schema.SingleNestedAttribute{ + Optional: true, + MarkdownDescription: "Write each batch as a blob to Azure Storage.", + Attributes: retryAttributes(map[string]schema.Attribute{ + "storage_account": requiredString("Storage account name."), + "container": requiredString("Container name."), + "blob_prefix": defaultedString("Prefix added to each blob name.", ""), + "file_type": defaultedString("`.json` or `.parquet`.", ".json", ".json", ".parquet"), + "compression": defaultedString("`none` or `gzip`.", "none", "none", "gzip"), + "sas_token": requiredSecret("Shared access signature token with write access to the container."), + }), + }, + client.DestinationPostgres: schema.SingleNestedAttribute{ + Optional: true, + MarkdownDescription: "Insert each batch into a PostgreSQL table.", + Attributes: retryAttributes(map[string]schema.Attribute{ + "host": requiredString("Database host."), + "port": defaultedInt("Database port.", 5432, 1), + "database": requiredString("Database name."), + "table_name": requiredString("Table the rows are written to."), + "username": requiredString("Database user."), + "password": requiredSecret("Database password."), + "sslmode": defaultedString("`require` or `disable`.", "require", "require", "disable"), + }), + }, + client.DestinationKafka: schema.SingleNestedAttribute{ + Optional: true, + MarkdownDescription: "Produce each batch as messages to a Kafka topic.", + Attributes: retryAttributes(map[string]schema.Attribute{ + "bootstrap_servers": requiredString("Comma-separated `host:port` list of brokers."), + "topic_name": requiredString("Topic the messages are produced to."), + "username": optionalString("SASL user."), + "password": secretString("SASL password."), + "mechanisms": optionalString("SASL mechanism: `PLAIN`, `GSSAPI`, `SCRAM-SHA-256`, `SCRAM-SHA-512` or `OAUTHBEARER`."), + "protocol": defaultedString("`plaintext`, `ssl`, `sasl_ssl` or `sasl_plaintext`.", "plaintext", "plaintext", "ssl", "sasl_ssl", "sasl_plaintext"), + "compression": defaultedString("`none`, `gzip`, `snappy`, `lz4` or `zstd`.", "zstd", "none", "gzip", "snappy", "lz4", "zstd"), + "batch_size": defaultedInt("Producer batch size in bytes.", 16384, 1), + "linger_ms": defaultedInt("Milliseconds the producer waits to fill a batch.", 5, 0), + "max_message_bytes": defaultedInt("Largest message the producer sends, in bytes.", 1048576, 1), + "timeout_sec": defaultedInt("Seconds to wait for the brokers to acknowledge.", 30, 1), + "ssl_ca_pem": optionalString("PEM-encoded CA certificate that signed the brokers' certificates."), + "ssl_certificate_pem": optionalString("PEM-encoded client certificate."), + "ssl_key_pem": secretString("PEM-encoded client private key."), + }), + }, + } +} + +var streamDestinationObjectType = types.ObjectType{AttrTypes: streamDestinationAttrTypes} + +var objectAsOptions = basetypes.ObjectAsOptions{} + +func knownString(value types.String) string { + if value.IsNull() || value.IsUnknown() { + return "" + } + return value.ValueString() +} + +func destinationFromObject(ctx context.Context, object types.Object) (client.StreamDestination, diag.Diagnostics) { + var diags diag.Diagnostics + var model streamDestinationModel + diags.Append(object.As(ctx, &model, objectAsOptions)...) + if diags.HasError() { + return client.StreamDestination{}, diags + } + + var destination client.StreamDestination + switch { + case !model.Webhook.IsNull(): + var webhook webhookDestinationModel + diags.Append(model.Webhook.As(ctx, &webhook, objectAsOptions)...) + headers := client.HeaderMap{} + if !webhook.Headers.IsNull() && !webhook.Headers.IsUnknown() { + diags.Append(webhook.Headers.ElementsAs(ctx, &headers, false)...) + } + destination.Webhook = &client.WebhookDestination{ + URL: webhook.URL.ValueString(), + Compression: webhook.Compression.ValueString(), + Headers: headers, + SecurityToken: knownString(webhook.SecurityToken), + MaxRetry: webhook.MaxRetry.ValueInt64(), + RetryIntervalSec: webhook.RetryIntervalSec.ValueInt64(), + PostTimeoutSec: webhook.PostTimeoutSec.ValueInt64(), + } + case !model.S3.IsNull(): + var s3 s3DestinationModel + diags.Append(model.S3.As(ctx, &s3, objectAsOptions)...) + destination.S3 = &client.S3Destination{ + Endpoint: s3.Endpoint.ValueString(), + Bucket: s3.Bucket.ValueString(), + Region: knownString(s3.Region), + ObjectPrefix: s3.ObjectPrefix.ValueString(), + FileType: s3.FileType.ValueString(), + FileCompressionType: s3.Compression.ValueString(), + UseSSL: s3.UseSSL.ValueBool(), + AccessKey: s3.AccessKey.ValueString(), + SecretKey: s3.SecretKey.ValueString(), + MaxRetry: s3.MaxRetry.ValueInt64(), + RetryIntervalSec: s3.RetryIntervalSec.ValueInt64(), + } + case !model.Azure.IsNull(): + var azure azureDestinationModel + diags.Append(model.Azure.As(ctx, &azure, objectAsOptions)...) + destination.Azure = &client.AzureDestination{ + StorageAccount: azure.StorageAccount.ValueString(), + Container: azure.Container.ValueString(), + BlobPrefix: azure.BlobPrefix.ValueString(), + FileType: azure.FileType.ValueString(), + FileCompressionType: azure.Compression.ValueString(), + SASToken: azure.SASToken.ValueString(), + MaxRetry: azure.MaxRetry.ValueInt64(), + RetryIntervalSec: azure.RetryIntervalSec.ValueInt64(), + } + case !model.Postgres.IsNull(): + var postgres postgresDestinationModel + diags.Append(model.Postgres.As(ctx, &postgres, objectAsOptions)...) + destination.Postgres = &client.PostgresDestination{ + Host: postgres.Host.ValueString(), + Port: postgres.Port.ValueInt64(), + Database: postgres.Database.ValueString(), + TableName: postgres.TableName.ValueString(), + Username: postgres.Username.ValueString(), + Password: postgres.Password.ValueString(), + SSLMode: postgres.SSLMode.ValueString(), + MaxRetry: postgres.MaxRetry.ValueInt64(), + RetryIntervalSec: postgres.RetryIntervalSec.ValueInt64(), + } + case !model.Kafka.IsNull(): + var kafka kafkaDestinationModel + diags.Append(model.Kafka.As(ctx, &kafka, objectAsOptions)...) + destination.Kafka = &client.KafkaDestination{ + BootstrapServers: kafka.BootstrapServers.ValueString(), + TopicName: kafka.TopicName.ValueString(), + Username: knownString(kafka.Username), + Password: knownString(kafka.Password), + Mechanisms: knownString(kafka.Mechanisms), + Protocol: kafka.Protocol.ValueString(), + CompressionType: kafka.Compression.ValueString(), + BatchSize: kafka.BatchSize.ValueInt64(), + LingerMs: kafka.LingerMs.ValueInt64(), + MaxMessageBytes: kafka.MaxMessageBytes.ValueInt64(), + TimeoutSec: kafka.TimeoutSec.ValueInt64(), + SSLCAPEM: knownString(kafka.SSLCAPEM), + SSLCertificatePEM: knownString(kafka.SSLCertificatePEM), + SSLKeyPEM: knownString(kafka.SSLKeyPEM), + MaxRetry: kafka.MaxRetry.ValueInt64(), + RetryIntervalSec: kafka.RetryIntervalSec.ValueInt64(), + } + default: + diags.AddError("Stream destination has no type", "Set exactly one of `webhook`, `s3`, `azure`, `postgres` or `kafka`.") + } + return destination, diags +} + +func destinationObject(destination client.StreamDestination) (types.Object, diag.Diagnostics) { + var diags diag.Diagnostics + values := map[string]attr.Value{} + for name, attrType := range streamDestinationAttrTypes { + values[name] = types.ObjectNull(attrType.(types.ObjectType).AttrTypes) + } + + var child types.Object + var childDiags diag.Diagnostics + switch { + case destination.Webhook != nil: + webhook := destination.Webhook + headers := types.MapNull(types.StringType) + if len(webhook.Headers) > 0 { + headers, childDiags = types.MapValueFrom(context.Background(), types.StringType, map[string]string(webhook.Headers)) + diags.Append(childDiags...) + } + child, childDiags = types.ObjectValue(webhookDestinationAttrTypes, map[string]attr.Value{ + "url": types.StringValue(webhook.URL), + "compression": types.StringValue(webhook.Compression), + "headers": headers, + "security_token": stringOrNull(webhook.SecurityToken), + "max_retry": types.Int64Value(webhook.MaxRetry), + "retry_interval_sec": types.Int64Value(webhook.RetryIntervalSec), + "post_timeout_sec": types.Int64Value(webhook.PostTimeoutSec), + }) + case destination.S3 != nil: + s3 := destination.S3 + child, childDiags = types.ObjectValue(s3DestinationAttrTypes, map[string]attr.Value{ + "endpoint": types.StringValue(s3.Endpoint), + "bucket": types.StringValue(s3.Bucket), + "region": stringOrNull(s3.Region), + "object_prefix": types.StringValue(s3.ObjectPrefix), + "file_type": types.StringValue(s3.FileType), + "compression": types.StringValue(s3.FileCompressionType), + "use_ssl": types.BoolValue(s3.UseSSL), + "access_key": types.StringValue(s3.AccessKey), + "secret_key": types.StringValue(s3.SecretKey), + "max_retry": types.Int64Value(s3.MaxRetry), + "retry_interval_sec": types.Int64Value(s3.RetryIntervalSec), + }) + case destination.Azure != nil: + azure := destination.Azure + child, childDiags = types.ObjectValue(azureDestinationAttrTypes, map[string]attr.Value{ + "storage_account": types.StringValue(azure.StorageAccount), + "container": types.StringValue(azure.Container), + "blob_prefix": types.StringValue(azure.BlobPrefix), + "file_type": types.StringValue(azure.FileType), + "compression": types.StringValue(azure.FileCompressionType), + "sas_token": types.StringValue(azure.SASToken), + "max_retry": types.Int64Value(azure.MaxRetry), + "retry_interval_sec": types.Int64Value(azure.RetryIntervalSec), + }) + case destination.Postgres != nil: + postgres := destination.Postgres + child, childDiags = types.ObjectValue(postgresDestinationAttrTypes, map[string]attr.Value{ + "host": types.StringValue(postgres.Host), + "port": types.Int64Value(postgres.Port), + "database": types.StringValue(postgres.Database), + "table_name": types.StringValue(postgres.TableName), + "username": types.StringValue(postgres.Username), + "password": types.StringValue(postgres.Password), + "sslmode": types.StringValue(postgres.SSLMode), + "max_retry": types.Int64Value(postgres.MaxRetry), + "retry_interval_sec": types.Int64Value(postgres.RetryIntervalSec), + }) + case destination.Kafka != nil: + kafka := destination.Kafka + child, childDiags = types.ObjectValue(kafkaDestinationAttrTypes, map[string]attr.Value{ + "bootstrap_servers": types.StringValue(kafka.BootstrapServers), + "topic_name": types.StringValue(kafka.TopicName), + "username": stringOrNull(kafka.Username), + "password": stringOrNull(kafka.Password), + "mechanisms": stringOrNull(kafka.Mechanisms), + "protocol": types.StringValue(kafka.Protocol), + "compression": types.StringValue(kafka.CompressionType), + "batch_size": types.Int64Value(kafka.BatchSize), + "linger_ms": types.Int64Value(kafka.LingerMs), + "max_message_bytes": types.Int64Value(kafka.MaxMessageBytes), + "timeout_sec": types.Int64Value(kafka.TimeoutSec), + "ssl_ca_pem": stringOrNull(kafka.SSLCAPEM), + "ssl_certificate_pem": stringOrNull(kafka.SSLCertificatePEM), + "ssl_key_pem": stringOrNull(kafka.SSLKeyPEM), + "max_retry": types.Int64Value(kafka.MaxRetry), + "retry_interval_sec": types.Int64Value(kafka.RetryIntervalSec), + }) + default: + diags.AddError( + "Unsupported stream destination", + fmt.Sprintf("The stream delivers to a %q destination, which the provider cannot manage. Change it in the dashboard to one of webhook, s3, azure, postgres or kafka.", destination.Type()), + ) + return types.ObjectNull(streamDestinationAttrTypes), diags + } + diags.Append(childDiags...) + values[destination.Type()] = child + + object, objectDiags := types.ObjectValue(streamDestinationAttrTypes, values) + diags.Append(objectDiags...) + return object, diags +} + +func destinationsFromList(ctx context.Context, list types.List) ([]client.StreamDestination, diag.Diagnostics) { + var diags diag.Diagnostics + destinations := []client.StreamDestination{} + if list.IsNull() || list.IsUnknown() { + return destinations, diags + } + var objects []types.Object + diags.Append(list.ElementsAs(ctx, &objects, false)...) + for _, object := range objects { + destination, destinationDiags := destinationFromObject(ctx, object) + diags.Append(destinationDiags...) + destinations = append(destinations, destination) + } + return destinations, diags +} + +// destinationsList reports no extra destinations as null, so a configuration +// that omits the attribute sees no difference. +func destinationsList(destinations []client.StreamDestination) (types.List, diag.Diagnostics) { + var diags diag.Diagnostics + if len(destinations) == 0 { + return types.ListNull(streamDestinationObjectType), diags + } + objects := make([]attr.Value, 0, len(destinations)) + for _, destination := range destinations { + object, objectDiags := destinationObject(destination) + diags.Append(objectDiags...) + objects = append(objects, object) + } + list, listDiags := types.ListValue(streamDestinationObjectType, objects) + diags.Append(listDiags...) + return list, diags +} + +// destinationTypeCount counts the types set on one destination, so +// ValidateConfig can require exactly one. known is false while any type is +// still unknown, such as one picked by a conditional on a variable. +func destinationTypeCount(ctx context.Context, object types.Object) (count int, known bool, diags diag.Diagnostics) { + var model streamDestinationModel + diags = object.As(ctx, &model, basetypes.ObjectAsOptions{UnhandledUnknownAsEmpty: true}) + for _, child := range []types.Object{model.Webhook, model.S3, model.Azure, model.Postgres, model.Kafka} { + if child.IsUnknown() { + return 0, false, diags + } + if !child.IsNull() { + count++ + } + } + return count, true, diags +} diff --git a/internal/provider/stream_resource.go b/internal/provider/stream_resource.go new file mode 100644 index 0000000..1664a7c --- /dev/null +++ b/internal/provider/stream_resource.go @@ -0,0 +1,650 @@ +package provider + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-plugin-framework-validators/int64validator" + "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/diag" + "github.com/hashicorp/terraform-plugin-framework/path" + "github.com/hashicorp/terraform-plugin-framework/resource" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/booldefault" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/boolplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/int64default" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/int64planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringdefault" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/quicknode/terraform-provider-quicknode/internal/client" +) + +const ( + resumeFromLast = "last" + resumeFromLatest = "latest" +) + +var _ resource.Resource = (*streamResource)(nil) +var _ resource.ResourceWithConfigure = (*streamResource)(nil) +var _ resource.ResourceWithImportState = (*streamResource)(nil) +var _ resource.ResourceWithModifyPlan = (*streamResource)(nil) +var _ resource.ResourceWithValidateConfig = (*streamResource)(nil) + +type streamResource struct { + client *client.Client +} + +type streamResourceModel struct { + ID types.String `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Network types.String `tfsdk:"network"` + Dataset types.String `tfsdk:"dataset"` + Region types.String `tfsdk:"region"` + Status types.String `tfsdk:"status"` + ResumeFrom types.String `tfsdk:"resume_from"` + State types.String `tfsdk:"state"` + FilterFunction types.String `tfsdk:"filter_function"` + FilterLanguage types.String `tfsdk:"filter_language"` + StartRange types.Int64 `tfsdk:"start_range"` + EndRange types.Int64 `tfsdk:"end_range"` + DatasetBatchSize types.Int64 `tfsdk:"dataset_batch_size"` + ElasticBatchEnabled types.Bool `tfsdk:"elastic_batch_enabled"` + FixBlockReorgs types.Bool `tfsdk:"fix_block_reorgs"` + KeepDistanceFromTip types.Int64 `tfsdk:"keep_distance_from_tip"` + RestreamBatchOnReorg types.Bool `tfsdk:"restream_batch_on_reorg"` + NotificationEmail types.String `tfsdk:"notification_email"` + Destination types.Object `tfsdk:"destination"` + ExtraDestinations types.List `tfsdk:"extra_destinations"` +} + +func NewStreamResource() resource.Resource { + return &streamResource{} +} + +func (r *streamResource) Metadata(_ context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_stream" +} + +func (r *streamResource) Schema(_ context.Context, _ resource.SchemaRequest, resp *resource.SchemaResponse) { + replace := []planmodifier.String{stringplanmodifier.RequiresReplace()} + resp.Schema = schema.Schema{ + MarkdownDescription: "A Quicknode Stream. It delivers blockchain data from a network to one or more destinations.\n\n" + + "Quicknode can stop a stream by itself: `terminated` when a destination keeps failing or the account's plan no longer allows it, " + + "and `completed` after it delivers `end_range`. `state` reports which. For a terminated stream the plan changes `status` back to the configured value, " + + "and applying it resumes or pauses the stream. A completed stream can't be changed or resumed, so any change replaces it.", + Attributes: map[string]schema.Attribute{ + "id": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "Stream id.", + PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, + }, + "name": requiredString("Name shown in the dashboard. Names are not unique."), + "network": schema.StringAttribute{ + Required: true, + MarkdownDescription: "Network slug, for example `ethereum-mainnet` or `solana-devnet`. Changing it replaces the stream.", + PlanModifiers: replace, + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + }, + "dataset": schema.StringAttribute{ + Required: true, + MarkdownDescription: "What each batch carries, for example `block`, `block_with_receipts`, `receipts`, `logs`, `transactions` or `trace_blocks`. The datasets available depend on the network. Changing it replaces the stream.", + PlanModifiers: replace, + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + }, + "region": schema.StringAttribute{ + Required: true, + MarkdownDescription: "Region the stream runs in: `usa_east`, `europe_central` or `asia_east`. Changing it replaces the stream.", + PlanModifiers: replace, + Validators: []validator.String{stringvalidator.OneOf("usa_east", "europe_central", "asia_east")}, + }, + "status": schema.StringAttribute{ + Required: true, + MarkdownDescription: "`active` or `paused`. Required, so importing a stream never starts or stops it.", + Validators: []validator.String{stringvalidator.OneOf(client.StreamStatusActive, client.StreamStatusPaused)}, + }, + "resume_from": schema.StringAttribute{ + Optional: true, + Computed: true, + Default: stringdefault.StaticString(resumeFromLast), + MarkdownDescription: "Where a paused or terminated stream continues when `status` changes to `active`: `last` continues after the last block delivered, " + + "`latest` skips ahead to the newest block. `latest` moves `start_range`, so it cannot be combined with a configured `start_range`. Defaults to `last`.", + Validators: []validator.String{stringvalidator.OneOf(resumeFromLast, resumeFromLatest)}, + }, + "state": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "The stream's actual status: `active`, `paused`, `terminated` or `completed`.", + PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, + }, + "filter_function": schema.StringAttribute{ + Optional: true, + MarkdownDescription: "Source code of the filter that shapes each batch, as plain text. Read it from a file with `file(\"filter.js\")`. " + + "Quicknode runs the filter against `start_range` when the stream is created and rejects the stream if it fails.", + Validators: []validator.String{stringvalidator.LengthAtLeast(1)}, + }, + "filter_language": defaultedString("Language of `filter_function`: `javascript` or `go`.", "javascript", "javascript", "go"), + "start_range": schema.Int64Attribute{ + Optional: true, + Computed: true, + MarkdownDescription: "First block to deliver. When omitted, the stream starts at the newest block and this attribute reports which one. " + + "Changing it moves the stream to that block, which can deliver blocks again or skip them.", + PlanModifiers: []planmodifier.Int64{int64planmodifier.UseStateForUnknown()}, + Validators: []validator.Int64{int64validator.AtLeast(0)}, + }, + "end_range": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Last block to deliver. When set, the stream stops as `completed` after delivering it. When omitted, the stream runs until it is paused or deleted.", + Validators: []validator.Int64{int64validator.AtLeast(0)}, + }, + "dataset_batch_size": schema.Int64Attribute{ + Optional: true, + Computed: true, + Default: int64default.StaticInt64(1), + MarkdownDescription: "Blocks per batch. Some fast networks require a larger minimum. Defaults to `1`.", + Validators: []validator.Int64{int64validator.AtLeast(1)}, + }, + "elastic_batch_enabled": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Grow batches while the stream catches up to the newest block. Only allowed without `end_range`. Defaults to `false`.", + }, + "fix_block_reorgs": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Deliver corrected blocks when the network reorganizes. Depends on the account's plan. Defaults to `false`.", + }, + "keep_distance_from_tip": schema.Int64Attribute{ + Optional: true, + Computed: true, + MarkdownDescription: "Stay this many blocks behind the newest block, so reorganized blocks are never delivered. Depends on the account's plan.", + PlanModifiers: []planmodifier.Int64{int64planmodifier.UseStateForUnknown()}, + Validators: []validator.Int64{int64validator.AtLeast(0)}, + }, + "restream_batch_on_reorg": schema.BoolAttribute{ + Optional: true, + Computed: true, + MarkdownDescription: "Deliver the whole batch again when a block in it is reorganized.", + PlanModifiers: []planmodifier.Bool{boolplanmodifier.UseStateForUnknown()}, + }, + "notification_email": optionalString("Address notified when the stream is terminated."), + "destination": schema.SingleNestedAttribute{ + Required: true, + MarkdownDescription: "Where the stream delivers. Set exactly one type. Changing the type replaces the stream.", + Attributes: streamDestinationAttributes(), + }, + "extra_destinations": schema.ListNestedAttribute{ + Optional: true, + MarkdownDescription: "More destinations that receive every batch, each set like `destination`. Types can repeat. " + + "The stream moves on only after every destination accepts a batch, and stops if one keeps failing. " + + "The limit depends on the account's plan.", + NestedObject: schema.NestedAttributeObject{Attributes: streamDestinationAttributes()}, + }, + }, + } +} + +func (r *streamResource) Configure(_ context.Context, req resource.ConfigureRequest, resp *resource.ConfigureResponse) { + if req.ProviderData == nil { + return + } + data, ok := req.ProviderData.(providerData) + if !ok { + resp.Diagnostics.AddError("Unexpected provider data", fmt.Sprintf("The stream resource expected providerData, got %T.", req.ProviderData)) + return + } + r.client = data.Client +} + +func (r *streamResource) ValidateConfig(ctx context.Context, req resource.ValidateConfigRequest, resp *resource.ValidateConfigResponse) { + var config streamResourceModel + resp.Diagnostics.Append(req.Config.Get(ctx, &config)...) + if resp.Diagnostics.HasError() { + return + } + + if config.ElasticBatchEnabled.ValueBool() && !config.EndRange.IsNull() { + resp.Diagnostics.AddAttributeError(path.Root("elastic_batch_enabled"), + "Elastic batching needs an open-ended stream", + "Quicknode turns elastic batching off for a stream with an end_range. Remove end_range or set elastic_batch_enabled to false.") + } + + checkOneType := func(object types.Object, at path.Path) { + if object.IsNull() || object.IsUnknown() { + return + } + count, known, diags := destinationTypeCount(ctx, object) + resp.Diagnostics.Append(diags...) + if known && count != 1 { + resp.Diagnostics.AddAttributeError(at, "Set exactly one destination type", + fmt.Sprintf("A destination takes one of webhook, s3, azure, postgres or kafka; %d are set.", count)) + } + } + checkOneType(config.Destination, path.Root("destination")) + + if config.ExtraDestinations.IsNull() || config.ExtraDestinations.IsUnknown() { + return + } + var extras []types.Object + resp.Diagnostics.Append(config.ExtraDestinations.ElementsAs(ctx, &extras, false)...) + for index, extra := range extras { + checkOneType(extra, path.Root("extra_destinations").AtListIndex(index)) + } +} + +// ModifyPlan covers what the API decides at run time: resuming from the newest +// block moves start_range, a status change can land on a different state, and +// a completed stream accepts no changes at all. +func (r *streamResource) ModifyPlan(ctx context.Context, req resource.ModifyPlanRequest, resp *resource.ModifyPlanResponse) { + if req.Plan.Raw.IsNull() || req.State.Raw.IsNull() { + return + } + + var plan, state, config streamResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + resp.Diagnostics.Append(req.Config.Get(ctx, &config)...) + if resp.Diagnostics.HasError() { + return + } + + if state.State.ValueString() == client.StreamStatusCompleted { + resp.RequiresReplace = append(resp.RequiresReplace, changedStreamAttributes(plan, state)...) + return + } + + if destinationTypeName(ctx, plan.Destination) != destinationTypeName(ctx, state.Destination) { + resp.RequiresReplace = append(resp.RequiresReplace, path.Root("destination")) + } + + if !plan.Status.Equal(state.Status) { + plan.State = types.StringUnknown() + } + + if resuming(plan, state) && plan.ResumeFrom.ValueString() == resumeFromLatest { + if !config.StartRange.IsNull() { + resp.Diagnostics.AddAttributeError(path.Root("resume_from"), + "resume_from = \"latest\" conflicts with start_range", + "Resuming from the newest block moves start_range, so the configured value would be applied again on the next plan. Remove start_range or use resume_from = \"last\".") + return + } + plan.StartRange = types.Int64Unknown() + } + + resp.Diagnostics.Append(resp.Plan.Set(ctx, &plan)...) +} + +func (r *streamResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { + var plan streamResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + if resp.Diagnostics.HasError() { + return + } + + destination, diags := destinationFromObject(ctx, plan.Destination) + resp.Diagnostics.Append(diags...) + extras, diags := destinationsFromList(ctx, plan.ExtraDestinations) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + input := client.StreamInput{ + Name: plan.Name.ValueString(), + Network: plan.Network.ValueString(), + Dataset: plan.Dataset.ValueString(), + Region: plan.Region.ValueString(), + Status: plan.Status.ValueString(), + FilterFunction: plan.FilterFunction.ValueString(), + FilterLanguage: plan.FilterLanguage.ValueString(), + StartRange: knownInt64(plan.StartRange), + EndRange: knownInt64(plan.EndRange), + DatasetBatchSize: plan.DatasetBatchSize.ValueInt64(), + ElasticBatchEnabled: plan.ElasticBatchEnabled.ValueBool(), + RestreamBatchOnReorg: knownBool(plan.RestreamBatchOnReorg), + FixBlockReorgs: reorgFlag(plan.FixBlockReorgs), + KeepDistanceFromTip: knownInt64(plan.KeepDistanceFromTip), + NotificationEmail: knownStringPointer(plan.NotificationEmail), + Destination: destination, + ExtraDestinations: extras, + } + + created, err := r.client.CreateStream(ctx, input) + if err != nil { + resp.Diagnostics.AddError("Could not create the Quicknode stream", err.Error()) + return + } + + planned := plan.Status + resp.Diagnostics.Append(applyStream(created, &plan)...) + plan.Status = planned + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +func (r *streamResource) Read(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) { + var state streamResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + stream, err := r.client.GetStream(ctx, state.ID.ValueString()) + if client.IsNotFound(err) { + resp.State.RemoveResource(ctx) + return + } + if err != nil { + resp.Diagnostics.AddError("Could not read the Quicknode stream", err.Error()) + return + } + + previous := state.Status + resp.Diagnostics.Append(applyStream(stream, &state)...) + if stream.Status == client.StreamStatusCompleted { + state.Status = previous + if state.Status.IsNull() { + state.Status = types.StringValue(client.StreamStatusActive) + } + } + resp.Diagnostics.Append(resp.State.Set(ctx, &state)...) +} + +func (r *streamResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) { + var plan, state streamResourceModel + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + id := state.ID.ValueString() + update, changed, diags := streamUpdate(ctx, plan, state) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } + + if changed { + if _, err := r.client.UpdateStream(ctx, id, update); err != nil { + resp.Diagnostics.AddError("Could not update the Quicknode stream", err.Error()) + return + } + } + + switch { + case resuming(plan, state): + if err := r.client.ActivateStream(ctx, id); err != nil { + resp.Diagnostics.AddError("Could not activate the Quicknode stream", err.Error()) + return + } + case plan.Status.ValueString() == client.StreamStatusPaused && state.Status.ValueString() != client.StreamStatusPaused: + if err := r.client.PauseStream(ctx, id); err != nil { + resp.Diagnostics.AddError("Could not pause the Quicknode stream", err.Error()) + return + } + } + + stream, err := r.client.GetStream(ctx, id) + if err != nil { + resp.Diagnostics.AddError("Could not read the Quicknode stream back", err.Error()) + return + } + planned := plan.Status + resp.Diagnostics.Append(applyStream(stream, &plan)...) + plan.Status = planned + resp.Diagnostics.Append(resp.State.Set(ctx, &plan)...) +} + +func (r *streamResource) Delete(ctx context.Context, req resource.DeleteRequest, resp *resource.DeleteResponse) { + var state streamResourceModel + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + if resp.Diagnostics.HasError() { + return + } + + err := r.client.DeleteStream(ctx, state.ID.ValueString()) + if err != nil && !client.IsNotFound(err) { + resp.Diagnostics.AddError("Could not delete the Quicknode stream", err.Error()) + } +} + +func (r *streamResource) ImportState(ctx context.Context, req resource.ImportStateRequest, resp *resource.ImportStateResponse) { + resource.ImportStatePassthroughID(ctx, path.Root("id"), req, resp) + resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("resume_from"), resumeFromLast)...) +} + +func resuming(plan, state streamResourceModel) bool { + return plan.Status.ValueString() == client.StreamStatusActive && state.Status.ValueString() != client.StreamStatusActive +} + +// streamUpdate builds a PATCH carrying only what the plan changes. Status is +// left out: Update moves it through the pause and activate calls instead. +func streamUpdate(ctx context.Context, plan, state streamResourceModel) (client.StreamUpdate, bool, diag.Diagnostics) { + var diags diag.Diagnostics + var update client.StreamUpdate + changed := false + + if !plan.Name.Equal(state.Name) { + update.Name = plan.Name.ValueStringPointer() + changed = true + } + if !plan.FilterFunction.Equal(state.FilterFunction) { + source := plan.FilterFunction.ValueString() + update.FilterFunction = &source + changed = true + } + if !plan.FilterLanguage.Equal(state.FilterLanguage) { + update.FilterLanguage = plan.FilterLanguage.ValueStringPointer() + changed = true + } + + switch { + case plan.StartRange.IsUnknown() && resuming(plan, state): + latest := client.StreamRangeUnset + update.StartRange = &latest + changed = true + case !plan.StartRange.IsUnknown() && !plan.StartRange.Equal(state.StartRange): + update.StartRange = plan.StartRange.ValueInt64Pointer() + changed = true + } + + if !plan.EndRange.Equal(state.EndRange) { + end := client.StreamRangeUnset + if !plan.EndRange.IsNull() { + end = plan.EndRange.ValueInt64() + } + update.EndRange = &end + changed = true + } + if !plan.DatasetBatchSize.Equal(state.DatasetBatchSize) { + update.DatasetBatchSize = plan.DatasetBatchSize.ValueInt64Pointer() + changed = true + } + if !plan.ElasticBatchEnabled.Equal(state.ElasticBatchEnabled) { + update.ElasticBatchEnabled = plan.ElasticBatchEnabled.ValueBoolPointer() + changed = true + } + if !plan.FixBlockReorgs.Equal(state.FixBlockReorgs) { + update.FixBlockReorgs = reorgFlag(plan.FixBlockReorgs) + changed = true + } + if !plan.KeepDistanceFromTip.IsUnknown() && !plan.KeepDistanceFromTip.Equal(state.KeepDistanceFromTip) { + update.KeepDistanceFromTip = plan.KeepDistanceFromTip.ValueInt64Pointer() + changed = true + } + if !plan.RestreamBatchOnReorg.IsUnknown() && !plan.RestreamBatchOnReorg.Equal(state.RestreamBatchOnReorg) { + update.RestreamBatchOnReorg = plan.RestreamBatchOnReorg.ValueBoolPointer() + changed = true + } + if !plan.NotificationEmail.Equal(state.NotificationEmail) { + email := plan.NotificationEmail.ValueString() + update.NotificationEmail = &email + changed = true + } + + if !plan.Destination.Equal(state.Destination) { + destination, destinationDiags := destinationFromObject(ctx, plan.Destination) + diags.Append(destinationDiags...) + update.Destination = &destination + changed = true + } + if extraDestinationsChanged(plan, state) { + extras, extraDiags := destinationsFromList(ctx, plan.ExtraDestinations) + diags.Append(extraDiags...) + update.ExtraDestinations = &extras + changed = true + } + + return update, changed, diags +} + +// changedStreamAttributes lists what a plan changes on a completed stream, all +// of which have to be applied by replacing it. +func changedStreamAttributes(plan, state streamResourceModel) []path.Path { + candidates := []struct { + name string + changed bool + }{ + {"name", !plan.Name.Equal(state.Name)}, + {"status", !plan.Status.Equal(state.Status)}, + {"filter_function", !plan.FilterFunction.Equal(state.FilterFunction)}, + {"filter_language", !plan.FilterLanguage.Equal(state.FilterLanguage)}, + {"start_range", !plan.StartRange.Equal(state.StartRange)}, + {"end_range", !plan.EndRange.Equal(state.EndRange)}, + {"dataset_batch_size", !plan.DatasetBatchSize.Equal(state.DatasetBatchSize)}, + {"elastic_batch_enabled", !plan.ElasticBatchEnabled.Equal(state.ElasticBatchEnabled)}, + {"fix_block_reorgs", !plan.FixBlockReorgs.Equal(state.FixBlockReorgs)}, + {"keep_distance_from_tip", !plan.KeepDistanceFromTip.Equal(state.KeepDistanceFromTip)}, + {"restream_batch_on_reorg", !plan.RestreamBatchOnReorg.Equal(state.RestreamBatchOnReorg)}, + {"notification_email", !plan.NotificationEmail.Equal(state.NotificationEmail)}, + {"destination", !plan.Destination.Equal(state.Destination)}, + {"extra_destinations", extraDestinationsChanged(plan, state)}, + } + var paths []path.Path + for _, candidate := range candidates { + if candidate.changed { + paths = append(paths, path.Root(candidate.name)) + } + } + return paths +} + +func destinationTypeName(ctx context.Context, object types.Object) string { + if object.IsNull() || object.IsUnknown() { + return "" + } + var model streamDestinationModel + if diags := object.As(ctx, &model, objectAsOptions); diags.HasError() { + return "" + } + names := map[string]types.Object{ + client.DestinationWebhook: model.Webhook, + client.DestinationS3: model.S3, + client.DestinationAzure: model.Azure, + client.DestinationPostgres: model.Postgres, + client.DestinationKafka: model.Kafka, + } + for name, child := range names { + if !child.IsNull() { + return name + } + } + return "" +} + +// applyStream copies what the API reports into the model. The caller decides +// what status means: Create and Update keep the planned status, Read keeps +// the previous one for a completed stream. +func applyStream(stream *client.Stream, model *streamResourceModel) diag.Diagnostics { + var diags diag.Diagnostics + + model.ID = types.StringValue(stream.ID) + model.Name = types.StringValue(stream.Name) + model.Network = types.StringValue(stream.Network) + model.Dataset = types.StringValue(stream.Dataset) + model.Region = types.StringValue(stream.Region) + model.Status = types.StringValue(stream.Status) + model.State = types.StringValue(stream.Status) + model.FilterFunction = stringOrNull(stream.FilterFunction) + model.FilterLanguage = types.StringValue(stream.FilterLanguage) + model.StartRange = types.Int64Value(stream.StartRange) + model.EndRange = rangeOrNull(stream.EndRange) + model.DatasetBatchSize = types.Int64Value(stream.DatasetBatchSize) + model.ElasticBatchEnabled = types.BoolValue(stream.ElasticBatchEnabled) + model.FixBlockReorgs = types.BoolValue(stream.FixBlockReorgs != 0) + model.KeepDistanceFromTip = types.Int64Value(stream.KeepDistanceFromTip) + model.RestreamBatchOnReorg = types.BoolValue(stream.RestreamBatchOnReorg) + model.NotificationEmail = stringOrNull(stream.NotificationEmail) + + destination, destinationDiags := destinationObject(stream.Destination) + diags.Append(destinationDiags...) + model.Destination = destination + + extras, extraDiags := destinationsList(stream.ExtraDestinations) + diags.Append(extraDiags...) + if len(stream.ExtraDestinations) > 0 || model.ExtraDestinations.IsNull() || model.ExtraDestinations.IsUnknown() { + model.ExtraDestinations = extras + } else { + model.ExtraDestinations = types.ListValueMust(streamDestinationObjectType, []attr.Value{}) + } + + if model.ResumeFrom.IsNull() || model.ResumeFrom.IsUnknown() { + model.ResumeFrom = types.StringValue(resumeFromLast) + } + return diags +} + +// extraDestinationsChanged treats an empty list and a missing one as the same, +// so switching between them never updates or replaces the stream. +func extraDestinationsChanged(plan, state streamResourceModel) bool { + if len(plan.ExtraDestinations.Elements()) == 0 && len(state.ExtraDestinations.Elements()) == 0 && + !plan.ExtraDestinations.IsUnknown() && !state.ExtraDestinations.IsUnknown() { + return false + } + return !plan.ExtraDestinations.Equal(state.ExtraDestinations) +} + +func rangeOrNull(value int64) types.Int64 { + if value == client.StreamRangeUnset { + return types.Int64Null() + } + return types.Int64Value(value) +} + +func knownInt64(value types.Int64) *int64 { + if value.IsNull() || value.IsUnknown() { + return nil + } + return value.ValueInt64Pointer() +} + +func knownBool(value types.Bool) *bool { + if value.IsNull() || value.IsUnknown() { + return nil + } + return value.ValueBoolPointer() +} + +func knownStringPointer(value types.String) *string { + if value.IsNull() || value.IsUnknown() { + return nil + } + return value.ValueStringPointer() +} + +// reorgFlag converts fix_block_reorgs to the 0 or 1 the API takes. +func reorgFlag(value types.Bool) *int64 { + if value.IsNull() || value.IsUnknown() { + return nil + } + flag := int64(0) + if value.ValueBool() { + flag = 1 + } + return &flag +} diff --git a/internal/provider/user_agent.go b/internal/provider/user_agent.go new file mode 100644 index 0000000..3f279b2 --- /dev/null +++ b/internal/provider/user_agent.go @@ -0,0 +1,33 @@ +package provider + +import ( + "fmt" + "runtime" +) + +var userAgentOS = map[string]string{ + "darwin": "macos", + "linux": "linux", + "windows": "windows", +} + +var userAgentArch = map[string]string{ + "amd64": "x86_64", + "arm64": "aarch64", + "386": "x86", +} + +// userAgent identifies the provider to Quicknode, in the same +// `client/version (os-arch; runtime-version)` form the other Quicknode tools +// send. The platform uses Rust's OS and architecture names to match them. +func userAgent(providerVersion, terraformVersion string) string { + return fmt.Sprintf("quicknode-terraform/%s (%s-%s; terraform-%s)", + providerVersion, platformName(userAgentOS, runtime.GOOS), platformName(userAgentArch, runtime.GOARCH), terraformVersion) +} + +func platformName(names map[string]string, goName string) string { + if name, ok := names[goName]; ok { + return name + } + return goName +} diff --git a/internal/provider/user_agent_test.go b/internal/provider/user_agent_test.go new file mode 100644 index 0000000..605b960 --- /dev/null +++ b/internal/provider/user_agent_test.go @@ -0,0 +1,20 @@ +package provider + +import ( + "regexp" + "testing" +) + +func TestUserAgentFollowsTheQuicknodeClientFormat(t *testing.T) { + got := userAgent("0.3.0", "1.9.8") + want := regexp.MustCompile(`^quicknode-terraform/0\.3\.0 \((linux|macos|windows)-(x86_64|aarch64|x86); terraform-1\.9\.8\)$`) + if !want.MatchString(got) { + t.Fatalf("userAgent = %q, want a match for %s", got, want) + } +} + +func TestPlatformNameKeepsUnmappedNames(t *testing.T) { + if got := platformName(userAgentArch, "riscv64"); got != "riscv64" { + t.Fatalf("platformName = %q, want riscv64", got) + } +}