sql: rewrite scid string literals to use scid() for index support

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [Unreleased]
+
+### Changed
+
+
+ - plugins: the `sql` plugin now stores short_channel_id as integers; use `scid('1x2x3')` in queries instead of string literals (breaking change for string comparisons) ([#9430])
+
+[#9430]: https://github.com/ElementsProject/lightning/pull/9430
+
 ## [26.06.6] - 2026-07-20: "Quantum-Resistant Lightning Channel III"
 
 v26.06.3, v26.06.4, and v26.06.5 had issues during publishing with the pypi releases and were deleted.

connectd/multiplex.c

@@ -1491,6 +1491,35 @@ static struct io_plan *read_body_from_peer_done(struct io_conn *peer_conn,
 	       return next_read(peer_conn, peer);
        }
 
+       /* BOLT #1:
+	*
+	* The receiving node:
+	*   - upon receiving `error`:
+	*     - if `channel_id` is all zero:
+	*       - MUST fail all channels with the sending node.
+	*     - otherwise:
+	*       - MUST fail the channel referred to by `channel_id`, if that channel is with the
+	*         sending node.
+	*/
+       /* channeld abort()s if it ever sees WIRE_ERROR ("swallowed by
+	* connectd").  Tell lightningd and do not enqueue, whether or not a
+	* subd exists: a dying channeld never reads the queue, so the channel
+	* would otherwise stay up.
+	*
+	* Log it here.  peer_read() in the subd is what normally emits
+	* "peer_in WIRE_ERROR", and tests wait on that line.
+	*/
+       if (type == WIRE_ERROR) {
+	       status_peer_io(LOG_IO_IN, &peer->id, decrypted);
+	       daemon_conn_send(peer->daemon->master,
+				take(towire_connectd_peer_spoke(NULL, &peer->id,
+								peer->counter,
+								type,
+								&channel_id,
+								is_peer_error(tmpctx, decrypted))));
+	       return next_read(peer_conn, peer);
+       }
+
        /* If we don't find a subdaemon for this, create a new one. */
        subd = find_subd(peer, &channel_id);
        if (!subd) {

contrib/msggen/msggen/schema.json

@@ -29747,7 +29747,8 @@
                           "INTEGER",
                           "BLOB",
                           "TEXT",
-                          "REAL"
+                          "REAL",
+                          "SCID"
                         ],
                         "description": [
                           "The SQL type of the column."
@@ -29875,15 +29876,15 @@
                   },
                   {
                     "name": "short_channel_id",
-                    "type": "TEXT"
+                    "type": "SCID"
                   },
                   {
                     "name": "alias_local",
-                    "type": "TEXT"
+                    "type": "SCID"
                   },
                   {
                     "name": "alias_remote",
-                    "type": "TEXT"
+                    "type": "SCID"
                   },
                   {
                     "name": "opener",
@@ -37959,9 +37960,11 @@
         "  * JSON: string",
         "  * sqlite3: TEXT",
         "",
-        "* *short_channel_id*. A short-channel-id of form 1x2x3.",
+        "* *short_channel_id*. A short-channel-id of form 1x2x3.  Stored as an integer internally for efficient indexing.",
         "  * JSON: string",
-        "  * sqlite3: TEXT"
+        "  * sqlite3: SCID (INTEGER affinity)",
+        "",
+        "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`.  The `fmt_scid()` function converts back to string form."
       ],
       "permitted_sqlite3_functions": [
         "Writing to the database is not permitted, and limits are placed on various other query parameters.",
@@ -37990,7 +37993,9 @@
         "* total",
         "* unixepoch",
         "* json_object",
-        "* json_group_array"
+        "* json_group_array",
+        "* scid",
+        "* fmt_scid"
       ],
       "tables": [
         "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh.  This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).",

doc/schemas/listsqlschemas.json

@@ -68,7 +68,8 @@
                       "INTEGER",
                       "BLOB",
                       "TEXT",
-                      "REAL"
+                      "REAL",
+                      "SCID"
                     ],
                     "description": [
                       "The SQL type of the column."
@@ -196,15 +197,15 @@
               },
               {
                 "name": "short_channel_id",
-                "type": "TEXT"
+                "type": "SCID"
               },
               {
                 "name": "alias_local",
-                "type": "TEXT"
+                "type": "SCID"
               },
               {
                 "name": "alias_remote",
-                "type": "TEXT"
+                "type": "SCID"
               },
               {
                 "name": "opener",

doc/schemas/sql-template.json

@@ -78,9 +78,11 @@
     "  * JSON: string",
     "  * sqlite3: TEXT",
     "",
-    "* *short_channel_id*. A short-channel-id of form 1x2x3.",
+    "* *short_channel_id*. A short-channel-id of form 1x2x3.  Stored as an integer internally for efficient indexing.",
     "  * JSON: string",
-    "  * sqlite3: TEXT"
+    "  * sqlite3: SCID (INTEGER affinity)",
+    "",
+    "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`.  The `fmt_scid()` function converts back to string form."
   ],
   "permitted_sqlite3_functions": [
     "Writing to the database is not permitted, and limits are placed on various other query parameters.",
@@ -109,7 +111,9 @@
     "* total",
     "* unixepoch",
     "* json_object",
-    "* json_group_array"
+    "* json_group_array",
+    "* scid",
+    "* fmt_scid"
   ],
   "tables": [
     "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh.  This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).",

plugins/sql.c

@@ -70,7 +70,7 @@ static const struct fieldtypemap fieldtypemap[] = {
 	{ "boolean", "INTEGER" }, /* FIELD_BOOL */
 	{ "number", "REAL" }, /* FIELD_NUMBER */
 	{ "string", "TEXT" }, /* FIELD_STRING */
-	{ "short_channel_id", "TEXT" }, /* FIELD_SCID */
+	{ "short_channel_id", "SCID" }, /* FIELD_SCID */
 	{ "outpoint", "TEXT" }, /* FIELD_OUTPOINT */
 };
 
@@ -245,6 +245,52 @@ static enum fieldtype find_fieldtype(const jsmntok_t *name)
 	     name->end - name->start, schemas + name->start);
 }
 
+/* SQLite custom function: scid('NNNxNNNxNNN') -> u64 integer.
+ * Allows efficient queries like: WHERE in_channel = scid('735095x480x1') */
+static void sql_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv)
+{
+	struct short_channel_id scid;
+	const char *str;
+
+	if (argc != 1) {
+		sqlite3_result_error(ctx, "scid() requires exactly one argument", -1);
+		return;
+	}
+	if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+		sqlite3_result_null(ctx);
+		return;
+	}
+
+	str = (const char *)sqlite3_value_text(argv[0]);
+	if (!str || !short_channel_id_from_str(str, strlen(str), &scid)) {
+		sqlite3_result_error(ctx, "invalid short_channel_id format, expected NNNxNNNxNNN", -1);
+		return;
+	}
+
+	sqlite3_result_int64(ctx, scid.u64);
+}
+
+/* SQLite custom function: fmt_scid(u64) -> 'NNNxNNNxNNN' string.
+ * Useful for displaying integer SCIDs in text format within SQL expressions. */
+static void sql_fmt_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv)
+{
+	struct short_channel_id scid;
+	char *str;
+
+	if (argc != 1) {
+		sqlite3_result_error(ctx, "fmt_scid() requires exactly one argument", -1);
+		return;
+	}
+	if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+		sqlite3_result_null(ctx);
+		return;
+	}
+
+	scid.u64 = sqlite3_value_int64(argv[0]);
+	str = fmt_short_channel_id(tmpctx, scid);
+	sqlite3_result_text(ctx, str, -1, SQLITE_TRANSIENT);
+}
+
 static struct sqlite3 *sqlite_setup(struct plugin *plugin)
 {
 	int err;
@@ -301,6 +347,12 @@ static struct sqlite3 *sqlite_setup(struct plugin *plugin)
 			plugin_err(plugin, "Could not disable sync: %s", errmsg);
 	}
 
+	/* Register custom SCID functions for integer<->text conversion */
+	sqlite3_create_function(db, "scid", 1, SQLITE_UTF8, NULL,
+				sql_scid_func, NULL, NULL);
+	sqlite3_create_function(db, "fmt_scid", 1, SQLITE_UTF8, NULL,
+				sql_fmt_scid_func, NULL, NULL);
+
 	return db;
 }
 
@@ -424,6 +476,10 @@ static int sqlite_authorize(void *dbq_, int code,
 			return SQLITE_OK;
 		if (streq(b, "json_group_array"))
 			return SQLITE_OK;
+		if (streq(b, "scid"))
+			return SQLITE_OK;
+		if (streq(b, "fmt_scid"))
+			return SQLITE_OK;
 	}
 
 	/* See https://www.sqlite.org/c3ref/c_alter_table.html to decode these! */
@@ -447,6 +503,13 @@ static struct command_result *refresh_complete(struct command *cmd,
 
 	num_cols = sqlite3_column_count(dbq->stmt);
 
+	/* decltype is fixed for the statement, not per row. */
+	bool *is_scid = tal_arr(tmpctx, bool, num_cols);
+	for (int i = 0; i < num_cols; i++) {
+		const char *decltype = sqlite3_column_decltype(dbq->stmt, i);
+		is_scid[i] = decltype && streq(decltype, "SCID");
+	}
+
 	/* We normally hit an error immediately, so return a simple error then */
 	ret = NULL;
 	num_rows = 0;
@@ -465,7 +528,14 @@ static struct command_result *refresh_complete(struct command *cmd,
 			switch (sqlite3_column_type(dbq->stmt, i)) {
 			case SQLITE_INTEGER: {
 				s64 v = sqlite3_column_int64(dbq->stmt, i);
-				json_add_s64(ret, NULL, v);
+				if (is_scid[i]) {
+					struct short_channel_id scid;
+					scid.u64 = (u64)v;
+					json_add_string(ret, NULL,
+							fmt_short_channel_id(tmpctx, scid));
+				} else {
+					json_add_s64(ret, NULL, v);
+				}
 				break;
 			}
 			case SQLITE_FLOAT: {
@@ -790,7 +860,18 @@ static struct command_result *process_json_obj(struct command *cmd,
 				}
 				sqlite3_bind_int64(stmt, (*sqloff)++, valmsat.millisatoshis /* Raw: db */);
 				break;
-			case FIELD_SCID:
+			case FIELD_SCID: {
+				struct short_channel_id scid;
+				if (!json_to_short_channel_id(buf, coltok, &scid)) {
+					return command_fail(cmd, LIGHTNINGD,
+							    "column %zu row %zu not a valid short_channel_id: %.*s",
+							    i, row,
+							    json_tok_full_len(coltok),
+							    json_tok_full(buf, coltok));
+				}
+				sqlite3_bind_int64(stmt, (*sqloff)++, scid.u64);
+				break;
+			}
 			case FIELD_STRING:
 			case FIELD_OUTPOINT:
 				sqlite3_bind_text(stmt, (*sqloff)++, buf + coltok->start,
@@ -1029,8 +1110,8 @@ static void delete_channel_from_db(struct command *cmd,
 	err = sqlite3_exec(sql->db,
 			   tal_fmt(tmpctx,
 				   "DELETE FROM channels"
-				   " WHERE short_channel_id = '%s'",
-				   fmt_short_channel_id(tmpctx, scid)),
+				   " WHERE short_channel_id = %"PRIu64,
+				   scid.u64),
 			   NULL, NULL, &errmsg);
 	if (err != SQLITE_OK)
 		plugin_err(cmd->plugin, "Could not delete from channels: %s",
@@ -1404,6 +1485,72 @@ static struct command_result *refresh_tables(struct command *cmd,
 	return td->refresh(cmd, dbq->tables[0], dbq);
 }
 
+/* Check if a string is a valid short_channel_id (NNNxNNNxNNN format) */
+static bool looks_like_scid(const char *str, size_t len)
+{
+	struct short_channel_id scid;
+
+	return short_channel_id_from_str(str, len, &scid);
+}
+
+/* Copy [start, end) onto result in one append. */
+static void append_span(char **result, const char *start, const char *end)
+{
+	if (end > start)
+		tal_append_fmt(result, "%.*s", (int)(end - start), start);
+}
+
+/* Rewrite SQL query to wrap scid string literals with scid() function.
+ * This transforms '735095x480x1' into scid('735095x480x1') so that
+ * SQLite can use indexes on integer SCID columns. */
+static const char *rewrite_scid_literals(const tal_t *ctx, const char *query)
+{
+	char *result = tal_strdup(ctx, "");
+	const char *p = query;
+	const char *span = query;
+
+	while (*p) {
+		const char *start, *end;
+
+		/* Copy the non-quoted span in one append, not a byte at a time. */
+		if (*p != '\'') {
+			p++;
+			continue;
+		}
+		append_span(&result, span, p);
+		span = p;
+
+		start = p + 1;
+		end = strchr(start, '\'');
+		if (!end) {
+			/* Unterminated quote, copy the rest and stop. */
+			tal_append_fmt(&result, "%s", p);
+			return result;
+		}
+
+		if (looks_like_scid(start, end - start)) {
+			/* Already wrapped: scid('735095x480x1'). Leave it. */
+			bool already_wrapped = false;
+			if (p - query >= 5 && strncmp(p - 5, "scid(", 5) == 0)
+				already_wrapped = true;
+			if (!already_wrapped) {
+				tal_append_fmt(&result, "scid('%.*s')",
+					       (int)(end - start), start);
+				p = end + 1;
+				span = p;
+				continue;
+			}
+		}
+		/* Not a scid, or already wrapped: keep the quoted literal. */
+		p = end + 1;
+		append_span(&result, span, p);
+		span = p;
+	}
+	append_span(&result, span, p);
+
+	return result;
+}
+
 static struct command_result *json_sql(struct command *cmd,
 				       const char *buffer,
 				       const jsmntok_t *params)
@@ -1418,6 +1565,10 @@ static struct command_result *json_sql(struct command *cmd,
 		   NULL))
 		return command_param_failed();
 
+	/* Rewrite scid string literals to use scid() function so
+	 * SQLite can use indexes on integer SCID columns. */
+	query = rewrite_scid_literals(tmpctx, query);
+
 	dbq->tables = tal_arr(dbq, struct table_desc *, 0);
 	dbq->authfail = NULL;
 	dbq->cmd = cmd;