Use binary COPY for flex and middle tables

src/db-copy.cpp

@@ -220,6 +220,10 @@ void db_copy_thread_t::thread_t::start_copy(
                        target->rows());
     }
 
+    if (target->binary()) {
+        fmt::format_to(std::back_inserter(sql), " (FORMAT binary)");
+    }
+
     if (!target->conditions().empty()) {
         fmt::format_to(std::back_inserter(sql), FMT_STRING(" WHERE {}"),
                        target->conditions());
@@ -228,12 +232,24 @@ void db_copy_thread_t::thread_t::start_copy(
     sql.push_back('\0');
     m_db_connection.copy_start(to_string(sql));
 
+    if (target->binary()) {
+        // Signature, flags (none), length of header extension (none)
+        static constexpr std::string_view HEADER{
+            "PGCOPY\n\xff\r\n\0\0\0\0\0\0\0\0\0", 19};
+        m_db_connection.copy_send(HEADER, target->name());
+    }
+
     m_inflight = target;
 }
 
 void db_copy_thread_t::thread_t::finish_copy()
 {
     if (m_inflight) {
+        if (m_inflight->binary()) {
+            // File trailer: a row with field count -1
+            static constexpr std::string_view TRAILER{"\xff\xff", 2};
+            m_db_connection.copy_send(TRAILER, m_inflight->name());
+        }
         m_db_connection.copy_end(m_inflight->name());
         m_inflight.reset();
     }

src/db-copy.hpp

@@ -15,8 +15,9 @@
 #include "pgsql-params.hpp"
 
 #include <cassert>
-#include <cstddef>
 #include <condition_variable>
+#include <cstddef>
+#include <cstdint>
 #include <deque>
 #include <future>
 #include <memory>
@@ -27,6 +28,27 @@
 #include <variant>
 #include <vector>
 
+/**
+ * The PostgreSQL type of a column as far as the binary COPY format is
+ * concerned. Every type has its own binary representation which must match
+ * the column type exactly, the server does not convert anything.
+ */
+enum class copy_field_type : uint8_t
+{
+    text, ///< text, char(n), json: the string itself
+    boolean,
+    int2,
+    int4,
+    int8,
+    float4,
+    float8,
+    int8_array, ///< one-dimensional int8[] without NULLs
+    hstore,
+    jsonb,
+    geometry,   ///< PostGIS geometry, sent as EWKB
+    timestamptz ///< from osmium::Timestamp only
+};
+
 /**
  * Table information necessary for building SQL queries.
  */
@@ -55,14 +77,36 @@ class db_target_descr_t
         m_conditions = std::move(conditions);
     }
 
+    /**
+     * Rows are sent in PostgreSQL's binary COPY format if the types of all
+     * columns are known, in the text format otherwise.
+     */
+    bool binary() const noexcept { return !m_binary_types.empty(); }
+
+    /// The types of the columns for the binary COPY format.
+    std::vector<copy_field_type> const &binary_types() const noexcept
+    {
+        return m_binary_types;
+    }
+
+    /**
+     * Set the types of all columns (in the order of rows()) to switch to the
+     * binary COPY format. An empty vector means text format.
+     */
+    void set_binary_types(std::vector<copy_field_type> types)
+    {
+        m_binary_types = std::move(types);
+    }
+
     /**
      * Check if the buffer would use exactly the same copy operation.
      */
     bool same_copy_target(db_target_descr_t const &other) const noexcept
     {
         return (this == &other) ||
                (m_schema == other.m_schema && m_name == other.m_name &&
-                m_id == other.m_id && m_rows == other.m_rows);
+                m_id == other.m_id && m_rows == other.m_rows &&
+                binary() == other.binary());
     }
 
 private:
@@ -76,6 +120,8 @@ class db_target_descr_t
     std::string m_rows;
     /// Conditions for the COPY command.
     std::string m_conditions;
+    /// Column types for binary COPY format (when empty: text format).
+    std::vector<copy_field_type> m_binary_types;
 };
 
 /**

src/flex-table-column.cpp

@@ -9,6 +9,7 @@
 
 #include "flex-table-column.hpp"
 
+#include "db-copy.hpp"
 #include "format.hpp"
 #include "geom-boost-adaptor.hpp"
 #include "overloaded.hpp"
@@ -128,6 +129,54 @@ void flex_table_column_t::set_projection(char const *projection)
     }
 }
 
+std::optional<copy_field_type>
+flex_table_column_t::binary_copy_type() const noexcept
+{
+    if (!m_sql_type.empty()) {
+        return {}; // we don't know the binary format of arbitrary types
+    }
+
+    switch (m_type) {
+    case table_column_type::text:
+    case table_column_type::json:
+    case table_column_type::id_type:
+        return copy_field_type::text;
+    case table_column_type::boolean:
+        return copy_field_type::boolean;
+    case table_column_type::int2:
+    case table_column_type::direction:
+        return copy_field_type::int2;
+    case table_column_type::int4:
+        return copy_field_type::int4;
+    case table_column_type::int8:
+    case table_column_type::id_num:
+        return copy_field_type::int8;
+    case table_column_type::real:
+        return copy_field_type::float4;
+    case table_column_type::double_precision:
+        return copy_field_type::float8;
+    case table_column_type::timestamp:
+    case table_column_type::timestamptz:
+        // These can get strings from Lua in any format PostgreSQL
+        // understands, so we leave the parsing to PostgreSQL.
+        return {};
+    case table_column_type::hstore:
+        return copy_field_type::hstore;
+    case table_column_type::jsonb:
+        return copy_field_type::jsonb;
+    case table_column_type::geometry:
+    case table_column_type::point:
+    case table_column_type::linestring:
+    case table_column_type::polygon:
+    case table_column_type::multipoint:
+    case table_column_type::multilinestring:
+    case table_column_type::multipolygon:
+    case table_column_type::geometrycollection:
+        return copy_field_type::geometry;
+    }
+    return {};
+}
+
 std::string flex_table_column_t::sql_type_name() const
 {
     if (!m_sql_type.empty()) {

src/flex-table-column.hpp

@@ -17,6 +17,7 @@
 
 #include <cassert>
 #include <cstdint>
+#include <optional>
 #include <string>
 #include <unordered_map>
 #include <vector>
@@ -58,6 +59,8 @@ enum class table_column_type : uint8_t
 
 class geometry_cache_t;
 
+enum class copy_field_type : uint8_t;
+
 /**
  * A column in a flex_table_t.
  */
@@ -71,6 +74,14 @@ class flex_table_column_t
 
     table_column_type type() const noexcept { return m_type; }
 
+    /**
+     * The type of this column in the binary COPY format. Nothing if the
+     * column can only be written in the text format, because it has a
+     * user-defined SQL type or because it can contain strings which only
+     * PostgreSQL can parse (timestamps).
+     */
+    std::optional<copy_field_type> binary_copy_type() const noexcept;
+
     bool is_point_column() const noexcept
     {
         return (m_type == table_column_type::point) ||

src/flex-table.cpp

@@ -207,6 +207,22 @@ flex_table_t::build_sql_create_table(table_type ttype,
     return sql;
 }
 
+std::vector<copy_field_type> flex_table_t::binary_copy_types() const
+{
+    std::vector<copy_field_type> types;
+    for (auto const &column : m_columns) {
+        if (column.create_only()) {
+            continue;
+        }
+        auto const type = column.binary_copy_type();
+        if (!type) {
+            return {};
+        }
+        types.push_back(*type);
+    }
+    return types;
+}
+
 std::string flex_table_t::build_sql_column_list() const
 {
     assert(!m_columns.empty());
@@ -295,6 +311,19 @@ namespace {
 void table_connection_t::start(pg_conn_t const &db_connection,
                                bool append) const
 {
+    if (m_target->binary()) {
+        log_debug("Table '{}' uses the binary COPY format.", table().name());
+    } else {
+        for (auto const &column : table().columns()) {
+            if (!column.create_only() && !column.binary_copy_type()) {
+                log_debug("Table '{}' uses the text COPY format because of"
+                          " column '{}'.",
+                          table().name(), column.name());
+                break;
+            }
+        }
+    }
+
     if (!append) {
         drop_table_if_exists(db_connection, table().schema(), table().name());
     }

src/flex-table.hpp

@@ -159,6 +159,12 @@ class flex_table_t
 
     std::string build_sql_copy_condition() const;
 
+    /**
+     * The types of all columns written by COPY for the binary COPY format.
+     * Empty if the table must use the text format.
+     */
+    std::vector<copy_field_type> binary_copy_types() const;
+
     std::string build_sql_create_id_index() const;
 
     /// Does this table take objects of the specified type?
@@ -293,6 +299,7 @@ class table_connection_t
           table->build_sql_column_list(), table->build_sql_copy_condition())),
       m_copy_mgr(copy_thread)
     {
+        m_target->set_binary_types(table->binary_copy_types());
     }
 
     void start(pg_conn_t const &db_connection, bool append) const;

src/middle-pgsql.cpp

@@ -18,12 +18,14 @@
 #include <cassert>
 #include <cstdint>
 #include <cstdlib>
+#include <initializer_list>
 #include <iterator>
 #include <memory>
 #include <stdexcept>
 #include <string>
 #include <unordered_map>
 #include <utility>
+#include <vector>
 
 #include <osmium/builder/osm_object_builder.hpp>
 #include <osmium/memory/buffer.hpp>
@@ -352,7 +354,7 @@ void members_to_json(osmium::RelationMemberList const &members,
 void middle_pgsql_t::copy_attributes(osmium::OSMObject const &obj)
 {
     if (obj.timestamp()) {
-        m_db_copy.add_column(obj.timestamp().to_iso());
+        m_db_copy.add_column(obj.timestamp());
     } else {
         m_db_copy.add_null_column();
     }
@@ -1070,6 +1072,8 @@ void middle_pgsql_t::write_users_table()
 
     auto const users_table = std::make_shared<db_target_descr_t>(
         m_options->dbschema, table_name, "id");
+    users_table->set_binary_types(
+        {copy_field_type::int4, copy_field_type::text});
 
     for (auto const &[id, name] : m_users) {
         m_db_copy.new_line(users_table);
@@ -1259,6 +1263,26 @@ middle_pgsql_t::middle_pgsql_t(std::shared_ptr<thread_pool_t> thread_pool,
     m_tables.nodes() = table_desc_t{*options, "nodes"};
     m_tables.ways() = table_desc_t{*options, "ways"};
     m_tables.relations() = table_desc_t{*options, "rels"};
+
+    // We know the types of all columns of the middle tables (see
+    // table_setup()), so they always use the binary COPY format.
+    using cft = copy_field_type;
+    std::vector<cft> attributes;
+    if (m_store_options.with_attributes) {
+        attributes = {cft::timestamptz, cft::int4, cft::int4, cft::int4};
+    }
+    auto const types = [&](std::vector<cft> columns,
+                           std::initializer_list<cft> after_attributes) {
+        columns.insert(columns.end(), attributes.cbegin(), attributes.cend());
+        columns.insert(columns.end(), after_attributes);
+        return columns;
+    };
+    m_tables.nodes().copy_target()->set_binary_types(
+        types({cft::int8, cft::int4, cft::int4}, {cft::jsonb}));
+    m_tables.ways().copy_target()->set_binary_types(
+        types({cft::int8}, {cft::int8_array, cft::jsonb}));
+    m_tables.relations().copy_target()->set_binary_types(
+        types({cft::int8}, {cft::jsonb, cft::jsonb}));
 }
 
 void middle_pgsql_t::set_requirements(