From 641a479b6ab5282d949b92359dd1ae55001770c3 Mon Sep 17 00:00:00 2001 From: Aaron Po Date: Thu, 30 Apr 2026 19:03:45 -0400 Subject: [PATCH] Refactor SQLite Export Service and ProcessRecord Method Signatures (#216) * Helper cleanup update bind to use dto for params consolidate translation units * Update planned class diagram --- docs/pipeline/diagrams/planned/class.puml | 12 +- tooling/pipeline/CMakeLists.txt | 11 +- .../includes/services/export_service.h | 2 +- .../services/sqlite_connection_helpers.h | 31 +++ .../includes/services/sqlite_export_service.h | 4 +- .../services/sqlite_export_service_helpers.h | 248 +----------------- .../includes/services/sqlite_handle_types.h | 36 +++ .../services/sqlite_statement_helpers.h | 116 ++++++++ .../src/services/sqlite/build_location_key.cc | 28 -- .../pipeline/src/services/sqlite/finalize.cc | 6 +- .../services/sqlite/finalize_statements.cc | 11 - .../helpers/sqlite_connection_helpers.cpp | 66 +++++ .../helpers/sqlite_statement_helpers.cpp | 108 ++++++++ .../src/services/sqlite/initialize.cc | 36 +++ .../src/services/sqlite/initialize_schema.cc | 16 -- .../src/services/sqlite/prepare_statements.cc | 16 -- .../src/services/sqlite/process_record.cc | 138 ++++++---- .../sqlite/rollback_and_close_no_throw.cc | 21 -- 18 files changed, 506 insertions(+), 400 deletions(-) create mode 100644 tooling/pipeline/includes/services/sqlite_connection_helpers.h create mode 100644 tooling/pipeline/includes/services/sqlite_handle_types.h create mode 100644 tooling/pipeline/includes/services/sqlite_statement_helpers.h delete mode 100644 tooling/pipeline/src/services/sqlite/build_location_key.cc delete mode 100644 tooling/pipeline/src/services/sqlite/finalize_statements.cc create mode 100644 tooling/pipeline/src/services/sqlite/helpers/sqlite_connection_helpers.cpp create mode 100644 tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cpp delete mode 100644 tooling/pipeline/src/services/sqlite/initialize_schema.cc delete mode 100644 tooling/pipeline/src/services/sqlite/prepare_statements.cc delete mode 100644 tooling/pipeline/src/services/sqlite/rollback_and_close_no_throw.cc diff --git a/docs/pipeline/diagrams/planned/class.puml b/docs/pipeline/diagrams/planned/class.puml index 3de4ae5..3b775cf 100644 --- a/docs/pipeline/diagrams/planned/class.puml +++ b/docs/pipeline/diagrams/planned/class.puml @@ -435,12 +435,12 @@ package "Infrastructure: Data Export" { - location_cache_ : std::unordered_map - brewery_cache_ : std::unordered_map + Initialize() : void - + ProcessBrewery(brewery : const GeneratedBrewery&) : uint64_t - + ProcessBeer(beer : const GeneratedBeer&) : uint64_t - + ProcessUser(user : const GeneratedUser&) : uint64_t - + ProcessCheckin(checkin : const GeneratedCheckin&) : uint64_t - + ProcessRating(rating : const GeneratedRating&) : void - + ProcessFollow(follow : const GeneratedFollow&) : void + + ProcessRecord(brewery : const GeneratedBrewery&) : uint64_t + + ProcessRecord(beer : const GeneratedBeer&) : uint64_t + + ProcessRecord(user : const GeneratedUser&) : uint64_t + + ProcessRecord(checkin : const GeneratedCheckin&) : uint64_t + + ProcessRecord(rating : const GeneratedRating&) : void + + ProcessRecord(follow : const GeneratedFollow&) : void + Finalize() : void - InitializeSchema() : void - PrepareStatements() : void diff --git a/tooling/pipeline/CMakeLists.txt b/tooling/pipeline/CMakeLists.txt index 0c0aab1..651b985 100644 --- a/tooling/pipeline/CMakeLists.txt +++ b/tooling/pipeline/CMakeLists.txt @@ -121,13 +121,8 @@ set(SOURCES src/services/wikipedia/fetch_extract.cc src/services/sqlite/sqlite_export_service.cc src/services/sqlite/build_database_path.cc - src/services/sqlite/build_location_key.cc - src/services/sqlite/initialize_schema.cc - src/services/sqlite/prepare_statements.cc - src/services/sqlite/initialize.cc - src/services/sqlite/process_record.cc - src/services/sqlite/finalize_statements.cc - src/services/sqlite/rollback_and_close_no_throw.cc + src/services/sqlite/process_record.cc + src/services/sqlite/initialize.cc src/services/sqlite/finalize.cc src/web_client/curl_global_state.cc src/web_client/curl_web_client_get.cc @@ -144,6 +139,8 @@ set(SOURCES src/data_generation/mock/generate_brewery.cc src/data_generation/mock/generate_user.cc src/json_handling/json_loader.cc + src/services/sqlite/helpers/sqlite_connection_helpers.cpp + src/services/sqlite/helpers/sqlite_statement_helpers.cpp ) # ============================================================================= diff --git a/tooling/pipeline/includes/services/export_service.h b/tooling/pipeline/includes/services/export_service.h index a45c483..55d0b06 100644 --- a/tooling/pipeline/includes/services/export_service.h +++ b/tooling/pipeline/includes/services/export_service.h @@ -31,7 +31,7 @@ class IExportService { * * @param brewery Generated brewery payload to store. */ - virtual void ProcessRecord(const GeneratedBrewery& brewery) = 0; + virtual uint64_t ProcessRecord(const GeneratedBrewery& brewery) = 0; /// @brief Finalizes the export destination. virtual void Finalize() = 0; diff --git a/tooling/pipeline/includes/services/sqlite_connection_helpers.h b/tooling/pipeline/includes/services/sqlite_connection_helpers.h new file mode 100644 index 0000000..7509c0e --- /dev/null +++ b/tooling/pipeline/includes/services/sqlite_connection_helpers.h @@ -0,0 +1,31 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_CONNECTION_HELPERS_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_CONNECTION_HELPERS_H_ + +/** + * @file services/sqlite_connection_helpers.h + * @brief Declarations for connection-level SQLite helper functions. + */ + +#include +#include +#include +#include + +#include "services/sqlite_handle_types.h" + +namespace sqlite_export_service_internal { + +void ThrowSqliteError(sqlite3* db_handle, std::string_view action); + +SqliteDatabaseHandle OpenDatabase(const std::filesystem::path& path); + +void ExecSql(const SqliteDatabaseHandle& db_handle, std::string_view sql, + const char* action); + +void RollbackTransactionNoThrow(const SqliteDatabaseHandle& db_handle) noexcept; + +} // namespace sqlite_export_service_internal + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_CONNECTION_HELPERS_H_ + + diff --git a/tooling/pipeline/includes/services/sqlite_export_service.h b/tooling/pipeline/includes/services/sqlite_export_service.h index d3a7377..0fa998f 100644 --- a/tooling/pipeline/includes/services/sqlite_export_service.h +++ b/tooling/pipeline/includes/services/sqlite_export_service.h @@ -29,7 +29,7 @@ class SqliteExportService final : public IExportService { SqliteExportService& operator=(SqliteExportService&&) = delete; void Initialize() override; - void ProcessRecord(const GeneratedBrewery& brewery) override; + uint64_t ProcessRecord(const GeneratedBrewery& brewery) override; void Finalize() override; private: @@ -38,7 +38,7 @@ class SqliteExportService final : public IExportService { using SqliteStatementHandle = sqlite_export_service_internal::SqliteStatementHandle; - void InitializeSchema(); + void InitializeSchema() const; void PrepareStatements(); void RollbackAndCloseNoThrow() noexcept; void FinalizeStatements() noexcept; diff --git a/tooling/pipeline/includes/services/sqlite_export_service_helpers.h b/tooling/pipeline/includes/services/sqlite_export_service_helpers.h index e4e9ae0..9b63aa5 100644 --- a/tooling/pipeline/includes/services/sqlite_export_service_helpers.h +++ b/tooling/pipeline/includes/services/sqlite_export_service_helpers.h @@ -1,250 +1,10 @@ #ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_EXPORT_SERVICE_HELPERS_H_ #define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_EXPORT_SERVICE_HELPERS_H_ -/** - * @file services/sqlite_export_service_helpers.h - * @brief Internal SQLite export helpers shared across per-method translation - * units. - */ +/* Umbrella header for backward compatibility. */ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace sqlite_export_service_internal { - -struct SqliteDatabaseDeleter { - void operator()(sqlite3* handle) const noexcept { - if (handle != nullptr) { - sqlite3_close(handle); - } - } -}; - -struct SqliteStatementDeleter { - void operator()(sqlite3_stmt* statement) const noexcept { - if (statement != nullptr) { - sqlite3_finalize(statement); - } - } -}; - -using SqliteDatabaseHandle = std::unique_ptr; -using SqliteStatementHandle = - std::unique_ptr; - -inline constexpr std::string_view kCreateLocationsTableSql = R"sql( - -CREATE TABLE IF NOT EXISTS locations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - city TEXT NOT NULL, - state_province TEXT NOT NULL, - iso3166_2 TEXT NOT NULL, - country TEXT NOT NULL, - iso3166_1 TEXT NOT NULL, - local_languages_json TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - UNIQUE(city, state_province, iso3166_2, country, latitude, longitude) -); - -)sql"; - -inline constexpr std::string_view kCreateBreweriesTableSql = R"sql( - -CREATE TABLE IF NOT EXISTS breweries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - location_id INTEGER NOT NULL, - name_en TEXT NOT NULL, - description_en TEXT NOT NULL, - name_local TEXT NOT NULL, - description_local TEXT NOT NULL, - FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE -); - -CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id); - -)sql"; - -inline constexpr std::string_view kInsertLocationSql = R"sql( -INSERT INTO locations ( - city, - state_province, - iso3166_2, - country, - iso3166_1, - local_languages_json, - latitude, - longitude -) VALUES (?, ?, ?, ?, ?, ?, ?, ?); -)sql"; - -inline constexpr std::string_view kInsertBrewerySql = R"sql( -INSERT INTO breweries ( - location_id, - name_en, - description_en, - name_local, - description_local -) VALUES (?, ?, ?, ?, ?); -)sql"; - -inline constexpr int kLocationCityBindIndex = 1; -inline constexpr int kLocationStateProvinceBindIndex = 2; -inline constexpr int kLocationIso31662BindIndex = 3; -inline constexpr int kLocationCountryBindIndex = 4; -inline constexpr int kLocationIso31661BindIndex = 5; -inline constexpr int kLocationLanguagesBindIndex = 6; -inline constexpr int kLocationLatitudeBindIndex = 7; -inline constexpr int kLocationLongitudeBindIndex = 8; - -inline constexpr int kBreweryLocationIdBindIndex = 1; -inline constexpr int kBreweryEnglishNameBindIndex = 2; -inline constexpr int kBreweryEnglishDescriptionBindIndex = 3; -inline constexpr int kBreweryLocalNameBindIndex = 4; -inline constexpr int kBreweryLocalDescriptionBindIndex = 5; - -inline void ThrowSqliteError(sqlite3* db_handle, std::string_view action) { - const std::string message = - db_handle != nullptr ? sqlite3_errmsg(db_handle) : "unknown SQLite error"; - throw std::runtime_error(std::string(action) + ": " + message); -} - -inline SqliteDatabaseHandle OpenDatabase(const std::filesystem::path& path) { - sqlite3* raw_handle = nullptr; - const std::string path_string = path.string(); - const int result = sqlite3_open(path_string.c_str(), &raw_handle); - SqliteDatabaseHandle handle(raw_handle); - if (result != SQLITE_OK) { - const std::string message = raw_handle != nullptr - ? sqlite3_errmsg(raw_handle) - : "unknown SQLite error"; - throw std::runtime_error("Failed to open SQLite export database: " + - message); - } - - return handle; -} - -inline void ExecSql(const SqliteDatabaseHandle& db_handle, std::string_view sql, - const char* action) { - char* error_message = nullptr; - const std::string sql_text(sql); - const int result = sqlite3_exec(db_handle.get(), sql_text.c_str(), nullptr, - nullptr, &error_message); - if (result != SQLITE_OK) { - const std::string message = error_message != nullptr - ? error_message - : sqlite3_errmsg(db_handle.get()); - sqlite3_free(error_message); - throw std::runtime_error(std::string(action) + ": " + message); - } -} - -inline SqliteStatementHandle PrepareStatement( - const SqliteDatabaseHandle& db_handle, std::string_view sql, - const char* action) { - sqlite3_stmt* raw_statement = nullptr; - const std::string sql_text(sql); - const int result = sqlite3_prepare_v2(db_handle.get(), sql_text.c_str(), -1, - &raw_statement, nullptr); - SqliteStatementHandle statement(raw_statement); - if (result != SQLITE_OK) { - ThrowSqliteError(db_handle.get(), action); - } - - return statement; -} - -inline void ResetStatement(SqliteStatementHandle& statement) { - if (statement != nullptr) { - sqlite3_reset(statement.get()); - sqlite3_clear_bindings(statement.get()); - } -} - -inline void DeleteCharArray(void* data) noexcept { - delete[] static_cast(data); -} - -inline void BindText(const SqliteStatementHandle& statement, int index, - std::string_view value, const char* action) { - const auto byte_count = value.size(); - if (byte_count > static_cast(std::numeric_limits::max())) { - ThrowSqliteError(sqlite3_db_handle(statement.get()), action); - } - - auto buffer = std::make_unique(byte_count + 1); - std::memcpy(buffer.get(), value.data(), byte_count); - buffer[byte_count] = '\0'; - - char* raw_buffer = buffer.release(); - const int result = - sqlite3_bind_text(statement.get(), index, raw_buffer, - static_cast(byte_count), DeleteCharArray); - if (result != SQLITE_OK) { - DeleteCharArray(raw_buffer); - ThrowSqliteError(sqlite3_db_handle(statement.get()), action); - } -} - -inline void BindDouble(const SqliteStatementHandle& statement, int index, - double value, std::string_view action) { - const int result = sqlite3_bind_double(statement.get(), index, value); - if (result != SQLITE_OK) { - ThrowSqliteError(sqlite3_db_handle(statement.get()), action); - } -} - -inline void BindInt64(const SqliteStatementHandle& statement, int index, - sqlite3_int64 value, std::string_view action) { - const int result = sqlite3_bind_int64(statement.get(), index, value); - if (result != SQLITE_OK) { - ThrowSqliteError(sqlite3_db_handle(statement.get()), action); - } -} - -inline void StepStatement(const SqliteDatabaseHandle& db_handle, - const SqliteStatementHandle& statement, - std::string_view action) { - const int result = sqlite3_step(statement.get()); - if (result != SQLITE_DONE) { - ThrowSqliteError(db_handle.get(), action); - } -} - -inline sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) { - return sqlite3_last_insert_rowid(db_handle.get()); -} - -inline void RollbackTransactionNoThrow( - const SqliteDatabaseHandle& db_handle) noexcept { - if (!db_handle) { - return; - } - - sqlite3_exec(db_handle.get(), "ROLLBACK;", nullptr, nullptr, nullptr); -} - -inline std::string SerializeLocalLanguages( - const std::vector& local_languages) { - boost::json::array array; - array.reserve(local_languages.size()); - for (const auto& language : local_languages) { - array.emplace_back(language); - } - return boost::json::serialize(array); -} - -} // namespace sqlite_export_service_internal +#include "services/sqlite_handle_types.h" +#include "services/sqlite_connection_helpers.h" +#include "services/sqlite_statement_helpers.h" #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_EXPORT_SERVICE_HELPERS_H_ diff --git a/tooling/pipeline/includes/services/sqlite_handle_types.h b/tooling/pipeline/includes/services/sqlite_handle_types.h new file mode 100644 index 0000000..6994c4a --- /dev/null +++ b/tooling/pipeline/includes/services/sqlite_handle_types.h @@ -0,0 +1,36 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_HANDLE_TYPES_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_HANDLE_TYPES_H_ + +/** + * Shared handle and parameter type declarations used by SQLite helper units. + */ + +#include +#include +#include + +namespace sqlite_export_service_internal { + +struct SqliteDatabaseDeleter { + void operator()(sqlite3* handle) const noexcept; +}; + +struct SqliteStatementDeleter { + void operator()(sqlite3_stmt* statement) const noexcept; +}; + +using SqliteDatabaseHandle = std::unique_ptr; +using SqliteStatementHandle = + std::unique_ptr; + +template +struct BindParam { + int index; + T value; + std::string_view action; +}; + +} // namespace sqlite_export_service_internal + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_HANDLE_TYPES_H_ + diff --git a/tooling/pipeline/includes/services/sqlite_statement_helpers.h b/tooling/pipeline/includes/services/sqlite_statement_helpers.h new file mode 100644 index 0000000..5f3315b --- /dev/null +++ b/tooling/pipeline/includes/services/sqlite_statement_helpers.h @@ -0,0 +1,116 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_STATEMENT_HELPERS_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_STATEMENT_HELPERS_H_ + +/** + * @file services/sqlite_statement_helpers.h + * @brief Declarations for statement-level SQLite helper functions and constants. + */ + +#include +#include +#include +#include + +#include "services/sqlite_handle_types.h" + +namespace sqlite_export_service_internal { + +inline constexpr std::string_view kCreateLocationsTableSql = R"sql( + +CREATE TABLE IF NOT EXISTS locations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + city TEXT NOT NULL, + state_province TEXT NOT NULL, + iso3166_2 TEXT NOT NULL, + country TEXT NOT NULL, + iso3166_1 TEXT NOT NULL, + local_languages_json TEXT NOT NULL, + latitude REAL NOT NULL, + longitude REAL NOT NULL, + UNIQUE(city, state_province, iso3166_2, country, latitude, longitude) +); + +)sql"; + +inline constexpr std::string_view kCreateBreweriesTableSql = R"sql( + +CREATE TABLE IF NOT EXISTS breweries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + location_id INTEGER NOT NULL, + name_en TEXT NOT NULL, + description_en TEXT NOT NULL, + name_local TEXT NOT NULL, + description_local TEXT NOT NULL, + FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id); + +)sql"; + +inline constexpr std::string_view kInsertLocationSql = R"sql( +INSERT INTO locations ( + city, + state_province, + iso3166_2, + country, + iso3166_1, + local_languages_json, + latitude, + longitude +) VALUES (?, ?, ?, ?, ?, ?, ?, ?); +)sql"; + +inline constexpr std::string_view kInsertBrewerySql = R"sql( +INSERT INTO breweries ( + location_id, + name_en, + description_en, + name_local, + description_local +) VALUES (?, ?, ?, ?, ?); +)sql"; + +inline constexpr int kLocationCityBindIndex = 1; +inline constexpr int kLocationStateProvinceBindIndex = 2; +inline constexpr int kLocationIso31662BindIndex = 3; +inline constexpr int kLocationCountryBindIndex = 4; +inline constexpr int kLocationIso31661BindIndex = 5; +inline constexpr int kLocationLanguagesBindIndex = 6; +inline constexpr int kLocationLatitudeBindIndex = 7; +inline constexpr int kLocationLongitudeBindIndex = 8; + +inline constexpr int kBreweryLocationIdBindIndex = 1; +inline constexpr int kBreweryEnglishNameBindIndex = 2; +inline constexpr int kBreweryEnglishDescriptionBindIndex = 3; +inline constexpr int kBreweryLocalNameBindIndex = 4; +inline constexpr int kBreweryLocalDescriptionBindIndex = 5; + +SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, + std::string_view sql, + const char* action); + +void ResetStatement(SqliteStatementHandle& statement); + +void Bind(const SqliteStatementHandle& statement, + const BindParam& param); + +void Bind(const SqliteStatementHandle& statement, + const BindParam& param); + +void Bind(const SqliteStatementHandle& statement, + const BindParam& param); + +void StepStatement(const SqliteDatabaseHandle& db_handle, + const SqliteStatementHandle& statement, + std::string_view action); + +sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle); + +std::string SerializeLocalLanguages(const std::vector& local_languages); +std::string SerializeVector(const std::vector& str_vec); + +} // namespace sqlite_export_service_internal + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_SQLITE_STATEMENT_HELPERS_H_ + diff --git a/tooling/pipeline/src/services/sqlite/build_location_key.cc b/tooling/pipeline/src/services/sqlite/build_location_key.cc deleted file mode 100644 index 472f2d7..0000000 --- a/tooling/pipeline/src/services/sqlite/build_location_key.cc +++ /dev/null @@ -1,28 +0,0 @@ -/** - * @file services/sqlite/build_location_key.cc - * @brief SqliteExportService::BuildLocationKey() implementation. - */ - -#include -#include - -#include "services/sqlite_export_service.h" -#include "services/sqlite_export_service_helpers.h" - -constexpr int kLocationPrecision = 17; - -std::string SqliteExportService::BuildLocationKey(const Location& location) { - std::ostringstream key_stream; - key_stream << location.city << '\n' - << location.state_province << '\n' - << location.iso3166_2 << '\n' - << location.country << '\n' - << location.iso3166_1 << '\n' - << std::setprecision(kLocationPrecision) << location.latitude - << '\n' - << std::setprecision(kLocationPrecision) << location.longitude - << '\n' - << sqlite_export_service_internal::SerializeLocalLanguages( - location.local_languages); - return key_stream.str(); -} diff --git a/tooling/pipeline/src/services/sqlite/finalize.cc b/tooling/pipeline/src/services/sqlite/finalize.cc index 156ee3c..ffb9d54 100644 --- a/tooling/pipeline/src/services/sqlite/finalize.cc +++ b/tooling/pipeline/src/services/sqlite/finalize.cc @@ -8,13 +8,15 @@ #include "services/sqlite_export_service.h" #include "services/sqlite_export_service_helpers.h" + void SqliteExportService::Finalize() { if (db_handle_ == nullptr) { return; } try { - FinalizeStatements(); + insert_brewery_stmt_.reset(); + insert_location_stmt_.reset(); if (transaction_open_) { sqlite_export_service_internal::ExecSql( db_handle_, "COMMIT;", "Failed to commit SQLite transaction"); @@ -27,4 +29,4 @@ void SqliteExportService::Finalize() { RollbackAndCloseNoThrow(); throw; } -} +} \ No newline at end of file diff --git a/tooling/pipeline/src/services/sqlite/finalize_statements.cc b/tooling/pipeline/src/services/sqlite/finalize_statements.cc deleted file mode 100644 index 15b29a6..0000000 --- a/tooling/pipeline/src/services/sqlite/finalize_statements.cc +++ /dev/null @@ -1,11 +0,0 @@ -/** - * @file services/sqlite/finalize_statements.cc - * @brief SqliteExportService::FinalizeStatements() implementation. - */ - -#include "services/sqlite_export_service.h" - -void SqliteExportService::FinalizeStatements() noexcept { - insert_brewery_stmt_.reset(); - insert_location_stmt_.reset(); -} diff --git a/tooling/pipeline/src/services/sqlite/helpers/sqlite_connection_helpers.cpp b/tooling/pipeline/src/services/sqlite/helpers/sqlite_connection_helpers.cpp new file mode 100644 index 0000000..1c302e6 --- /dev/null +++ b/tooling/pipeline/src/services/sqlite/helpers/sqlite_connection_helpers.cpp @@ -0,0 +1,66 @@ +#include "services/sqlite_connection_helpers.h" + +#include + +namespace sqlite_export_service_internal { + +void SqliteDatabaseDeleter::operator()(sqlite3* handle) const noexcept { + if (handle != nullptr) { + sqlite3_close(handle); + } +} + +void SqliteStatementDeleter::operator()(sqlite3_stmt* statement) const noexcept { + if (statement != nullptr) { + sqlite3_finalize(statement); + } +} + +void ThrowSqliteError(sqlite3* db_handle, std::string_view action) { + const std::string message = + db_handle != nullptr ? sqlite3_errmsg(db_handle) : "unknown SQLite error"; + throw std::runtime_error(std::string(action) + ": " + message); +} + +SqliteDatabaseHandle OpenDatabase(const std::filesystem::path& path) { + + sqlite3* raw_handle = nullptr; + const int result = sqlite3_open(path.string().c_str(), &raw_handle); + + SqliteDatabaseHandle handle(raw_handle); + if (result != SQLITE_OK) { + const std::string message = raw_handle != nullptr + ? sqlite3_errmsg(raw_handle) + : "unknown SQLite error"; + throw std::runtime_error("Failed to open SQLite export database: " + + message); + } + + return handle; +} + +void ExecSql(const SqliteDatabaseHandle& db_handle, std::string_view sql, + const char* action) { + char* error_message = nullptr; + const std::string sql_text(sql); + const int result = sqlite3_exec(db_handle.get(), sql_text.c_str(), nullptr, + nullptr, &error_message); + if (result != SQLITE_OK) { + const std::string message = error_message != nullptr + ? error_message + : sqlite3_errmsg(db_handle.get()); + sqlite3_free(error_message); + throw std::runtime_error(std::string(action) + ": " + message); + } +} + +void RollbackTransactionNoThrow(const SqliteDatabaseHandle& db_handle) noexcept { + if (!db_handle) { + return; + } + + sqlite3_exec(db_handle.get(), "ROLLBACK;", nullptr, nullptr, nullptr); +} + +} // namespace sqlite_export_service_internal + diff --git a/tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cpp b/tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cpp new file mode 100644 index 0000000..fd09056 --- /dev/null +++ b/tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cpp @@ -0,0 +1,108 @@ +#include "services/sqlite_statement_helpers.h" +#include "services/sqlite_connection_helpers.h" + +#include +#include +#include +#include +#include + +namespace sqlite_export_service_internal { + +SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, + std::string_view sql, + const char* action) { + sqlite3_stmt* raw_statement = nullptr; + const std::string sql_text(sql); + const int result = sqlite3_prepare_v2(db_handle.get(), sql_text.c_str(), -1, + &raw_statement, nullptr); + SqliteStatementHandle statement(raw_statement); + if (result != SQLITE_OK) { + ThrowSqliteError(db_handle.get(), action); + } + + return statement; +} + +void ResetStatement(SqliteStatementHandle& statement) { + if (statement != nullptr) { + sqlite3_reset(statement.get()); + sqlite3_clear_bindings(statement.get()); + } +} + +void Bind(const SqliteStatementHandle& statement, + const BindParam& param) { + const auto byte_count = param.value.size(); + if (byte_count > static_cast(std::numeric_limits::max())) { + ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action); + } + + auto delete_char_array = [](void* data) noexcept { + // NOLINT(cppcoreguidelines-owning-memory) + delete[] static_cast(data); + }; + + // NOLINT(cppcoreguidelines-avoid-c-arrays, modernize-avoid-c-arrays) + auto buffer = std::make_unique(byte_count + 1); + std::memcpy(buffer.get(), param.value.data(), byte_count); + buffer[byte_count] = '\0'; + + char* raw_buffer = buffer.release(); + + if (sqlite3_bind_text(statement.get(), param.index, raw_buffer, + static_cast(byte_count), + delete_char_array) != SQLITE_OK) { + delete_char_array(raw_buffer); + ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action); + } +} + +void Bind(const SqliteStatementHandle& statement, + const BindParam& param) { + if (sqlite3_bind_double(statement.get(), param.index, param.value) != + SQLITE_OK) { + ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action); + } +} + +void Bind(const SqliteStatementHandle& statement, + const BindParam& param) { + if (sqlite3_bind_int64(statement.get(), param.index, param.value) != + SQLITE_OK) { + ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action); + } +} + +void StepStatement(const SqliteDatabaseHandle& db_handle, + const SqliteStatementHandle& statement, + std::string_view action) { + if (sqlite3_step(statement.get()) != SQLITE_DONE) { + ThrowSqliteError(db_handle.get(), action); + } +} + +sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) { + return sqlite3_last_insert_rowid(db_handle.get()); +} + +std::string SerializeLocalLanguages( + const std::vector& local_languages) { + boost::json::array array; + array.reserve(local_languages.size()); + for (const auto& language : local_languages) { + array.emplace_back(language); + } + return boost::json::serialize(array); +} + +std::string SerializeVector(const std::vector& str_vec) { + boost::json::array array(str_vec.size()); + for (const auto& s : str_vec) { + array.emplace_back(s); + } + return boost::json::serialize(array); +} + +} // namespace sqlite_export_service_internal + diff --git a/tooling/pipeline/src/services/sqlite/initialize.cc b/tooling/pipeline/src/services/sqlite/initialize.cc index 0d3fa3c..f3f7560 100644 --- a/tooling/pipeline/src/services/sqlite/initialize.cc +++ b/tooling/pipeline/src/services/sqlite/initialize.cc @@ -11,6 +11,42 @@ #include "services/sqlite_export_service.h" #include "services/sqlite_export_service_helpers.h" + +void SqliteExportService::InitializeSchema() const { + sqlite_export_service_internal::ExecSql( + db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql, + "Failed to create SQLite locations table"); + sqlite_export_service_internal::ExecSql( + db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql, + "Failed to create SQLite breweries table"); +} + +void SqliteExportService::PrepareStatements() { + insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement( + db_handle_, sqlite_export_service_internal::kInsertLocationSql, + "Failed to prepare SQLite location insert statement"); + insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement( + db_handle_, sqlite_export_service_internal::kInsertBrewerySql, + "Failed to prepare SQLite brewery insert statement"); +} + +void SqliteExportService::RollbackAndCloseNoThrow() noexcept { + if (db_handle_ == nullptr) { + return; + } + + if (transaction_open_) { + sqlite_export_service_internal::RollbackTransactionNoThrow(db_handle_); + transaction_open_ = false; + } + + insert_brewery_stmt_.reset(); + insert_location_stmt_.reset(); + db_handle_.reset(); + location_cache_.clear(); +} + + void SqliteExportService::Initialize() { if (db_handle_ != nullptr) { throw std::runtime_error("SQLite export service is already initialized"); diff --git a/tooling/pipeline/src/services/sqlite/initialize_schema.cc b/tooling/pipeline/src/services/sqlite/initialize_schema.cc deleted file mode 100644 index e9e22ef..0000000 --- a/tooling/pipeline/src/services/sqlite/initialize_schema.cc +++ /dev/null @@ -1,16 +0,0 @@ -/** - * @file services/sqlite/initialize_schema.cc - * @brief SqliteExportService::InitializeSchema() implementation. - */ - -#include "services/sqlite_export_service.h" -#include "services/sqlite_export_service_helpers.h" - -void SqliteExportService::InitializeSchema() { - sqlite_export_service_internal::ExecSql( - db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql, - "Failed to create SQLite locations table"); - sqlite_export_service_internal::ExecSql( - db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql, - "Failed to create SQLite breweries table"); -} diff --git a/tooling/pipeline/src/services/sqlite/prepare_statements.cc b/tooling/pipeline/src/services/sqlite/prepare_statements.cc deleted file mode 100644 index ee61e4a..0000000 --- a/tooling/pipeline/src/services/sqlite/prepare_statements.cc +++ /dev/null @@ -1,16 +0,0 @@ -/** - * @file services/sqlite/prepare_statements.cc - * @brief SqliteExportService::PrepareStatements() implementation. - */ - -#include "services/sqlite_export_service.h" -#include "services/sqlite_export_service_helpers.h" - -void SqliteExportService::PrepareStatements() { - insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement( - db_handle_, sqlite_export_service_internal::kInsertLocationSql, - "Failed to prepare SQLite location insert statement"); - insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement( - db_handle_, sqlite_export_service_internal::kInsertBrewerySql, - "Failed to prepare SQLite brewery insert statement"); -} diff --git a/tooling/pipeline/src/services/sqlite/process_record.cc b/tooling/pipeline/src/services/sqlite/process_record.cc index 4a84d80..786aa19 100644 --- a/tooling/pipeline/src/services/sqlite/process_record.cc +++ b/tooling/pipeline/src/services/sqlite/process_record.cc @@ -9,7 +9,25 @@ #include "services/sqlite_export_service.h" #include "services/sqlite_export_service_helpers.h" -void SqliteExportService::ProcessRecord(const GeneratedBrewery& brewery) { +constexpr int kLocationPrecision = 17; + +std::string SqliteExportService::BuildLocationKey(const Location& location) { + std::ostringstream key_stream; + key_stream << location.city << '\n' + << location.state_province << '\n' + << location.iso3166_2 << '\n' + << location.country << '\n' + << location.iso3166_1 << '\n' + << std::setprecision(kLocationPrecision) << location.latitude + << '\n' + << std::setprecision(kLocationPrecision) << location.longitude + << '\n' + << sqlite_export_service_internal::SerializeVector( + location.local_languages); + return key_stream.str(); +} + +uint64_t SqliteExportService::ProcessRecord(const GeneratedBrewery& brewery) { if (db_handle_ == nullptr || !transaction_open_) { throw std::runtime_error("SQLite export service is not initialized"); } @@ -22,44 +40,60 @@ void SqliteExportService::ProcessRecord(const GeneratedBrewery& brewery) { location_id = cached_location->second; } else { const std::string local_languages_json = - sqlite_export_service_internal::SerializeLocalLanguages( + sqlite_export_service_internal::SerializeVector( brewery.location.local_languages); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationCityBindIndex, - brewery.location.city, "Failed to bind SQLite location city"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kLocationCityBindIndex, + .value = brewery.location.city, + .action = "Failed to bind SQLite location city"}); + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationStateProvinceBindIndex, - brewery.location.state_province, - "Failed to bind SQLite location state/province"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = + sqlite_export_service_internal::kLocationStateProvinceBindIndex, + .value = brewery.location.state_province, + .action = "Failed to bind SQLite location state/province"}); + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationIso31662BindIndex, - brewery.location.iso3166_2, - "Failed to bind SQLite location ISO 3166-2 code"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kLocationIso31662BindIndex, + .value = brewery.location.iso3166_2, + .action = "Failed to bind SQLite location ISO 3166-2 code"}); + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationCountryBindIndex, - brewery.location.country, "Failed to bind SQLite location country"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kLocationCountryBindIndex, + .value = brewery.location.country, + .action = "Failed to bind SQLite location country"}); + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationIso31661BindIndex, - brewery.location.iso3166_1, - "Failed to bind SQLite location ISO 3166-1 code"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kLocationIso31661BindIndex, + .value = brewery.location.iso3166_1, + .action = "Failed to bind SQLite location ISO 3166-1 code"}); + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationLanguagesBindIndex, - local_languages_json, "Failed to bind SQLite location languages"); - sqlite_export_service_internal::BindDouble( + sqlite_export_service_internal::BindParam{ + .index = + sqlite_export_service_internal::kLocationLanguagesBindIndex, + .value = local_languages_json, + .action = "Failed to bind SQLite location languages"}); + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationLatitudeBindIndex, - brewery.location.latitude, "Failed to bind SQLite location latitude"); - sqlite_export_service_internal::BindDouble( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kLocationLatitudeBindIndex, + .value = brewery.location.latitude, + .action = "Failed to bind SQLite location latitude"}); + sqlite_export_service_internal::Bind( insert_location_stmt_, - sqlite_export_service_internal::kLocationLongitudeBindIndex, - brewery.location.longitude, "Failed to bind SQLite location longitude"); + sqlite_export_service_internal::BindParam{ + .index = + sqlite_export_service_internal::kLocationLongitudeBindIndex, + .value = brewery.location.longitude, + .action = "Failed to bind SQLite location longitude"}); sqlite_export_service_internal::StepStatement( db_handle_, insert_location_stmt_, @@ -70,31 +104,43 @@ void SqliteExportService::ProcessRecord(const GeneratedBrewery& brewery) { sqlite_export_service_internal::ResetStatement(insert_location_stmt_); } - sqlite_export_service_internal::BindInt64( + sqlite_export_service_internal::Bind( insert_brewery_stmt_, - sqlite_export_service_internal::kBreweryLocationIdBindIndex, location_id, - "Failed to bind SQLite brewery location id"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kBreweryLocationIdBindIndex, + .value = location_id, + .action = "Failed to bind SQLite brewery location id"}); + sqlite_export_service_internal::Bind( insert_brewery_stmt_, - sqlite_export_service_internal::kBreweryEnglishNameBindIndex, - brewery.brewery.name_en, "Failed to bind SQLite brewery English name"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kBreweryEnglishNameBindIndex, + .value = brewery.brewery.name_en, + .action = "Failed to bind SQLite brewery English name"}); + sqlite_export_service_internal::Bind( insert_brewery_stmt_, - sqlite_export_service_internal::kBreweryEnglishDescriptionBindIndex, - brewery.brewery.description_en, - "Failed to bind SQLite brewery English description"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal:: + kBreweryEnglishDescriptionBindIndex, + .value = brewery.brewery.description_en, + .action = "Failed to bind SQLite brewery English description"}); + sqlite_export_service_internal::Bind( insert_brewery_stmt_, - sqlite_export_service_internal::kBreweryLocalNameBindIndex, - brewery.brewery.name_local, "Failed to bind SQLite brewery local name"); - sqlite_export_service_internal::BindText( + sqlite_export_service_internal::BindParam{ + .index = sqlite_export_service_internal::kBreweryLocalNameBindIndex, + .value = brewery.brewery.name_local, + .action = "Failed to bind SQLite brewery local name"}); + sqlite_export_service_internal::Bind( insert_brewery_stmt_, - sqlite_export_service_internal::kBreweryLocalDescriptionBindIndex, - brewery.brewery.description_local, - "Failed to bind SQLite brewery local description"); + sqlite_export_service_internal::BindParam{ + .index = + sqlite_export_service_internal::kBreweryLocalDescriptionBindIndex, + .value = brewery.brewery.description_local, + .action = "Failed to bind SQLite brewery local description"}); sqlite_export_service_internal::StepStatement( db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row"); sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_); + + return sqlite_export_service_internal::LastInsertRowId(db_handle_); } diff --git a/tooling/pipeline/src/services/sqlite/rollback_and_close_no_throw.cc b/tooling/pipeline/src/services/sqlite/rollback_and_close_no_throw.cc deleted file mode 100644 index c90a395..0000000 --- a/tooling/pipeline/src/services/sqlite/rollback_and_close_no_throw.cc +++ /dev/null @@ -1,21 +0,0 @@ -/** - * @file services/sqlite/rollback_and_close_no_throw.cc - * @brief SqliteExportService::RollbackAndCloseNoThrow() implementation. - */ - -#include "services/sqlite_export_service.h" - -void SqliteExportService::RollbackAndCloseNoThrow() noexcept { - if (db_handle_ == nullptr) { - return; - } - - if (transaction_open_) { - sqlite_export_service_internal::RollbackTransactionNoThrow(db_handle_); - transaction_open_ = false; - } - - FinalizeStatements(); - db_handle_.reset(); - location_cache_.clear(); -}