mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-15 01:54:43 +00:00
Compare commits
2 Commits
5a8a9f4652
...
6c5f65c86e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c5f65c86e | ||
|
|
320696fb92 |
@@ -58,6 +58,10 @@
|
|||||||
#include "services/enrichment/mock_enrichment.h"
|
#include "services/enrichment/mock_enrichment.h"
|
||||||
#include "services/enrichment/wikipedia_service.h"
|
#include "services/enrichment/wikipedia_service.h"
|
||||||
|
|
||||||
|
// --- services: postal_code ---
|
||||||
|
#include "services/postal_code/mock_postal_code_service.h"
|
||||||
|
#include "services/postal_code/postal_code_service.h"
|
||||||
|
|
||||||
// --- services: logging ---
|
// --- services: logging ---
|
||||||
#include "services/logging/log_dispatcher.h"
|
#include "services/logging/log_dispatcher.h"
|
||||||
#include "services/logging/log_entry.h"
|
#include "services/logging/log_entry.h"
|
||||||
|
|||||||
@@ -16,6 +16,7 @@
|
|||||||
#include "services/database/export_service.h"
|
#include "services/database/export_service.h"
|
||||||
#include "services/enrichment/enrichment_service.h"
|
#include "services/enrichment/enrichment_service.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
#include "services/postal_code/postal_code_service.h"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Main orchestrator for the Biergarten data generation pipeline.
|
* @brief Main orchestrator for the Biergarten data generation pipeline.
|
||||||
@@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator {
|
|||||||
* @param exporter Database backend for persisting generated records.
|
* @param exporter Database backend for persisting generated records.
|
||||||
* @param curated_data_service Loads curated location, persona, and name
|
* @param curated_data_service Loads curated location, persona, and name
|
||||||
* data used to seed generation.
|
* data used to seed generation.
|
||||||
|
* @param postal_code_service Resolves a postal code for a city from its
|
||||||
|
* curated postal-code data.
|
||||||
* @param application_options CLI configuration and paths.
|
* @param application_options CLI configuration and paths.
|
||||||
*/
|
*/
|
||||||
BiergartenPipelineOrchestrator(
|
BiergartenPipelineOrchestrator(
|
||||||
@@ -42,6 +45,7 @@ class BiergartenPipelineOrchestrator {
|
|||||||
std::unique_ptr<DataGenerator> generator,
|
std::unique_ptr<DataGenerator> generator,
|
||||||
std::unique_ptr<IExportService> exporter,
|
std::unique_ptr<IExportService> exporter,
|
||||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||||
|
std::unique_ptr<IPostalCodeService> postal_code_service,
|
||||||
const ApplicationOptions& application_options);
|
const ApplicationOptions& application_options);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -78,6 +82,7 @@ class BiergartenPipelineOrchestrator {
|
|||||||
*/
|
*/
|
||||||
std::unique_ptr<IExportService> exporter_;
|
std::unique_ptr<IExportService> exporter_;
|
||||||
std::unique_ptr<ICuratedDataService> curated_data_service_;
|
std::unique_ptr<ICuratedDataService> curated_data_service_;
|
||||||
|
std::unique_ptr<IPostalCodeService> postal_code_service_;
|
||||||
|
|
||||||
ApplicationOptions application_options_;
|
ApplicationOptions application_options_;
|
||||||
|
|
||||||
@@ -87,7 +92,7 @@ class BiergartenPipelineOrchestrator {
|
|||||||
* @return Vector of locations randomly sampled per
|
* @return Vector of locations randomly sampled per
|
||||||
* PipelineOptions::location_count.
|
* PipelineOptions::location_count.
|
||||||
*/
|
*/
|
||||||
std::vector<Location> QueryLocations();
|
std::vector<City> QueryLocations();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Generate breweries for enriched cities.
|
* @brief Generate breweries for enriched cities.
|
||||||
|
|||||||
@@ -20,12 +20,10 @@ class DataGenerator {
|
|||||||
/**
|
/**
|
||||||
* @brief Generates brewery data for a location.
|
* @brief Generates brewery data for a location.
|
||||||
*
|
*
|
||||||
* @param location Location data
|
* @param city Enriched city the brewery is associated with.
|
||||||
* @param region_context Additional regional context text.
|
|
||||||
* @return Brewery generation result.
|
* @return Brewery generation result.
|
||||||
*/
|
*/
|
||||||
virtual BreweryResult GenerateBrewery(const Location& location,
|
virtual BreweryResult GenerateBrewery(const EnrichedCity& city) = 0;
|
||||||
const std::string& region_context) = 0;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Generates a user profile grounded in a sampled name and persona.
|
* @brief Generates a user profile grounded in a sampled name and persona.
|
||||||
|
|||||||
@@ -51,12 +51,11 @@ class LlamaGenerator final : public DataGenerator {
|
|||||||
/**
|
/**
|
||||||
* @brief Generates brewery data for a specific location.
|
* @brief Generates brewery data for a specific location.
|
||||||
*
|
*
|
||||||
* @param location Location object.
|
* @param location City object.
|
||||||
* @param region_context Additional regional context.
|
* @param region_context Additional regional context.
|
||||||
* @return Generated brewery result.
|
* @return Generated brewery result.
|
||||||
*/
|
*/
|
||||||
BreweryResult GenerateBrewery(const Location& location,
|
BreweryResult GenerateBrewery(const EnrichedCity&) override;
|
||||||
const std::string& region_context) override;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Generates a user profile grounded in a sampled name and persona.
|
* @brief Generates a user profile grounded in a sampled name and persona.
|
||||||
|
|||||||
@@ -20,20 +20,15 @@ class MockGenerator final : public DataGenerator {
|
|||||||
/**
|
/**
|
||||||
* @brief Generates deterministic brewery data for a location.
|
* @brief Generates deterministic brewery data for a location.
|
||||||
*
|
*
|
||||||
* @param location City and country names.
|
* @param enriched_city
|
||||||
* @param region_context Unused for mock generation.
|
|
||||||
* @return Generated brewery result.
|
* @return Generated brewery result.
|
||||||
*/
|
*/
|
||||||
BreweryResult GenerateBrewery(const Location& location,
|
|
||||||
const std::string& region_context) override;
|
|
||||||
|
|
||||||
|
BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override;
|
||||||
/**
|
/**
|
||||||
* @brief Generates deterministic user data grounded in a sampled name and
|
* @brief Generates deterministic user data grounded in a sampled name and
|
||||||
* persona.
|
* persona.
|
||||||
*
|
|
||||||
* @param city Enriched city the user is associated with.
|
|
||||||
* @param persona Persona archetype.
|
|
||||||
* @param name Sampled first/last name, copied directly into the result.
|
|
||||||
* @return Generated user result.
|
* @return Generated user result.
|
||||||
*/
|
*/
|
||||||
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
|
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
|
||||||
@@ -46,7 +41,7 @@ class MockGenerator final : public DataGenerator {
|
|||||||
* @param location City and country names.
|
* @param location City and country names.
|
||||||
* @return Deterministic hash value.
|
* @return Deterministic hash value.
|
||||||
*/
|
*/
|
||||||
static size_t DeterministicHash(const Location& location);
|
static size_t DeterministicHash(const City& location);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Combines city, persona, and name into a stable hash value.
|
* @brief Combines city, persona, and name into a stable hash value.
|
||||||
@@ -56,7 +51,7 @@ class MockGenerator final : public DataGenerator {
|
|||||||
* @param name Sampled first/last name.
|
* @param name Sampled first/last name.
|
||||||
* @return Deterministic hash value.
|
* @return Deterministic hash value.
|
||||||
*/
|
*/
|
||||||
static size_t DeterministicHash(const Location& location,
|
static size_t DeterministicHash(const City& location,
|
||||||
const UserPersona& persona, const Name& name);
|
const UserPersona& persona, const Name& name);
|
||||||
|
|
||||||
// Hash stride constants for deterministic distribution across fixed-size
|
// Hash stride constants for deterministic distribution across fixed-size
|
||||||
|
|||||||
@@ -87,15 +87,33 @@ struct UserResult {
|
|||||||
* @brief Enriched city data with Wikipedia context.
|
* @brief Enriched city data with Wikipedia context.
|
||||||
*/
|
*/
|
||||||
struct EnrichedCity {
|
struct EnrichedCity {
|
||||||
Location location;
|
City location;
|
||||||
std::string region_context{};
|
std::string region_context{};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief A physical address: the city plus a concrete postal code selected
|
||||||
|
* from that city's curated postal-code examples.
|
||||||
|
*/
|
||||||
|
struct BreweryAddress {
|
||||||
|
City city;
|
||||||
|
std::string postal_code{};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief A physical address for a generated user: the city plus a concrete
|
||||||
|
* postal code selected from that city's curated postal-code examples.
|
||||||
|
*/
|
||||||
|
struct UserAddress {
|
||||||
|
City city;
|
||||||
|
std::string postal_code{};
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Helper struct to store generated brewery data.
|
* @brief Helper struct to store generated brewery data.
|
||||||
*/
|
*/
|
||||||
struct BreweryRecord {
|
struct BreweryRecord {
|
||||||
Location location;
|
BreweryAddress address;
|
||||||
BreweryResult brewery;
|
BreweryResult brewery;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -107,7 +125,7 @@ struct BreweryRecord {
|
|||||||
* consumer can register real accounts from the pipeline's SQLite export.
|
* consumer can register real accounts from the pipeline's SQLite export.
|
||||||
*/
|
*/
|
||||||
struct UserRecord {
|
struct UserRecord {
|
||||||
Location location;
|
UserAddress address;
|
||||||
UserResult user;
|
UserResult user;
|
||||||
std::string email{};
|
std::string email{};
|
||||||
std::string date_of_birth{};
|
std::string date_of_birth{};
|
||||||
|
|||||||
@@ -22,16 +22,39 @@ class ILogger;
|
|||||||
namespace prog_opts = boost::program_options;
|
namespace prog_opts = boost::program_options;
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Location Models
|
// City Models
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Canonical location record for city-level generation.
|
* @brief Curated postal-code metadata for a city, mirroring the
|
||||||
|
* `postal_code` object in the locations dataset.
|
||||||
*/
|
*/
|
||||||
struct Location {
|
struct PostalCodeSpec {
|
||||||
|
/**
|
||||||
|
* @brief Permissive country-wide postal-code format regex, used as the
|
||||||
|
* validation fallback for a selected example.
|
||||||
|
*/
|
||||||
|
std::string country_format_regex{};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief City-specific postal-code regexes (more precise than the country
|
||||||
|
* format).
|
||||||
|
*/
|
||||||
|
std::vector<std::string> city_regexes{};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Concrete example postal codes for this city. The postal code
|
||||||
|
* service selects from these rather than synthesizing codes.
|
||||||
|
*/
|
||||||
|
std::vector<std::string> examples{};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Canonical city record for city-level generation.
|
||||||
|
*/
|
||||||
|
struct City {
|
||||||
std::string city{};
|
std::string city{};
|
||||||
std::string state_province{};
|
std::string state_province{};
|
||||||
std::string postal_code{};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief ISO 3166-2 subdivision code.
|
* @brief ISO 3166-2 subdivision code.
|
||||||
@@ -51,14 +74,10 @@ struct Location {
|
|||||||
std::vector<std::string> local_languages{};
|
std::vector<std::string> local_languages{};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Latitude in decimal degrees.
|
* @brief Postal-code metadata (format regex, city regexes, and concrete
|
||||||
|
* examples) sourced directly from the curated locations dataset.
|
||||||
*/
|
*/
|
||||||
double latitude{};
|
PostalCodeSpec postal_code{};
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Longitude in decimal degrees.
|
|
||||||
*/
|
|
||||||
double longitude{};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
|
|
||||||
using ForenameList = std::vector<ForenameEntry>;
|
using ForenameList = std::vector<ForenameEntry>;
|
||||||
using SurnameList = std::vector<std::string>;
|
using SurnameList = std::vector<std::string>;
|
||||||
using LocationsList = std::vector<Location>;
|
using LocationsList = std::vector<City>;
|
||||||
using PersonasList = std::vector<UserPersona>;
|
using PersonasList = std::vector<UserPersona>;
|
||||||
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
|
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
|
||||||
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;
|
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;
|
||||||
|
|||||||
@@ -45,27 +45,29 @@ class SqliteExportService final : public IExportService {
|
|||||||
void RollbackAndCloseNoThrow() noexcept;
|
void RollbackAndCloseNoThrow() noexcept;
|
||||||
|
|
||||||
[[nodiscard]] std::filesystem::path BuildDatabasePath() const;
|
[[nodiscard]] std::filesystem::path BuildDatabasePath() const;
|
||||||
[[nodiscard]] static std::string BuildLocationKey(const Location& location);
|
[[nodiscard]] static std::string BuildCityKey(const City& city);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Returns the row id for @p location, inserting it first if it has
|
* @brief Returns the row id for @p city, inserting it first if it has
|
||||||
* not already been seen during this run.
|
* not already been seen during this run.
|
||||||
*
|
*
|
||||||
* Shared by both ProcessRecord() overloads so breweries and users
|
* Shared by both ProcessRecord() overloads so brewery and user addresses
|
||||||
* referencing the same location resolve to the same row.
|
* referencing the same city resolve to the same row.
|
||||||
*/
|
*/
|
||||||
[[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location);
|
[[nodiscard]] sqlite3_int64 ResolveCityId(const City& city);
|
||||||
|
|
||||||
std::unique_ptr<IDateTimeProvider> date_time_provider_;
|
std::unique_ptr<IDateTimeProvider> date_time_provider_;
|
||||||
std::filesystem::path output_path_;
|
std::filesystem::path output_path_;
|
||||||
std::string run_timestamp_utc_;
|
std::string run_timestamp_utc_;
|
||||||
std::filesystem::path database_path_;
|
std::filesystem::path database_path_;
|
||||||
SqliteDatabaseHandle db_handle_;
|
SqliteDatabaseHandle db_handle_;
|
||||||
SqliteStatementHandle insert_location_stmt_;
|
SqliteStatementHandle insert_city_stmt_;
|
||||||
SqliteStatementHandle insert_brewery_stmt_;
|
SqliteStatementHandle insert_brewery_stmt_;
|
||||||
|
SqliteStatementHandle insert_brewery_address_stmt_;
|
||||||
SqliteStatementHandle insert_user_stmt_;
|
SqliteStatementHandle insert_user_stmt_;
|
||||||
|
SqliteStatementHandle insert_user_address_stmt_;
|
||||||
bool transaction_open_ = false;
|
bool transaction_open_ = false;
|
||||||
std::unordered_map<std::string, sqlite3_int64> location_cache_;
|
std::unordered_map<std::string, sqlite3_int64> city_cache_;
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_
|
||||||
|
|||||||
@@ -17,9 +17,9 @@
|
|||||||
|
|
||||||
namespace sqlite_export_service_internal {
|
namespace sqlite_export_service_internal {
|
||||||
|
|
||||||
inline constexpr std::string_view kCreateLocationsTableSql = R"sql(
|
inline constexpr std::string_view kCreateCitiesTableSql = R"sql(
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS locations (
|
CREATE TABLE IF NOT EXISTS cities (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
city TEXT NOT NULL,
|
city TEXT NOT NULL,
|
||||||
state_province TEXT NOT NULL,
|
state_province TEXT NOT NULL,
|
||||||
@@ -27,9 +27,9 @@ CREATE TABLE IF NOT EXISTS locations (
|
|||||||
country TEXT NOT NULL,
|
country TEXT NOT NULL,
|
||||||
iso3166_1 TEXT NOT NULL,
|
iso3166_1 TEXT NOT NULL,
|
||||||
local_languages_json TEXT NOT NULL,
|
local_languages_json TEXT NOT NULL,
|
||||||
latitude REAL NOT NULL,
|
postal_code_country_format_regex TEXT NOT NULL,
|
||||||
longitude REAL NOT NULL,
|
postal_code_city_regex_json TEXT NOT NULL,
|
||||||
UNIQUE(city, state_province, iso3166_2, country, latitude, longitude)
|
UNIQUE(city, state_province, iso3166_2, country)
|
||||||
);
|
);
|
||||||
|
|
||||||
)sql";
|
)sql";
|
||||||
@@ -38,15 +38,29 @@ inline constexpr std::string_view kCreateBreweriesTableSql = R"sql(
|
|||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS breweries (
|
CREATE TABLE IF NOT EXISTS breweries (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
location_id INTEGER NOT NULL,
|
|
||||||
name_en TEXT NOT NULL,
|
name_en TEXT NOT NULL,
|
||||||
description_en TEXT NOT NULL,
|
description_en TEXT NOT NULL,
|
||||||
name_local TEXT NOT NULL,
|
name_local TEXT NOT NULL,
|
||||||
description_local TEXT NOT NULL,
|
description_local TEXT NOT NULL
|
||||||
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
|
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id);
|
)sql";
|
||||||
|
|
||||||
|
inline constexpr std::string_view kCreateBreweryAddressesTableSql = R"sql(
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS brewery_addresses (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
brewery_id INTEGER NOT NULL,
|
||||||
|
city_id INTEGER NOT NULL,
|
||||||
|
postal_code TEXT NOT NULL,
|
||||||
|
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_brewery_id
|
||||||
|
ON brewery_addresses(brewery_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_city_id
|
||||||
|
ON brewery_addresses(city_id);
|
||||||
|
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
@@ -54,7 +68,6 @@ inline constexpr std::string_view kCreateUsersTableSql = R"sql(
|
|||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS users (
|
CREATE TABLE IF NOT EXISTS users (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
location_id INTEGER NOT NULL,
|
|
||||||
first_name TEXT NOT NULL,
|
first_name TEXT NOT NULL,
|
||||||
last_name TEXT NOT NULL,
|
last_name TEXT NOT NULL,
|
||||||
gender TEXT NOT NULL,
|
gender TEXT NOT NULL,
|
||||||
@@ -62,40 +75,61 @@ CREATE TABLE IF NOT EXISTS users (
|
|||||||
bio TEXT NOT NULL,
|
bio TEXT NOT NULL,
|
||||||
activity_weight REAL NOT NULL,
|
activity_weight REAL NOT NULL,
|
||||||
email TEXT NOT NULL UNIQUE,
|
email TEXT NOT NULL UNIQUE,
|
||||||
date_of_birth TEXT NOT NULL,
|
date_of_birth TEXT NOT NULL
|
||||||
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
|
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id);
|
|
||||||
|
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
inline constexpr std::string_view kInsertLocationSql = R"sql(
|
inline constexpr std::string_view kCreateUserAddressesTableSql = R"sql(
|
||||||
INSERT INTO locations (
|
|
||||||
|
CREATE TABLE IF NOT EXISTS user_addresses (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
user_id INTEGER NOT NULL,
|
||||||
|
city_id INTEGER NOT NULL,
|
||||||
|
postal_code TEXT NOT NULL,
|
||||||
|
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_user_addresses_user_id
|
||||||
|
ON user_addresses(user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_user_addresses_city_id
|
||||||
|
ON user_addresses(city_id);
|
||||||
|
|
||||||
|
)sql";
|
||||||
|
|
||||||
|
inline constexpr std::string_view kInsertCitySql = R"sql(
|
||||||
|
INSERT INTO cities (
|
||||||
city,
|
city,
|
||||||
state_province,
|
state_province,
|
||||||
iso3166_2,
|
iso3166_2,
|
||||||
country,
|
country,
|
||||||
iso3166_1,
|
iso3166_1,
|
||||||
local_languages_json,
|
local_languages_json,
|
||||||
latitude,
|
postal_code_country_format_regex,
|
||||||
longitude
|
postal_code_city_regex_json
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
inline constexpr std::string_view kInsertBrewerySql = R"sql(
|
inline constexpr std::string_view kInsertBrewerySql = R"sql(
|
||||||
INSERT INTO breweries (
|
INSERT INTO breweries (
|
||||||
location_id,
|
|
||||||
name_en,
|
name_en,
|
||||||
description_en,
|
description_en,
|
||||||
name_local,
|
name_local,
|
||||||
description_local
|
description_local
|
||||||
) VALUES (?, ?, ?, ?, ?);
|
) VALUES (?, ?, ?, ?);
|
||||||
|
)sql";
|
||||||
|
|
||||||
|
inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
|
||||||
|
INSERT INTO brewery_addresses (
|
||||||
|
brewery_id,
|
||||||
|
city_id,
|
||||||
|
postal_code
|
||||||
|
) VALUES (?, ?, ?);
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
inline constexpr std::string_view kInsertUserSql = R"sql(
|
inline constexpr std::string_view kInsertUserSql = R"sql(
|
||||||
INSERT INTO users (
|
INSERT INTO users (
|
||||||
location_id,
|
|
||||||
first_name,
|
first_name,
|
||||||
last_name,
|
last_name,
|
||||||
gender,
|
gender,
|
||||||
@@ -104,33 +138,45 @@ INSERT INTO users (
|
|||||||
activity_weight,
|
activity_weight,
|
||||||
email,
|
email,
|
||||||
date_of_birth
|
date_of_birth
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
|
||||||
|
)sql";
|
||||||
|
|
||||||
|
inline constexpr std::string_view kInsertUserAddressSql = R"sql(
|
||||||
|
INSERT INTO user_addresses (
|
||||||
|
user_id,
|
||||||
|
city_id,
|
||||||
|
postal_code
|
||||||
|
) VALUES (?, ?, ?);
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
||||||
// placeholder order in the SQL above.
|
// placeholder order in the SQL above.
|
||||||
enum LocationBindIndex {
|
enum CityBindIndex {
|
||||||
kLocationCityBindIndex = 1,
|
kCityCityBindIndex = 1,
|
||||||
kLocationStateProvinceBindIndex,
|
kCityStateProvinceBindIndex,
|
||||||
kLocationIso31662BindIndex,
|
kCityIso31662BindIndex,
|
||||||
kLocationCountryBindIndex,
|
kCityCountryBindIndex,
|
||||||
kLocationIso31661BindIndex,
|
kCityIso31661BindIndex,
|
||||||
kLocationLanguagesBindIndex,
|
kCityLanguagesBindIndex,
|
||||||
kLocationLatitudeBindIndex,
|
kCityPostalCodeCountryFormatRegexBindIndex,
|
||||||
kLocationLongitudeBindIndex,
|
kCityPostalCodeCityRegexJsonBindIndex,
|
||||||
};
|
};
|
||||||
|
|
||||||
enum BreweryBindIndex {
|
enum BreweryBindIndex {
|
||||||
kBreweryLocationIdBindIndex = 1,
|
kBreweryEnglishNameBindIndex = 1,
|
||||||
kBreweryEnglishNameBindIndex,
|
|
||||||
kBreweryEnglishDescriptionBindIndex,
|
kBreweryEnglishDescriptionBindIndex,
|
||||||
kBreweryLocalNameBindIndex,
|
kBreweryLocalNameBindIndex,
|
||||||
kBreweryLocalDescriptionBindIndex,
|
kBreweryLocalDescriptionBindIndex,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum BreweryAddressBindIndex {
|
||||||
|
kBreweryAddressBreweryIdBindIndex = 1,
|
||||||
|
kBreweryAddressCityIdBindIndex,
|
||||||
|
kBreweryAddressPostalCodeBindIndex,
|
||||||
|
};
|
||||||
|
|
||||||
enum UserBindIndex {
|
enum UserBindIndex {
|
||||||
kUserLocationIdBindIndex = 1,
|
kUserFirstNameBindIndex = 1,
|
||||||
kUserFirstNameBindIndex,
|
|
||||||
kUserLastNameBindIndex,
|
kUserLastNameBindIndex,
|
||||||
kUserGenderBindIndex,
|
kUserGenderBindIndex,
|
||||||
kUserUsernameBindIndex,
|
kUserUsernameBindIndex,
|
||||||
@@ -140,6 +186,12 @@ enum UserBindIndex {
|
|||||||
kUserDateOfBirthBindIndex,
|
kUserDateOfBirthBindIndex,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum UserAddressBindIndex {
|
||||||
|
kUserAddressUserIdBindIndex = 1,
|
||||||
|
kUserAddressCityIdBindIndex,
|
||||||
|
kUserAddressPostalCodeBindIndex,
|
||||||
|
};
|
||||||
|
|
||||||
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
||||||
std::string_view sql,
|
std::string_view sql,
|
||||||
const char* action);
|
const char* action);
|
||||||
|
|||||||
@@ -20,10 +20,10 @@ class IEnrichmentService {
|
|||||||
/**
|
/**
|
||||||
* @brief Resolves contextual enrichment for a location.
|
* @brief Resolves contextual enrichment for a location.
|
||||||
*
|
*
|
||||||
* @param loc Location to enrich.
|
* @param loc City to enrich.
|
||||||
* @return Context text, or an empty string if unavailable.
|
* @return Context text, or an empty string if unavailable.
|
||||||
*/
|
*/
|
||||||
virtual std::string GetLocationContext(const Location& loc) = 0;
|
virtual std::string GetLocationContext(const City& loc) = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
*/
|
*/
|
||||||
class MockEnrichmentService final : public IEnrichmentService {
|
class MockEnrichmentService final : public IEnrichmentService {
|
||||||
public:
|
public:
|
||||||
std::string GetLocationContext(const Location& /*loc*/) override {
|
std::string GetLocationContext(const City& /*loc*/) override {
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ class WikipediaEnrichmentService final : public IEnrichmentService {
|
|||||||
/**
|
/**
|
||||||
* @brief Returns the Wikipedia-derived context for a location.
|
* @brief Returns the Wikipedia-derived context for a location.
|
||||||
*/
|
*/
|
||||||
[[nodiscard]] std::string GetLocationContext(const Location& loc) override;
|
[[nodiscard]] std::string GetLocationContext(const City& loc) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::string FetchExtract(std::string_view query);
|
std::string FetchExtract(std::string_view query);
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ enum class PipelinePhase {
|
|||||||
*/
|
*/
|
||||||
Startup,
|
Startup,
|
||||||
/**
|
/**
|
||||||
* @brief Location/context enrichment (e.g. Wikipedia lookups).
|
* @brief City/context enrichment (e.g. Wikipedia lookups).
|
||||||
*/
|
*/
|
||||||
Enrichment,
|
Enrichment,
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
|
||||||
|
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file services/postal_code/mock_postal_code_service.h
|
||||||
|
* @brief Data-driven postal code service that selects the first curated
|
||||||
|
* example for a city and validates it against the city's postal-code regex.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <format>
|
||||||
|
#include <regex>
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "services/postal_code/postal_code_service.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Postal code service backed entirely by the curated locations
|
||||||
|
* dataset: it returns the city's first example postal code after checking it
|
||||||
|
* against the curated postal-code regex.
|
||||||
|
*
|
||||||
|
* A precise city regex is preferred when present; otherwise the permissive
|
||||||
|
* country format regex is used as the fallback. A city with no examples, or an
|
||||||
|
* example that fails validation, throws so the caller can skip the city.
|
||||||
|
*/
|
||||||
|
class MockPostalCodeService final : public IPostalCodeService {
|
||||||
|
public:
|
||||||
|
std::string GeneratePostalCode(const City& city) override {
|
||||||
|
const PostalCodeSpec& spec = city.postal_code;
|
||||||
|
|
||||||
|
if (spec.examples.empty()) {
|
||||||
|
throw std::runtime_error(std::format(
|
||||||
|
"No postal-code examples available for city '{}'", city.city));
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string& candidate = spec.examples.front();
|
||||||
|
|
||||||
|
if (!MatchesAnyPattern(candidate, spec)) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
std::format("Postal code '{}' for city '{}' does not match the "
|
||||||
|
"curated postal-code regex",
|
||||||
|
candidate, city.city));
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
/**
|
||||||
|
* @brief Returns true if @p candidate matches any city regex, or the
|
||||||
|
* country format regex when no city regex matches. When the spec carries no
|
||||||
|
* usable regex at all, validation is skipped (returns true).
|
||||||
|
*/
|
||||||
|
static bool MatchesAnyPattern(const std::string& candidate,
|
||||||
|
const PostalCodeSpec& spec) {
|
||||||
|
bool has_pattern = false;
|
||||||
|
|
||||||
|
for (const std::string& pattern : spec.city_regexes) {
|
||||||
|
if (pattern.empty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
has_pattern = true;
|
||||||
|
if (std::regex_match(candidate, std::regex(pattern))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!spec.country_format_regex.empty()) {
|
||||||
|
has_pattern = true;
|
||||||
|
if (std::regex_match(candidate, std::regex(spec.country_format_regex))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return !has_pattern;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
|
||||||
|
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file services/postal_code/postal_code_service.h
|
||||||
|
* @brief Abstraction for resolving a postal code for a city.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "data_model/models.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Interface for services that resolve a postal code for a city.
|
||||||
|
*/
|
||||||
|
class IPostalCodeService {
|
||||||
|
public:
|
||||||
|
IPostalCodeService() = default;
|
||||||
|
virtual ~IPostalCodeService() = default;
|
||||||
|
|
||||||
|
IPostalCodeService(const IPostalCodeService&) = delete;
|
||||||
|
IPostalCodeService& operator=(const IPostalCodeService&) = delete;
|
||||||
|
IPostalCodeService(IPostalCodeService&&) = delete;
|
||||||
|
IPostalCodeService& operator=(IPostalCodeService&&) = delete;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Resolves a postal code for @p city.
|
||||||
|
*
|
||||||
|
* @param city City whose curated postal-code examples supply the result.
|
||||||
|
* @return A postal code string, or an empty string if none is available.
|
||||||
|
*/
|
||||||
|
virtual std::string GeneratePostalCode(const City& city) = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -13,10 +13,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
|
|||||||
std::unique_ptr<DataGenerator> generator,
|
std::unique_ptr<DataGenerator> generator,
|
||||||
std::unique_ptr<IExportService> exporter,
|
std::unique_ptr<IExportService> exporter,
|
||||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||||
|
std::unique_ptr<IPostalCodeService> postal_code_service,
|
||||||
const ApplicationOptions& application_options)
|
const ApplicationOptions& application_options)
|
||||||
: logger_(std::move(logger)),
|
: logger_(std::move(logger)),
|
||||||
context_service_(std::move(context_service)),
|
context_service_(std::move(context_service)),
|
||||||
generator_(std::move(generator)),
|
generator_(std::move(generator)),
|
||||||
exporter_(std::move(exporter)),
|
exporter_(std::move(exporter)),
|
||||||
curated_data_service_(std::move(curated_data_service)),
|
curated_data_service_(std::move(curated_data_service)),
|
||||||
|
postal_code_service_(std::move(postal_code_service)),
|
||||||
application_options_(application_options) {}
|
application_options_(application_options) {}
|
||||||
@@ -22,12 +22,15 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
|||||||
|
|
||||||
const auto generate_record =
|
const auto generate_record =
|
||||||
[this, &skipped_count](
|
[this, &skipped_count](
|
||||||
const Location& location,
|
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
|
||||||
const std::string& region_context) -> std::optional<BreweryRecord> {
|
|
||||||
try {
|
try {
|
||||||
const BreweryResult brewery =
|
const BreweryResult brewery = generator_->GenerateBrewery(enriched_city);
|
||||||
generator_->GenerateBrewery(location, region_context);
|
const std::string postal_code =
|
||||||
return BreweryRecord{.location = location, .brewery = brewery};
|
postal_code_service_->GeneratePostalCode(enriched_city.location);
|
||||||
|
return BreweryRecord{
|
||||||
|
.address = BreweryAddress{.city = enriched_city.location,
|
||||||
|
.postal_code = postal_code},
|
||||||
|
.brewery = brewery};
|
||||||
} catch (const std::exception& e) {
|
} catch (const std::exception& e) {
|
||||||
++skipped_count;
|
++skipped_count;
|
||||||
|
|
||||||
@@ -36,7 +39,8 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
|||||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||||
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
|
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
|
||||||
"generation failed: {}",
|
"generation failed: {}",
|
||||||
location.city, location.country, e.what())});
|
enriched_city.location.city,
|
||||||
|
enriched_city.location.country, e.what())});
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -47,19 +51,18 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
|||||||
exporter_->ProcessRecord(record);
|
exporter_->ProcessRecord(record);
|
||||||
} catch (const std::exception& export_exception) {
|
} catch (const std::exception& export_exception) {
|
||||||
++export_failed_count;
|
++export_failed_count;
|
||||||
logger_->Log(
|
logger_->Log({.level = LogLevel::Warn,
|
||||||
{.level = LogLevel::Warn,
|
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
.message = std::format(
|
||||||
.message = std::format("[Pipeline] Generated brewery for '{}' ({}) "
|
"[Pipeline] Generated brewery for '{}' ({}) "
|
||||||
"but SQLite export failed: {}",
|
"but SQLite export failed: {}",
|
||||||
record.location.city, record.location.country,
|
record.address.city.city, record.address.city.country,
|
||||||
export_exception.what())});
|
export_exception.what())});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for (const auto& [location, region_context] : cities) {
|
for (const EnrichedCity& enriched_city : cities) {
|
||||||
const std::optional<BreweryRecord> record =
|
const std::optional<BreweryRecord> record = generate_record(enriched_city);
|
||||||
generate_record(location, region_context);
|
|
||||||
if (!record.has_value()) {
|
if (!record.has_value()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -135,9 +135,12 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
|||||||
try {
|
try {
|
||||||
const UserResult user =
|
const UserResult user =
|
||||||
generator_->GenerateUser(city, persona, sampled_name);
|
generator_->GenerateUser(city, persona, sampled_name);
|
||||||
|
const std::string postal_code =
|
||||||
|
postal_code_service_->GeneratePostalCode(city.location);
|
||||||
|
|
||||||
return UserRecord{
|
return UserRecord{
|
||||||
.location = city.location,
|
.address =
|
||||||
|
UserAddress{.city = city.location, .postal_code = postal_code},
|
||||||
.user = user,
|
.user = user,
|
||||||
.email = BuildEmail(sampled_name, used_email_local_parts),
|
.email = BuildEmail(sampled_name, used_email_local_parts),
|
||||||
.date_of_birth = GenerateDateOfBirth(rng),
|
.date_of_birth = GenerateDateOfBirth(rng),
|
||||||
@@ -161,13 +164,13 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
|||||||
} catch (const std::exception& export_exception) {
|
} catch (const std::exception& export_exception) {
|
||||||
++export_failed_count;
|
++export_failed_count;
|
||||||
|
|
||||||
logger_->Log(
|
logger_->Log({.level = LogLevel::Warn,
|
||||||
{.level = LogLevel::Warn,
|
.phase = PipelinePhase::UserGeneration,
|
||||||
.phase = PipelinePhase::UserGeneration,
|
.message = std::format(
|
||||||
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
|
"[Pipeline] Generated user for '{}' ({}) but "
|
||||||
"SQLite export failed: {}",
|
"SQLite export failed: {}",
|
||||||
record.location.city, record.location.country,
|
record.address.city.city, record.address.city.country,
|
||||||
export_exception.what())});
|
export_exception.what())});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -14,20 +14,20 @@
|
|||||||
void BiergartenPipelineOrchestrator::LogResults() const {
|
void BiergartenPipelineOrchestrator::LogResults() const {
|
||||||
boost::json::array brewery_output;
|
boost::json::array brewery_output;
|
||||||
|
|
||||||
for (const auto& [location, brewery] : generated_breweries_) {
|
for (const auto& [address, brewery] : generated_breweries_) {
|
||||||
|
const City& city = address.city;
|
||||||
brewery_output.push_back(boost::json::object{
|
brewery_output.push_back(boost::json::object{
|
||||||
{"name_en", brewery.name_en},
|
{"name_en", brewery.name_en},
|
||||||
{"description_en", brewery.description_en},
|
{"description_en", brewery.description_en},
|
||||||
{"name_local", brewery.name_local},
|
{"name_local", brewery.name_local},
|
||||||
{"description_local", brewery.description_local},
|
{"description_local", brewery.description_local},
|
||||||
{"location", boost::json::object{
|
{"address", boost::json::object{
|
||||||
{"city", location.city},
|
{"city", city.city},
|
||||||
{"country", location.country},
|
{"country", city.country},
|
||||||
{"state_province", location.state_province},
|
{"state_province", city.state_province},
|
||||||
{"iso3166_2", location.iso3166_2},
|
{"iso3166_2", city.iso3166_2},
|
||||||
{"latitude", location.latitude},
|
{"postal_code", address.postal_code},
|
||||||
{"longitude", location.longitude},
|
}}});
|
||||||
}}});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
std::ostringstream brewery_oss;
|
std::ostringstream brewery_oss;
|
||||||
|
|||||||
@@ -14,13 +14,13 @@
|
|||||||
#include "services/curated_data/curated_json_data_service.h"
|
#include "services/curated_data/curated_json_data_service.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
std::vector<Location>
|
std::vector<City>
|
||||||
BiergartenPipelineOrchestrator::QueryLocations() {
|
BiergartenPipelineOrchestrator::QueryLocations() {
|
||||||
logger_->Log({.level = LogLevel::Info,
|
logger_->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
|
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
|
||||||
|
|
||||||
const std::vector<Location>& all_locations =
|
const std::vector<City>& all_locations =
|
||||||
curated_data_service_->LoadLocations();
|
curated_data_service_->LoadLocations();
|
||||||
|
|
||||||
const size_t sample_count = std::min(
|
const size_t sample_count = std::min(
|
||||||
@@ -31,7 +31,7 @@ BiergartenPipelineOrchestrator::QueryLocations() {
|
|||||||
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
|
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
|
||||||
sample_count);
|
sample_count);
|
||||||
|
|
||||||
std::vector<Location> sampled_locations;
|
std::vector<City> sampled_locations;
|
||||||
sampled_locations.reserve(sample_count);
|
sampled_locations.reserve(sample_count);
|
||||||
|
|
||||||
std::random_device random_generator;
|
std::random_device random_generator;
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ bool BiergartenPipelineOrchestrator::Run() {
|
|||||||
try {
|
try {
|
||||||
exporter_->Initialize();
|
exporter_->Initialize();
|
||||||
|
|
||||||
std::vector<Location> cities = QueryLocations();
|
std::vector<City> cities = QueryLocations();
|
||||||
std::vector<EnrichedCity> enriched;
|
std::vector<EnrichedCity> enriched;
|
||||||
enriched.reserve(cities.size());
|
enriched.reserve(cities.size());
|
||||||
|
|
||||||
|
|||||||
@@ -36,7 +36,9 @@ static std::string FormatLocalLanguageCodes(
|
|||||||
static constexpr int kBreweryInitialMaxTokens = 2800;
|
static constexpr int kBreweryInitialMaxTokens = 2800;
|
||||||
|
|
||||||
BreweryResult LlamaGenerator::GenerateBrewery(
|
BreweryResult LlamaGenerator::GenerateBrewery(
|
||||||
const Location& location, const std::string& region_context) {
|
const EnrichedCity& enriched_city) {
|
||||||
|
const City& location = enriched_city.location;
|
||||||
|
const std::string& region_context = enriched_city.region_context;
|
||||||
/**
|
/**
|
||||||
* Preprocess and truncate region context to manageable size
|
* Preprocess and truncate region context to manageable size
|
||||||
*/
|
*/
|
||||||
@@ -66,7 +68,7 @@ BreweryResult LlamaGenerator::GenerateBrewery(
|
|||||||
* Store location context for retry prompts (without repeating full context)
|
* Store location context for retry prompts (without repeating full context)
|
||||||
*/
|
*/
|
||||||
const std::string retry_location =
|
const std::string retry_location =
|
||||||
std::format("Location: {}{}\nLocal language codes: {}", location.city,
|
std::format("City: {}{}\nLocal language codes: {}", location.city,
|
||||||
country_suffix, local_language_codes);
|
country_suffix, local_language_codes);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -8,14 +8,14 @@
|
|||||||
|
|
||||||
#include "data_generation/mock_generator.h"
|
#include "data_generation/mock_generator.h"
|
||||||
|
|
||||||
size_t MockGenerator::DeterministicHash(const Location& location) {
|
size_t MockGenerator::DeterministicHash(const City& location) {
|
||||||
size_t seed = 0;
|
size_t seed = 0;
|
||||||
boost::hash_combine(seed, location.city);
|
boost::hash_combine(seed, location.city);
|
||||||
boost::hash_combine(seed, location.country);
|
boost::hash_combine(seed, location.country);
|
||||||
return seed;
|
return seed;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t MockGenerator::DeterministicHash(const Location& location,
|
size_t MockGenerator::DeterministicHash(const City& location,
|
||||||
const UserPersona& persona,
|
const UserPersona& persona,
|
||||||
const Name& name) {
|
const Name& name) {
|
||||||
size_t seed = DeterministicHash(location);
|
size_t seed = DeterministicHash(location);
|
||||||
|
|||||||
@@ -11,7 +11,8 @@
|
|||||||
#include "data_generation/mock_generator.h"
|
#include "data_generation/mock_generator.h"
|
||||||
|
|
||||||
BreweryResult MockGenerator::GenerateBrewery(
|
BreweryResult MockGenerator::GenerateBrewery(
|
||||||
const Location& location, const std::string& /*region_context*/) {
|
const EnrichedCity& enriched_city) {
|
||||||
|
const City& location = enriched_city.location;
|
||||||
const size_t hash = DeterministicHash(location);
|
const size_t hash = DeterministicHash(location);
|
||||||
|
|
||||||
const std::string_view adjective =
|
const std::string_view adjective =
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ int main(const int argc, char** argv) {
|
|||||||
di::bind<ApplicationOptions>().to(options),
|
di::bind<ApplicationOptions>().to(options),
|
||||||
di::bind<std::string>().to(model_path),
|
di::bind<std::string>().to(model_path),
|
||||||
di::bind<IExportService>().to<SqliteExportService>(),
|
di::bind<IExportService>().to<SqliteExportService>(),
|
||||||
|
di::bind<IPostalCodeService>().to<MockPostalCodeService>(),
|
||||||
di::bind<ICuratedDataService>().to(
|
di::bind<ICuratedDataService>().to(
|
||||||
[options, &log_producer]() -> std::unique_ptr<ICuratedDataService> {
|
[options, &log_producer]() -> std::unique_ptr<ICuratedDataService> {
|
||||||
if (options.generator.use_mocked) {
|
if (options.generator.use_mocked) {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
/**
|
/**
|
||||||
* @file json_handling/json_loader.cc
|
* @file json_handling/json_loader.cc
|
||||||
* @brief Parses curated location JSON input into strongly typed Location
|
* @brief Parses curated location JSON input into strongly typed City
|
||||||
* records with strict field validation and descriptive error reporting.
|
* records with strict field validation and descriptive error reporting.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@@ -30,16 +30,6 @@ static std::string ReadRequiredString(const boost::json::object& object,
|
|||||||
return std::string(text);
|
return std::string(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
static double ReadRequiredNumber(const boost::json::object& object,
|
|
||||||
const char* key) {
|
|
||||||
const boost::json::value* value = object.if_contains(key);
|
|
||||||
if (value == nullptr || !value->is_number()) {
|
|
||||||
throw std::runtime_error(
|
|
||||||
std::format("Missing or invalid numeric field: {}", key));
|
|
||||||
}
|
|
||||||
return value->to_number<double>();
|
|
||||||
}
|
|
||||||
|
|
||||||
static std::vector<std::string> ReadRequiredStringArray(
|
static std::vector<std::string> ReadRequiredStringArray(
|
||||||
const boost::json::object& object, const char* key) {
|
const boost::json::object& object, const char* key) {
|
||||||
const boost::json::value* value = object.if_contains(key);
|
const boost::json::value* value = object.if_contains(key);
|
||||||
@@ -61,6 +51,21 @@ static std::vector<std::string> ReadRequiredStringArray(
|
|||||||
return items;
|
return items;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
|
||||||
|
const boost::json::value* value = object.if_contains("postal_code");
|
||||||
|
if (value == nullptr || !value->is_object()) {
|
||||||
|
throw std::runtime_error("Missing or invalid object field: postal_code");
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto& postal_code = value->as_object();
|
||||||
|
return PostalCodeSpec{
|
||||||
|
.country_format_regex =
|
||||||
|
ReadRequiredString(postal_code, "country_format_regex"),
|
||||||
|
.city_regexes = ReadRequiredStringArray(postal_code, "city_regex"),
|
||||||
|
.examples = ReadRequiredStringArray(postal_code, "examples"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
|
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
|
||||||
@@ -133,15 +138,14 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const auto& object = item.as_object();
|
const auto& object = item.as_object();
|
||||||
locations.push_back(Location{
|
locations.push_back(City{
|
||||||
.city = ReadRequiredString(object, "city"),
|
.city = ReadRequiredString(object, "city"),
|
||||||
.state_province = ReadRequiredString(object, "state_province"),
|
.state_province = ReadRequiredString(object, "state_province"),
|
||||||
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
|
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
|
||||||
.country = ReadRequiredString(object, "country"),
|
.country = ReadRequiredString(object, "country"),
|
||||||
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
|
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
|
||||||
.local_languages = ReadRequiredStringArray(object, "local_languages"),
|
.local_languages = ReadRequiredStringArray(object, "local_languages"),
|
||||||
.latitude = ReadRequiredNumber(object, "latitude"),
|
.postal_code = ReadPostalCodeSpec(object),
|
||||||
.longitude = ReadRequiredNumber(object, "longitude"),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
cache_.locations = std::move(locations);
|
cache_.locations = std::move(locations);
|
||||||
|
|||||||
@@ -7,38 +7,46 @@
|
|||||||
|
|
||||||
MockCuratedDataService::MockCuratedDataService()
|
MockCuratedDataService::MockCuratedDataService()
|
||||||
: locations_{
|
: locations_{
|
||||||
Location{.city = "Portland",
|
City{.city = "Portland",
|
||||||
.state_province = "Oregon",
|
.state_province = "Oregon",
|
||||||
.iso3166_2 = "US-OR",
|
.iso3166_2 = "US-OR",
|
||||||
.country = "United States",
|
.country = "United States",
|
||||||
.iso3166_1 = "US",
|
.iso3166_1 = "US",
|
||||||
.local_languages = {"en"},
|
.local_languages = {"en"},
|
||||||
.latitude = 45.5152,
|
.postal_code =
|
||||||
.longitude = -122.6784},
|
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
|
||||||
Location{.city = "Munich",
|
.city_regexes = {R"(^972\d{2}$)"},
|
||||||
.state_province = "Bavaria",
|
.examples = {"97201", "97202", "97209"}}},
|
||||||
.iso3166_2 = "DE-BY",
|
City{.city = "Munich",
|
||||||
.country = "Germany",
|
.state_province = "Bavaria",
|
||||||
.iso3166_1 = "DE",
|
.iso3166_2 = "DE-BY",
|
||||||
.local_languages = {"de"},
|
.country = "Germany",
|
||||||
.latitude = 48.1351,
|
.iso3166_1 = "DE",
|
||||||
.longitude = 11.5820},
|
.local_languages = {"de"},
|
||||||
Location{.city = "Lyon",
|
.postal_code =
|
||||||
.state_province = "Auvergne-Rhone-Alpes",
|
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
|
||||||
.iso3166_2 = "FR-ARA",
|
.city_regexes = {R"(^8\d{4}$)"},
|
||||||
.country = "France",
|
.examples = {"80331", "80333", "81667"}}},
|
||||||
.iso3166_1 = "FR",
|
City{.city = "Lyon",
|
||||||
.local_languages = {"fr"},
|
.state_province = "Auvergne-Rhone-Alpes",
|
||||||
.latitude = 45.7640,
|
.iso3166_2 = "FR-ARA",
|
||||||
.longitude = 4.8357},
|
.country = "France",
|
||||||
Location{.city = "Brussels",
|
.iso3166_1 = "FR",
|
||||||
.state_province = "Brussels-Capital",
|
.local_languages = {"fr"},
|
||||||
.iso3166_2 = "BE-BRU",
|
.postal_code =
|
||||||
.country = "Belgium",
|
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
|
||||||
.iso3166_1 = "BE",
|
.city_regexes = {R"(^690\d{2}$)"},
|
||||||
.local_languages = {"nl", "fr"},
|
.examples = {"69001", "69002", "69007"}}},
|
||||||
.latitude = 50.8503,
|
City{.city = "Brussels",
|
||||||
.longitude = 4.3517},
|
.state_province = "Brussels-Capital",
|
||||||
|
.iso3166_2 = "BE-BRU",
|
||||||
|
.country = "Belgium",
|
||||||
|
.iso3166_1 = "BE",
|
||||||
|
.local_languages = {"nl", "fr"},
|
||||||
|
.postal_code =
|
||||||
|
PostalCodeSpec{.country_format_regex = R"(^\d{4}$)",
|
||||||
|
.city_regexes = {R"(^1\d{3}$)"},
|
||||||
|
.examples = {"1000", "1050", "1180"}}},
|
||||||
},
|
},
|
||||||
personas_{
|
personas_{
|
||||||
UserPersona{.name = "Hophead Explorer",
|
UserPersona{.name = "Hophead Explorer",
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
#include "services/enrichment/wikipedia_service.h"
|
#include "services/enrichment/wikipedia_service.h"
|
||||||
|
|
||||||
std::string WikipediaEnrichmentService::GetLocationContext(
|
std::string WikipediaEnrichmentService::GetLocationContext(
|
||||||
const Location& loc) {
|
const City& loc) {
|
||||||
using namespace std::literals::chrono_literals;
|
using namespace std::literals::chrono_literals;
|
||||||
if (!this->client_) {
|
if (!this->client_) {
|
||||||
if (logger_) {
|
if (logger_) {
|
||||||
|
|||||||
@@ -14,9 +14,11 @@ void SqliteExportService::Finalize() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
insert_user_address_stmt_.reset();
|
||||||
insert_user_stmt_.reset();
|
insert_user_stmt_.reset();
|
||||||
|
insert_brewery_address_stmt_.reset();
|
||||||
insert_brewery_stmt_.reset();
|
insert_brewery_stmt_.reset();
|
||||||
insert_location_stmt_.reset();
|
insert_city_stmt_.reset();
|
||||||
if (transaction_open_) {
|
if (transaction_open_) {
|
||||||
sqlite_export_service_internal::ExecSql(
|
sqlite_export_service_internal::ExecSql(
|
||||||
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
|
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
|
||||||
@@ -24,7 +26,7 @@ void SqliteExportService::Finalize() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
db_handle_.reset();
|
db_handle_.reset();
|
||||||
location_cache_.clear();
|
city_cache_.clear();
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
RollbackAndCloseNoThrow();
|
RollbackAndCloseNoThrow();
|
||||||
throw;
|
throw;
|
||||||
|
|||||||
@@ -88,7 +88,8 @@ sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
std::string SerializeVector(const std::vector<std::string>& str_vec) {
|
std::string SerializeVector(const std::vector<std::string>& str_vec) {
|
||||||
boost::json::array array(str_vec.size());
|
boost::json::array array;
|
||||||
|
array.reserve(str_vec.size());
|
||||||
for (const auto& s : str_vec) {
|
for (const auto& s : str_vec) {
|
||||||
array.emplace_back(s);
|
array.emplace_back(s);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,26 +28,40 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
|
|||||||
|
|
||||||
void SqliteExportService::InitializeSchema() const {
|
void SqliteExportService::InitializeSchema() const {
|
||||||
sqlite_export_service_internal::ExecSql(
|
sqlite_export_service_internal::ExecSql(
|
||||||
db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql,
|
db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql,
|
||||||
"Failed to create SQLite locations table");
|
"Failed to create SQLite cities table");
|
||||||
sqlite_export_service_internal::ExecSql(
|
sqlite_export_service_internal::ExecSql(
|
||||||
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
|
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
|
||||||
"Failed to create SQLite breweries table");
|
"Failed to create SQLite breweries table");
|
||||||
|
sqlite_export_service_internal::ExecSql(
|
||||||
|
db_handle_,
|
||||||
|
sqlite_export_service_internal::kCreateBreweryAddressesTableSql,
|
||||||
|
"Failed to create SQLite brewery_addresses table");
|
||||||
sqlite_export_service_internal::ExecSql(
|
sqlite_export_service_internal::ExecSql(
|
||||||
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
|
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
|
||||||
"Failed to create SQLite users table");
|
"Failed to create SQLite users table");
|
||||||
|
sqlite_export_service_internal::ExecSql(
|
||||||
|
db_handle_, sqlite_export_service_internal::kCreateUserAddressesTableSql,
|
||||||
|
"Failed to create SQLite user_addresses table");
|
||||||
}
|
}
|
||||||
|
|
||||||
void SqliteExportService::PrepareStatements() {
|
void SqliteExportService::PrepareStatements() {
|
||||||
insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement(
|
insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement(
|
||||||
db_handle_, sqlite_export_service_internal::kInsertLocationSql,
|
db_handle_, sqlite_export_service_internal::kInsertCitySql,
|
||||||
"Failed to prepare SQLite location insert statement");
|
"Failed to prepare SQLite city insert statement");
|
||||||
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
|
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
|
||||||
db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
|
db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
|
||||||
"Failed to prepare SQLite brewery insert statement");
|
"Failed to prepare SQLite brewery insert statement");
|
||||||
|
insert_brewery_address_stmt_ =
|
||||||
|
sqlite_export_service_internal::PrepareStatement(
|
||||||
|
db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql,
|
||||||
|
"Failed to prepare SQLite brewery address insert statement");
|
||||||
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
|
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
|
||||||
db_handle_, sqlite_export_service_internal::kInsertUserSql,
|
db_handle_, sqlite_export_service_internal::kInsertUserSql,
|
||||||
"Failed to prepare SQLite user insert statement");
|
"Failed to prepare SQLite user insert statement");
|
||||||
|
insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement(
|
||||||
|
db_handle_, sqlite_export_service_internal::kInsertUserAddressSql,
|
||||||
|
"Failed to prepare SQLite user address insert statement");
|
||||||
}
|
}
|
||||||
|
|
||||||
void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
|
void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
|
||||||
@@ -60,11 +74,13 @@ void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
|
|||||||
transaction_open_ = false;
|
transaction_open_ = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
insert_user_address_stmt_.reset();
|
||||||
insert_user_stmt_.reset();
|
insert_user_stmt_.reset();
|
||||||
|
insert_brewery_address_stmt_.reset();
|
||||||
insert_brewery_stmt_.reset();
|
insert_brewery_stmt_.reset();
|
||||||
insert_location_stmt_.reset();
|
insert_city_stmt_.reset();
|
||||||
db_handle_.reset();
|
db_handle_.reset();
|
||||||
location_cache_.clear();
|
city_cache_.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
void SqliteExportService::Initialize() {
|
void SqliteExportService::Initialize() {
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
/**
|
/**
|
||||||
* @file services/sqlite/process_record.cc
|
* @file services/sqlite/process_record.cc
|
||||||
* @brief SqliteExportService::ProcessRecord() implementation
|
* @brief SqliteExportService::ProcessRecord() implementation
|
||||||
* and the shared location-resolution helper.
|
* and the shared city-resolution helper.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <iomanip>
|
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
#include <string>
|
||||||
@@ -12,94 +11,91 @@
|
|||||||
#include "services/database/sqlite_export_service.h"
|
#include "services/database/sqlite_export_service.h"
|
||||||
#include "services/database/sqlite_export_service_helpers.h"
|
#include "services/database/sqlite_export_service_helpers.h"
|
||||||
|
|
||||||
constexpr int kLocationPrecision = 17;
|
std::string SqliteExportService::BuildCityKey(const City& city) {
|
||||||
|
|
||||||
std::string SqliteExportService::BuildLocationKey(const Location& location) {
|
|
||||||
std::ostringstream key_stream;
|
std::ostringstream key_stream;
|
||||||
key_stream << location.city << '\n'
|
key_stream << city.city << '\n'
|
||||||
<< location.state_province << '\n'
|
<< city.state_province << '\n'
|
||||||
<< location.iso3166_2 << '\n'
|
<< city.iso3166_2 << '\n'
|
||||||
<< location.country << '\n'
|
<< city.country << '\n'
|
||||||
<< location.iso3166_1 << '\n'
|
<< city.iso3166_1 << '\n'
|
||||||
<< std::setprecision(kLocationPrecision) << location.latitude
|
|
||||||
<< '\n'
|
|
||||||
<< std::setprecision(kLocationPrecision) << location.longitude
|
|
||||||
<< '\n'
|
|
||||||
<< sqlite_export_service_internal::SerializeVector(
|
<< sqlite_export_service_internal::SerializeVector(
|
||||||
location.local_languages);
|
city.local_languages);
|
||||||
return key_stream.str();
|
return key_stream.str();
|
||||||
}
|
}
|
||||||
|
|
||||||
sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
|
sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) {
|
||||||
const std::string location_key = BuildLocationKey(location);
|
const std::string city_key = BuildCityKey(city);
|
||||||
const auto cached_location = location_cache_.find(location_key);
|
const auto cached_city = city_cache_.find(city_key);
|
||||||
if (cached_location != location_cache_.end()) {
|
if (cached_city != city_cache_.end()) {
|
||||||
return cached_location->second;
|
return cached_city->second;
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::string local_languages_json =
|
const std::string local_languages_json =
|
||||||
sqlite_export_service_internal::SerializeVector(location.local_languages);
|
sqlite_export_service_internal::SerializeVector(city.local_languages);
|
||||||
|
const std::string city_regex_json =
|
||||||
|
sqlite_export_service_internal::SerializeVector(
|
||||||
|
city.postal_code.city_regexes);
|
||||||
|
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index = sqlite_export_service_internal::kLocationCityBindIndex,
|
.index = sqlite_export_service_internal::kCityCityBindIndex,
|
||||||
.value = location.city,
|
.value = city.city,
|
||||||
.action = "Failed to bind SQLite location city"});
|
.action = "Failed to bind SQLite city name"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index =
|
.index = sqlite_export_service_internal::kCityStateProvinceBindIndex,
|
||||||
sqlite_export_service_internal::kLocationStateProvinceBindIndex,
|
.value = city.state_province,
|
||||||
.value = location.state_province,
|
.action = "Failed to bind SQLite city state/province"});
|
||||||
.action = "Failed to bind SQLite location state/province"});
|
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index = sqlite_export_service_internal::kLocationIso31662BindIndex,
|
.index = sqlite_export_service_internal::kCityIso31662BindIndex,
|
||||||
.value = location.iso3166_2,
|
.value = city.iso3166_2,
|
||||||
.action = "Failed to bind SQLite location ISO 3166-2 code"});
|
.action = "Failed to bind SQLite city ISO 3166-2 code"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index = sqlite_export_service_internal::kLocationCountryBindIndex,
|
.index = sqlite_export_service_internal::kCityCountryBindIndex,
|
||||||
.value = location.country,
|
.value = city.country,
|
||||||
.action = "Failed to bind SQLite location country"});
|
.action = "Failed to bind SQLite city country"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index = sqlite_export_service_internal::kLocationIso31661BindIndex,
|
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
|
||||||
.value = location.iso3166_1,
|
.value = city.iso3166_1,
|
||||||
.action = "Failed to bind SQLite location ISO 3166-1 code"});
|
.action = "Failed to bind SQLite city ISO 3166-1 code"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index = sqlite_export_service_internal::kLocationLanguagesBindIndex,
|
.index = sqlite_export_service_internal::kCityLanguagesBindIndex,
|
||||||
.value = local_languages_json,
|
.value = local_languages_json,
|
||||||
.action = "Failed to bind SQLite location languages"});
|
.action = "Failed to bind SQLite city languages"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index = sqlite_export_service_internal::kLocationLatitudeBindIndex,
|
.index = sqlite_export_service_internal::
|
||||||
.value = location.latitude,
|
kCityPostalCodeCountryFormatRegexBindIndex,
|
||||||
.action = "Failed to bind SQLite location latitude"});
|
.value = city.postal_code.country_format_regex,
|
||||||
|
.action = "Failed to bind SQLite city postal-code format regex"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_location_stmt_,
|
insert_city_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
.index = sqlite_export_service_internal::kLocationLongitudeBindIndex,
|
.index = sqlite_export_service_internal::
|
||||||
.value = location.longitude,
|
kCityPostalCodeCityRegexJsonBindIndex,
|
||||||
.action = "Failed to bind SQLite location longitude"});
|
.value = city_regex_json,
|
||||||
|
.action = "Failed to bind SQLite city postal-code city regexes"});
|
||||||
|
|
||||||
sqlite_export_service_internal::StepStatement(
|
sqlite_export_service_internal::StepStatement(
|
||||||
db_handle_, insert_location_stmt_,
|
db_handle_, insert_city_stmt_, "Failed to insert SQLite city row");
|
||||||
"Failed to insert SQLite location row");
|
|
||||||
|
|
||||||
const sqlite3_int64 location_id =
|
const sqlite3_int64 city_id =
|
||||||
sqlite_export_service_internal::LastInsertRowId(db_handle_);
|
sqlite_export_service_internal::LastInsertRowId(db_handle_);
|
||||||
location_cache_.emplace(location_key, location_id);
|
city_cache_.emplace(city_key, city_id);
|
||||||
sqlite_export_service_internal::ResetStatement(insert_location_stmt_);
|
sqlite_export_service_internal::ResetStatement(insert_city_stmt_);
|
||||||
|
|
||||||
return location_id;
|
return city_id;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
|
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
|
||||||
@@ -107,14 +103,7 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
|
|||||||
throw std::runtime_error("SQLite export service is not initialized");
|
throw std::runtime_error("SQLite export service is not initialized");
|
||||||
}
|
}
|
||||||
|
|
||||||
const sqlite3_int64 location_id = ResolveLocationId(brewery.location);
|
const sqlite3_int64 city_id = ResolveCityId(brewery.address.city);
|
||||||
|
|
||||||
sqlite_export_service_internal::Bind(
|
|
||||||
insert_brewery_stmt_,
|
|
||||||
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
|
|
||||||
.index = sqlite_export_service_internal::kBreweryLocationIdBindIndex,
|
|
||||||
.value = location_id,
|
|
||||||
.action = "Failed to bind SQLite brewery location id"});
|
|
||||||
|
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_brewery_stmt_,
|
insert_brewery_stmt_,
|
||||||
@@ -149,7 +138,38 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
|
|||||||
sqlite_export_service_internal::StepStatement(
|
sqlite_export_service_internal::StepStatement(
|
||||||
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
|
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
|
||||||
|
|
||||||
|
const sqlite3_int64 brewery_id =
|
||||||
|
sqlite_export_service_internal::LastInsertRowId(db_handle_);
|
||||||
|
|
||||||
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
|
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
|
||||||
|
|
||||||
return sqlite_export_service_internal::LastInsertRowId(db_handle_);
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_brewery_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
|
||||||
|
.index =
|
||||||
|
sqlite_export_service_internal::kBreweryAddressBreweryIdBindIndex,
|
||||||
|
.value = brewery_id,
|
||||||
|
.action = "Failed to bind SQLite brewery address brewery id"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_brewery_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
|
||||||
|
.index =
|
||||||
|
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
|
||||||
|
.value = city_id,
|
||||||
|
.action = "Failed to bind SQLite brewery address city id"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_brewery_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
|
.index = sqlite_export_service_internal::
|
||||||
|
kBreweryAddressPostalCodeBindIndex,
|
||||||
|
.value = brewery.address.postal_code,
|
||||||
|
.action = "Failed to bind SQLite brewery address postal code"});
|
||||||
|
|
||||||
|
sqlite_export_service_internal::StepStatement(
|
||||||
|
db_handle_, insert_brewery_address_stmt_,
|
||||||
|
"Failed to insert SQLite brewery address row");
|
||||||
|
|
||||||
|
sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_);
|
||||||
|
|
||||||
|
return static_cast<uint64_t>(brewery_id);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,14 +13,8 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
|
|||||||
throw std::runtime_error("SQLite export service is not initialized");
|
throw std::runtime_error("SQLite export service is not initialized");
|
||||||
}
|
}
|
||||||
|
|
||||||
const sqlite3_int64 location_id = ResolveLocationId(user.location);
|
const sqlite3_int64 city_id = ResolveCityId(user.address.city);
|
||||||
|
|
||||||
sqlite_export_service_internal::Bind(
|
|
||||||
insert_user_stmt_,
|
|
||||||
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
|
|
||||||
.index = sqlite_export_service_internal::kUserLocationIdBindIndex,
|
|
||||||
.value = location_id,
|
|
||||||
.action = "Failed to bind SQLite user location id"});
|
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_user_stmt_,
|
insert_user_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
@@ -73,7 +67,36 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
|
|||||||
sqlite_export_service_internal::StepStatement(
|
sqlite_export_service_internal::StepStatement(
|
||||||
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
|
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
|
||||||
|
|
||||||
|
const sqlite3_int64 user_id =
|
||||||
|
sqlite_export_service_internal::LastInsertRowId(db_handle_);
|
||||||
|
|
||||||
sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
|
sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
|
||||||
|
|
||||||
return sqlite_export_service_internal::LastInsertRowId(db_handle_);
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_user_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
|
||||||
|
.index = sqlite_export_service_internal::kUserAddressUserIdBindIndex,
|
||||||
|
.value = user_id,
|
||||||
|
.action = "Failed to bind SQLite user address user id"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_user_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
|
||||||
|
.index = sqlite_export_service_internal::kUserAddressCityIdBindIndex,
|
||||||
|
.value = city_id,
|
||||||
|
.action = "Failed to bind SQLite user address city id"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_user_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||||
|
.index =
|
||||||
|
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
|
||||||
|
.value = user.address.postal_code,
|
||||||
|
.action = "Failed to bind SQLite user address postal code"});
|
||||||
|
|
||||||
|
sqlite_export_service_internal::StepStatement(
|
||||||
|
db_handle_, insert_user_address_stmt_,
|
||||||
|
"Failed to insert SQLite user address row");
|
||||||
|
|
||||||
|
sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_);
|
||||||
|
|
||||||
|
return static_cast<uint64_t>(user_id);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user