Compare commits

...

2 Commits

Author SHA1 Message Date
Aaron Po
6c5f65c86e Pipeline: rename Location to City, add address tables + postal codes 2026-07-17 21:56:36 -04:00
Aaron Po
320696fb92 use enriched city in generation functions 2026-07-17 21:13:14 -04:00
35 changed files with 1557 additions and 357 deletions

View File

@@ -58,6 +58,10 @@
#include "services/enrichment/mock_enrichment.h" #include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h" #include "services/enrichment/wikipedia_service.h"
// --- services: postal_code ---
#include "services/postal_code/mock_postal_code_service.h"
#include "services/postal_code/postal_code_service.h"
// --- services: logging --- // --- services: logging ---
#include "services/logging/log_dispatcher.h" #include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h" #include "services/logging/log_entry.h"

View File

@@ -16,6 +16,7 @@
#include "services/database/export_service.h" #include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h" #include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
#include "services/postal_code/postal_code_service.h"
/** /**
* @brief Main orchestrator for the Biergarten data generation pipeline. * @brief Main orchestrator for the Biergarten data generation pipeline.
@@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator {
* @param exporter Database backend for persisting generated records. * @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name * @param curated_data_service Loads curated location, persona, and name
* data used to seed generation. * data used to seed generation.
* @param postal_code_service Resolves a postal code for a city from its
* curated postal-code data.
* @param application_options CLI configuration and paths. * @param application_options CLI configuration and paths.
*/ */
BiergartenPipelineOrchestrator( BiergartenPipelineOrchestrator(
@@ -42,6 +45,7 @@ class BiergartenPipelineOrchestrator {
std::unique_ptr<DataGenerator> generator, std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options); const ApplicationOptions& application_options);
/** /**
@@ -78,6 +82,7 @@ class BiergartenPipelineOrchestrator {
*/ */
std::unique_ptr<IExportService> exporter_; std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_; std::unique_ptr<ICuratedDataService> curated_data_service_;
std::unique_ptr<IPostalCodeService> postal_code_service_;
ApplicationOptions application_options_; ApplicationOptions application_options_;
@@ -87,7 +92,7 @@ class BiergartenPipelineOrchestrator {
* @return Vector of locations randomly sampled per * @return Vector of locations randomly sampled per
* PipelineOptions::location_count. * PipelineOptions::location_count.
*/ */
std::vector<Location> QueryLocations(); std::vector<City> QueryLocations();
/** /**
* @brief Generate breweries for enriched cities. * @brief Generate breweries for enriched cities.

View File

@@ -20,12 +20,10 @@ class DataGenerator {
/** /**
* @brief Generates brewery data for a location. * @brief Generates brewery data for a location.
* *
* @param location Location data * @param city Enriched city the brewery is associated with.
* @param region_context Additional regional context text.
* @return Brewery generation result. * @return Brewery generation result.
*/ */
virtual BreweryResult GenerateBrewery(const Location& location, virtual BreweryResult GenerateBrewery(const EnrichedCity& city) = 0;
const std::string& region_context) = 0;
/** /**
* @brief Generates a user profile grounded in a sampled name and persona. * @brief Generates a user profile grounded in a sampled name and persona.

View File

@@ -51,12 +51,11 @@ class LlamaGenerator final : public DataGenerator {
/** /**
* @brief Generates brewery data for a specific location. * @brief Generates brewery data for a specific location.
* *
* @param location Location object. * @param location City object.
* @param region_context Additional regional context. * @param region_context Additional regional context.
* @return Generated brewery result. * @return Generated brewery result.
*/ */
BreweryResult GenerateBrewery(const Location& location, BreweryResult GenerateBrewery(const EnrichedCity&) override;
const std::string& region_context) override;
/** /**
* @brief Generates a user profile grounded in a sampled name and persona. * @brief Generates a user profile grounded in a sampled name and persona.

View File

@@ -20,20 +20,15 @@ class MockGenerator final : public DataGenerator {
/** /**
* @brief Generates deterministic brewery data for a location. * @brief Generates deterministic brewery data for a location.
* *
* @param location City and country names. * @param enriched_city
* @param region_context Unused for mock generation.
* @return Generated brewery result. * @return Generated brewery result.
*/ */
BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) override;
BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override;
/** /**
* @brief Generates deterministic user data grounded in a sampled name and * @brief Generates deterministic user data grounded in a sampled name and
* persona. * persona.
*
* @param city Enriched city the user is associated with.
* @param persona Persona archetype.
* @param name Sampled first/last name, copied directly into the result.
* @return Generated user result. * @return Generated user result.
*/ */
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona, UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
@@ -46,7 +41,7 @@ class MockGenerator final : public DataGenerator {
* @param location City and country names. * @param location City and country names.
* @return Deterministic hash value. * @return Deterministic hash value.
*/ */
static size_t DeterministicHash(const Location& location); static size_t DeterministicHash(const City& location);
/** /**
* @brief Combines city, persona, and name into a stable hash value. * @brief Combines city, persona, and name into a stable hash value.
@@ -56,7 +51,7 @@ class MockGenerator final : public DataGenerator {
* @param name Sampled first/last name. * @param name Sampled first/last name.
* @return Deterministic hash value. * @return Deterministic hash value.
*/ */
static size_t DeterministicHash(const Location& location, static size_t DeterministicHash(const City& location,
const UserPersona& persona, const Name& name); const UserPersona& persona, const Name& name);
// Hash stride constants for deterministic distribution across fixed-size // Hash stride constants for deterministic distribution across fixed-size

View File

@@ -87,15 +87,33 @@ struct UserResult {
* @brief Enriched city data with Wikipedia context. * @brief Enriched city data with Wikipedia context.
*/ */
struct EnrichedCity { struct EnrichedCity {
Location location; City location;
std::string region_context{}; std::string region_context{};
}; };
/**
* @brief A physical address: the city plus a concrete postal code selected
* from that city's curated postal-code examples.
*/
struct BreweryAddress {
City city;
std::string postal_code{};
};
/**
* @brief A physical address for a generated user: the city plus a concrete
* postal code selected from that city's curated postal-code examples.
*/
struct UserAddress {
City city;
std::string postal_code{};
};
/** /**
* @brief Helper struct to store generated brewery data. * @brief Helper struct to store generated brewery data.
*/ */
struct BreweryRecord { struct BreweryRecord {
Location location; BreweryAddress address;
BreweryResult brewery; BreweryResult brewery;
}; };
@@ -107,7 +125,7 @@ struct BreweryRecord {
* consumer can register real accounts from the pipeline's SQLite export. * consumer can register real accounts from the pipeline's SQLite export.
*/ */
struct UserRecord { struct UserRecord {
Location location; UserAddress address;
UserResult user; UserResult user;
std::string email{}; std::string email{};
std::string date_of_birth{}; std::string date_of_birth{};

View File

@@ -22,16 +22,39 @@ class ILogger;
namespace prog_opts = boost::program_options; namespace prog_opts = boost::program_options;
// ============================================================================ // ============================================================================
// Location Models // City Models
// ============================================================================ // ============================================================================
/** /**
* @brief Canonical location record for city-level generation. * @brief Curated postal-code metadata for a city, mirroring the
* `postal_code` object in the locations dataset.
*/ */
struct Location { struct PostalCodeSpec {
/**
* @brief Permissive country-wide postal-code format regex, used as the
* validation fallback for a selected example.
*/
std::string country_format_regex{};
/**
* @brief City-specific postal-code regexes (more precise than the country
* format).
*/
std::vector<std::string> city_regexes{};
/**
* @brief Concrete example postal codes for this city. The postal code
* service selects from these rather than synthesizing codes.
*/
std::vector<std::string> examples{};
};
/**
* @brief Canonical city record for city-level generation.
*/
struct City {
std::string city{}; std::string city{};
std::string state_province{}; std::string state_province{};
std::string postal_code{};
/** /**
* @brief ISO 3166-2 subdivision code. * @brief ISO 3166-2 subdivision code.
@@ -51,14 +74,10 @@ struct Location {
std::vector<std::string> local_languages{}; std::vector<std::string> local_languages{};
/** /**
* @brief Latitude in decimal degrees. * @brief Postal-code metadata (format regex, city regexes, and concrete
* examples) sourced directly from the curated locations dataset.
*/ */
double latitude{}; PostalCodeSpec postal_code{};
/**
* @brief Longitude in decimal degrees.
*/
double longitude{};
}; };
// ============================================================================ // ============================================================================

View File

@@ -15,7 +15,7 @@
using ForenameList = std::vector<ForenameEntry>; using ForenameList = std::vector<ForenameEntry>;
using SurnameList = std::vector<std::string>; using SurnameList = std::vector<std::string>;
using LocationsList = std::vector<Location>; using LocationsList = std::vector<City>;
using PersonasList = std::vector<UserPersona>; using PersonasList = std::vector<UserPersona>;
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>; using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>; using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;

View File

@@ -45,27 +45,29 @@ class SqliteExportService final : public IExportService {
void RollbackAndCloseNoThrow() noexcept; void RollbackAndCloseNoThrow() noexcept;
[[nodiscard]] std::filesystem::path BuildDatabasePath() const; [[nodiscard]] std::filesystem::path BuildDatabasePath() const;
[[nodiscard]] static std::string BuildLocationKey(const Location& location); [[nodiscard]] static std::string BuildCityKey(const City& city);
/** /**
* @brief Returns the row id for @p location, inserting it first if it has * @brief Returns the row id for @p city, inserting it first if it has
* not already been seen during this run. * not already been seen during this run.
* *
* Shared by both ProcessRecord() overloads so breweries and users * Shared by both ProcessRecord() overloads so brewery and user addresses
* referencing the same location resolve to the same row. * referencing the same city resolve to the same row.
*/ */
[[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location); [[nodiscard]] sqlite3_int64 ResolveCityId(const City& city);
std::unique_ptr<IDateTimeProvider> date_time_provider_; std::unique_ptr<IDateTimeProvider> date_time_provider_;
std::filesystem::path output_path_; std::filesystem::path output_path_;
std::string run_timestamp_utc_; std::string run_timestamp_utc_;
std::filesystem::path database_path_; std::filesystem::path database_path_;
SqliteDatabaseHandle db_handle_; SqliteDatabaseHandle db_handle_;
SqliteStatementHandle insert_location_stmt_; SqliteStatementHandle insert_city_stmt_;
SqliteStatementHandle insert_brewery_stmt_; SqliteStatementHandle insert_brewery_stmt_;
SqliteStatementHandle insert_brewery_address_stmt_;
SqliteStatementHandle insert_user_stmt_; SqliteStatementHandle insert_user_stmt_;
SqliteStatementHandle insert_user_address_stmt_;
bool transaction_open_ = false; bool transaction_open_ = false;
std::unordered_map<std::string, sqlite3_int64> location_cache_; std::unordered_map<std::string, sqlite3_int64> city_cache_;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_

View File

@@ -17,9 +17,9 @@
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {
inline constexpr std::string_view kCreateLocationsTableSql = R"sql( inline constexpr std::string_view kCreateCitiesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS locations ( CREATE TABLE IF NOT EXISTS cities (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
city TEXT NOT NULL, city TEXT NOT NULL,
state_province TEXT NOT NULL, state_province TEXT NOT NULL,
@@ -27,9 +27,9 @@ CREATE TABLE IF NOT EXISTS locations (
country TEXT NOT NULL, country TEXT NOT NULL,
iso3166_1 TEXT NOT NULL, iso3166_1 TEXT NOT NULL,
local_languages_json TEXT NOT NULL, local_languages_json TEXT NOT NULL,
latitude REAL NOT NULL, postal_code_country_format_regex TEXT NOT NULL,
longitude REAL NOT NULL, postal_code_city_regex_json TEXT NOT NULL,
UNIQUE(city, state_province, iso3166_2, country, latitude, longitude) UNIQUE(city, state_province, iso3166_2, country)
); );
)sql"; )sql";
@@ -38,15 +38,29 @@ inline constexpr std::string_view kCreateBreweriesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS breweries ( CREATE TABLE IF NOT EXISTS breweries (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
location_id INTEGER NOT NULL,
name_en TEXT NOT NULL, name_en TEXT NOT NULL,
description_en TEXT NOT NULL, description_en TEXT NOT NULL,
name_local TEXT NOT NULL, name_local TEXT NOT NULL,
description_local TEXT NOT NULL, description_local TEXT NOT NULL
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
); );
CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id); )sql";
inline constexpr std::string_view kCreateBreweryAddressesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS brewery_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
brewery_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_brewery_id
ON brewery_addresses(brewery_id);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_city_id
ON brewery_addresses(city_id);
)sql"; )sql";
@@ -54,7 +68,6 @@ inline constexpr std::string_view kCreateUsersTableSql = R"sql(
CREATE TABLE IF NOT EXISTS users ( CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
location_id INTEGER NOT NULL,
first_name TEXT NOT NULL, first_name TEXT NOT NULL,
last_name TEXT NOT NULL, last_name TEXT NOT NULL,
gender TEXT NOT NULL, gender TEXT NOT NULL,
@@ -62,40 +75,61 @@ CREATE TABLE IF NOT EXISTS users (
bio TEXT NOT NULL, bio TEXT NOT NULL,
activity_weight REAL NOT NULL, activity_weight REAL NOT NULL,
email TEXT NOT NULL UNIQUE, email TEXT NOT NULL UNIQUE,
date_of_birth TEXT NOT NULL, date_of_birth TEXT NOT NULL
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
); );
CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id);
)sql"; )sql";
inline constexpr std::string_view kInsertLocationSql = R"sql( inline constexpr std::string_view kCreateUserAddressesTableSql = R"sql(
INSERT INTO locations (
CREATE TABLE IF NOT EXISTS user_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_user_addresses_user_id
ON user_addresses(user_id);
CREATE INDEX IF NOT EXISTS idx_user_addresses_city_id
ON user_addresses(city_id);
)sql";
inline constexpr std::string_view kInsertCitySql = R"sql(
INSERT INTO cities (
city, city,
state_province, state_province,
iso3166_2, iso3166_2,
country, country,
iso3166_1, iso3166_1,
local_languages_json, local_languages_json,
latitude, postal_code_country_format_regex,
longitude postal_code_city_regex_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?); ) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
)sql"; )sql";
inline constexpr std::string_view kInsertBrewerySql = R"sql( inline constexpr std::string_view kInsertBrewerySql = R"sql(
INSERT INTO breweries ( INSERT INTO breweries (
location_id,
name_en, name_en,
description_en, description_en,
name_local, name_local,
description_local description_local
) VALUES (?, ?, ?, ?, ?); ) VALUES (?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
INSERT INTO brewery_addresses (
brewery_id,
city_id,
postal_code
) VALUES (?, ?, ?);
)sql"; )sql";
inline constexpr std::string_view kInsertUserSql = R"sql( inline constexpr std::string_view kInsertUserSql = R"sql(
INSERT INTO users ( INSERT INTO users (
location_id,
first_name, first_name,
last_name, last_name,
gender, gender,
@@ -104,33 +138,45 @@ INSERT INTO users (
activity_weight, activity_weight,
email, email,
date_of_birth date_of_birth
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); ) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertUserAddressSql = R"sql(
INSERT INTO user_addresses (
user_id,
city_id,
postal_code
) VALUES (?, ?, ?);
)sql"; )sql";
// sqlite3_bind_*() parameter indices are 1-based, matching the "?" // sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above. // placeholder order in the SQL above.
enum LocationBindIndex { enum CityBindIndex {
kLocationCityBindIndex = 1, kCityCityBindIndex = 1,
kLocationStateProvinceBindIndex, kCityStateProvinceBindIndex,
kLocationIso31662BindIndex, kCityIso31662BindIndex,
kLocationCountryBindIndex, kCityCountryBindIndex,
kLocationIso31661BindIndex, kCityIso31661BindIndex,
kLocationLanguagesBindIndex, kCityLanguagesBindIndex,
kLocationLatitudeBindIndex, kCityPostalCodeCountryFormatRegexBindIndex,
kLocationLongitudeBindIndex, kCityPostalCodeCityRegexJsonBindIndex,
}; };
enum BreweryBindIndex { enum BreweryBindIndex {
kBreweryLocationIdBindIndex = 1, kBreweryEnglishNameBindIndex = 1,
kBreweryEnglishNameBindIndex,
kBreweryEnglishDescriptionBindIndex, kBreweryEnglishDescriptionBindIndex,
kBreweryLocalNameBindIndex, kBreweryLocalNameBindIndex,
kBreweryLocalDescriptionBindIndex, kBreweryLocalDescriptionBindIndex,
}; };
enum BreweryAddressBindIndex {
kBreweryAddressBreweryIdBindIndex = 1,
kBreweryAddressCityIdBindIndex,
kBreweryAddressPostalCodeBindIndex,
};
enum UserBindIndex { enum UserBindIndex {
kUserLocationIdBindIndex = 1, kUserFirstNameBindIndex = 1,
kUserFirstNameBindIndex,
kUserLastNameBindIndex, kUserLastNameBindIndex,
kUserGenderBindIndex, kUserGenderBindIndex,
kUserUsernameBindIndex, kUserUsernameBindIndex,
@@ -140,6 +186,12 @@ enum UserBindIndex {
kUserDateOfBirthBindIndex, kUserDateOfBirthBindIndex,
}; };
enum UserAddressBindIndex {
kUserAddressUserIdBindIndex = 1,
kUserAddressCityIdBindIndex,
kUserAddressPostalCodeBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql, std::string_view sql,
const char* action); const char* action);

View File

@@ -20,10 +20,10 @@ class IEnrichmentService {
/** /**
* @brief Resolves contextual enrichment for a location. * @brief Resolves contextual enrichment for a location.
* *
* @param loc Location to enrich. * @param loc City to enrich.
* @return Context text, or an empty string if unavailable. * @return Context text, or an empty string if unavailable.
*/ */
virtual std::string GetLocationContext(const Location& loc) = 0; virtual std::string GetLocationContext(const City& loc) = 0;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_

View File

@@ -15,7 +15,7 @@
*/ */
class MockEnrichmentService final : public IEnrichmentService { class MockEnrichmentService final : public IEnrichmentService {
public: public:
std::string GetLocationContext(const Location& /*loc*/) override { std::string GetLocationContext(const City& /*loc*/) override {
return {}; return {};
} }
}; };

View File

@@ -29,7 +29,7 @@ class WikipediaEnrichmentService final : public IEnrichmentService {
/** /**
* @brief Returns the Wikipedia-derived context for a location. * @brief Returns the Wikipedia-derived context for a location.
*/ */
[[nodiscard]] std::string GetLocationContext(const Location& loc) override; [[nodiscard]] std::string GetLocationContext(const City& loc) override;
private: private:
std::string FetchExtract(std::string_view query); std::string FetchExtract(std::string_view query);

View File

@@ -51,7 +51,7 @@ enum class PipelinePhase {
*/ */
Startup, Startup,
/** /**
* @brief Location/context enrichment (e.g. Wikipedia lookups). * @brief City/context enrichment (e.g. Wikipedia lookups).
*/ */
Enrichment, Enrichment,
/** /**

View File

@@ -0,0 +1,79 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/mock_postal_code_service.h
* @brief Data-driven postal code service that selects the first curated
* example for a city and validates it against the city's postal-code regex.
*/
#include <format>
#include <regex>
#include <stdexcept>
#include <string>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service backed entirely by the curated locations
* dataset: it returns the city's first example postal code after checking it
* against the curated postal-code regex.
*
* A precise city regex is preferred when present; otherwise the permissive
* country format regex is used as the fallback. A city with no examples, or an
* example that fails validation, throws so the caller can skip the city.
*/
class MockPostalCodeService final : public IPostalCodeService {
public:
std::string GeneratePostalCode(const City& city) override {
const PostalCodeSpec& spec = city.postal_code;
if (spec.examples.empty()) {
throw std::runtime_error(std::format(
"No postal-code examples available for city '{}'", city.city));
}
const std::string& candidate = spec.examples.front();
if (!MatchesAnyPattern(candidate, spec)) {
throw std::runtime_error(
std::format("Postal code '{}' for city '{}' does not match the "
"curated postal-code regex",
candidate, city.city));
}
return candidate;
}
private:
/**
* @brief Returns true if @p candidate matches any city regex, or the
* country format regex when no city regex matches. When the spec carries no
* usable regex at all, validation is skipped (returns true).
*/
static bool MatchesAnyPattern(const std::string& candidate,
const PostalCodeSpec& spec) {
bool has_pattern = false;
for (const std::string& pattern : spec.city_regexes) {
if (pattern.empty()) {
continue;
}
has_pattern = true;
if (std::regex_match(candidate, std::regex(pattern))) {
return true;
}
}
if (!spec.country_format_regex.empty()) {
has_pattern = true;
if (std::regex_match(candidate, std::regex(spec.country_format_regex))) {
return true;
}
}
return !has_pattern;
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_

View File

@@ -0,0 +1,35 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/postal_code_service.h
* @brief Abstraction for resolving a postal code for a city.
*/
#include <string>
#include "data_model/models.h"
/**
* @brief Interface for services that resolve a postal code for a city.
*/
class IPostalCodeService {
public:
IPostalCodeService() = default;
virtual ~IPostalCodeService() = default;
IPostalCodeService(const IPostalCodeService&) = delete;
IPostalCodeService& operator=(const IPostalCodeService&) = delete;
IPostalCodeService(IPostalCodeService&&) = delete;
IPostalCodeService& operator=(IPostalCodeService&&) = delete;
/**
* @brief Resolves a postal code for @p city.
*
* @param city City whose curated postal-code examples supply the result.
* @return A postal code string, or an empty string if none is available.
*/
virtual std::string GeneratePostalCode(const City& city) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_

File diff suppressed because it is too large Load Diff

View File

@@ -13,10 +13,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<DataGenerator> generator, std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options) const ApplicationOptions& application_options)
: logger_(std::move(logger)), : logger_(std::move(logger)),
context_service_(std::move(context_service)), context_service_(std::move(context_service)),
generator_(std::move(generator)), generator_(std::move(generator)),
exporter_(std::move(exporter)), exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)), curated_data_service_(std::move(curated_data_service)),
postal_code_service_(std::move(postal_code_service)),
application_options_(application_options) {} application_options_(application_options) {}

View File

@@ -22,12 +22,15 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
const auto generate_record = const auto generate_record =
[this, &skipped_count]( [this, &skipped_count](
const Location& location, const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
const std::string& region_context) -> std::optional<BreweryRecord> {
try { try {
const BreweryResult brewery = const BreweryResult brewery = generator_->GenerateBrewery(enriched_city);
generator_->GenerateBrewery(location, region_context); const std::string postal_code =
return BreweryRecord{.location = location, .brewery = brewery}; postal_code_service_->GeneratePostalCode(enriched_city.location);
return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code},
.brewery = brewery};
} catch (const std::exception& e) { } catch (const std::exception& e) {
++skipped_count; ++skipped_count;
@@ -36,7 +39,8 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
.phase = PipelinePhase::BreweryAndBeerGeneration, .phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery " .message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
"generation failed: {}", "generation failed: {}",
location.city, location.country, e.what())}); enriched_city.location.city,
enriched_city.location.country, e.what())});
return std::nullopt; return std::nullopt;
} }
}; };
@@ -47,19 +51,18 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
exporter_->ProcessRecord(record); exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) { } catch (const std::exception& export_exception) {
++export_failed_count; ++export_failed_count;
logger_->Log( logger_->Log({.level = LogLevel::Warn,
{.level = LogLevel::Warn, .phase = PipelinePhase::BreweryAndBeerGeneration,
.phase = PipelinePhase::BreweryAndBeerGeneration, .message = std::format(
.message = std::format("[Pipeline] Generated brewery for '{}' ({}) " "[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}", "but SQLite export failed: {}",
record.location.city, record.location.country, record.address.city.city, record.address.city.country,
export_exception.what())}); export_exception.what())});
} }
}; };
for (const auto& [location, region_context] : cities) { for (const EnrichedCity& enriched_city : cities) {
const std::optional<BreweryRecord> record = const std::optional<BreweryRecord> record = generate_record(enriched_city);
generate_record(location, region_context);
if (!record.has_value()) { if (!record.has_value()) {
continue; continue;
} }

View File

@@ -135,9 +135,12 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
try { try {
const UserResult user = const UserResult user =
generator_->GenerateUser(city, persona, sampled_name); generator_->GenerateUser(city, persona, sampled_name);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(city.location);
return UserRecord{ return UserRecord{
.location = city.location, .address =
UserAddress{.city = city.location, .postal_code = postal_code},
.user = user, .user = user,
.email = BuildEmail(sampled_name, used_email_local_parts), .email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng), .date_of_birth = GenerateDateOfBirth(rng),
@@ -161,13 +164,13 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
} catch (const std::exception& export_exception) { } catch (const std::exception& export_exception) {
++export_failed_count; ++export_failed_count;
logger_->Log( logger_->Log({.level = LogLevel::Warn,
{.level = LogLevel::Warn, .phase = PipelinePhase::UserGeneration,
.phase = PipelinePhase::UserGeneration, .message = std::format(
.message = std::format("[Pipeline] Generated user for '{}' ({}) but " "[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}", "SQLite export failed: {}",
record.location.city, record.location.country, record.address.city.city, record.address.city.country,
export_exception.what())}); export_exception.what())});
} }
}; };

View File

@@ -14,20 +14,20 @@
void BiergartenPipelineOrchestrator::LogResults() const { void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array brewery_output; boost::json::array brewery_output;
for (const auto& [location, brewery] : generated_breweries_) { for (const auto& [address, brewery] : generated_breweries_) {
const City& city = address.city;
brewery_output.push_back(boost::json::object{ brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en}, {"name_en", brewery.name_en},
{"description_en", brewery.description_en}, {"description_en", brewery.description_en},
{"name_local", brewery.name_local}, {"name_local", brewery.name_local},
{"description_local", brewery.description_local}, {"description_local", brewery.description_local},
{"location", boost::json::object{ {"address", boost::json::object{
{"city", location.city}, {"city", city.city},
{"country", location.country}, {"country", city.country},
{"state_province", location.state_province}, {"state_province", city.state_province},
{"iso3166_2", location.iso3166_2}, {"iso3166_2", city.iso3166_2},
{"latitude", location.latitude}, {"postal_code", address.postal_code},
{"longitude", location.longitude}, }}});
}}});
} }
std::ostringstream brewery_oss; std::ostringstream brewery_oss;

View File

@@ -14,13 +14,13 @@
#include "services/curated_data/curated_json_data_service.h" #include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
std::vector<Location> std::vector<City>
BiergartenPipelineOrchestrator::QueryLocations() { BiergartenPipelineOrchestrator::QueryLocations() {
logger_->Log({.level = LogLevel::Info, logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="}); .message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
const std::vector<Location>& all_locations = const std::vector<City>& all_locations =
curated_data_service_->LoadLocations(); curated_data_service_->LoadLocations();
const size_t sample_count = std::min( const size_t sample_count = std::min(
@@ -31,7 +31,7 @@ BiergartenPipelineOrchestrator::QueryLocations() {
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>( static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count); sample_count);
std::vector<Location> sampled_locations; std::vector<City> sampled_locations;
sampled_locations.reserve(sample_count); sampled_locations.reserve(sample_count);
std::random_device random_generator; std::random_device random_generator;

View File

@@ -14,7 +14,7 @@ bool BiergartenPipelineOrchestrator::Run() {
try { try {
exporter_->Initialize(); exporter_->Initialize();
std::vector<Location> cities = QueryLocations(); std::vector<City> cities = QueryLocations();
std::vector<EnrichedCity> enriched; std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size()); enriched.reserve(cities.size());

View File

@@ -36,7 +36,9 @@ static std::string FormatLocalLanguageCodes(
static constexpr int kBreweryInitialMaxTokens = 2800; static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery( BreweryResult LlamaGenerator::GenerateBrewery(
const Location& location, const std::string& region_context) { const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const std::string& region_context = enriched_city.region_context;
/** /**
* Preprocess and truncate region context to manageable size * Preprocess and truncate region context to manageable size
*/ */
@@ -66,7 +68,7 @@ BreweryResult LlamaGenerator::GenerateBrewery(
* Store location context for retry prompts (without repeating full context) * Store location context for retry prompts (without repeating full context)
*/ */
const std::string retry_location = const std::string retry_location =
std::format("Location: {}{}\nLocal language codes: {}", location.city, std::format("City: {}{}\nLocal language codes: {}", location.city,
country_suffix, local_language_codes); country_suffix, local_language_codes);
/** /**

View File

@@ -8,14 +8,14 @@
#include "data_generation/mock_generator.h" #include "data_generation/mock_generator.h"
size_t MockGenerator::DeterministicHash(const Location& location) { size_t MockGenerator::DeterministicHash(const City& location) {
size_t seed = 0; size_t seed = 0;
boost::hash_combine(seed, location.city); boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country); boost::hash_combine(seed, location.country);
return seed; return seed;
} }
size_t MockGenerator::DeterministicHash(const Location& location, size_t MockGenerator::DeterministicHash(const City& location,
const UserPersona& persona, const UserPersona& persona,
const Name& name) { const Name& name) {
size_t seed = DeterministicHash(location); size_t seed = DeterministicHash(location);

View File

@@ -11,7 +11,8 @@
#include "data_generation/mock_generator.h" #include "data_generation/mock_generator.h"
BreweryResult MockGenerator::GenerateBrewery( BreweryResult MockGenerator::GenerateBrewery(
const Location& location, const std::string& /*region_context*/) { const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const size_t hash = DeterministicHash(location); const size_t hash = DeterministicHash(location);
const std::string_view adjective = const std::string_view adjective =

View File

@@ -86,6 +86,7 @@ int main(const int argc, char** argv) {
di::bind<ApplicationOptions>().to(options), di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path), di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(), di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPostalCodeService>().to<MockPostalCodeService>(),
di::bind<ICuratedDataService>().to( di::bind<ICuratedDataService>().to(
[options, &log_producer]() -> std::unique_ptr<ICuratedDataService> { [options, &log_producer]() -> std::unique_ptr<ICuratedDataService> {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {

View File

@@ -1,6 +1,6 @@
/** /**
* @file json_handling/json_loader.cc * @file json_handling/json_loader.cc
* @brief Parses curated location JSON input into strongly typed Location * @brief Parses curated location JSON input into strongly typed City
* records with strict field validation and descriptive error reporting. * records with strict field validation and descriptive error reporting.
*/ */
@@ -30,16 +30,6 @@ static std::string ReadRequiredString(const boost::json::object& object,
return std::string(text); return std::string(text);
} }
static double ReadRequiredNumber(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_number()) {
throw std::runtime_error(
std::format("Missing or invalid numeric field: {}", key));
}
return value->to_number<double>();
}
static std::vector<std::string> ReadRequiredStringArray( static std::vector<std::string> ReadRequiredStringArray(
const boost::json::object& object, const char* key) { const boost::json::object& object, const char* key) {
const boost::json::value* value = object.if_contains(key); const boost::json::value* value = object.if_contains(key);
@@ -61,6 +51,21 @@ static std::vector<std::string> ReadRequiredStringArray(
return items; return items;
} }
static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
const boost::json::value* value = object.if_contains("postal_code");
if (value == nullptr || !value->is_object()) {
throw std::runtime_error("Missing or invalid object field: postal_code");
}
const auto& postal_code = value->as_object();
return PostalCodeSpec{
.country_format_regex =
ReadRequiredString(postal_code, "country_format_regex"),
.city_regexes = ReadRequiredStringArray(postal_code, "city_regex"),
.examples = ReadRequiredStringArray(postal_code, "examples"),
};
}
namespace { namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath, boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
@@ -133,15 +138,14 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
} }
const auto& object = item.as_object(); const auto& object = item.as_object();
locations.push_back(Location{ locations.push_back(City{
.city = ReadRequiredString(object, "city"), .city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"), .state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"), .iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"), .country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"), .iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"), .local_languages = ReadRequiredStringArray(object, "local_languages"),
.latitude = ReadRequiredNumber(object, "latitude"), .postal_code = ReadPostalCodeSpec(object),
.longitude = ReadRequiredNumber(object, "longitude"),
}); });
} }
cache_.locations = std::move(locations); cache_.locations = std::move(locations);

View File

@@ -7,38 +7,46 @@
MockCuratedDataService::MockCuratedDataService() MockCuratedDataService::MockCuratedDataService()
: locations_{ : locations_{
Location{.city = "Portland", City{.city = "Portland",
.state_province = "Oregon", .state_province = "Oregon",
.iso3166_2 = "US-OR", .iso3166_2 = "US-OR",
.country = "United States", .country = "United States",
.iso3166_1 = "US", .iso3166_1 = "US",
.local_languages = {"en"}, .local_languages = {"en"},
.latitude = 45.5152, .postal_code =
.longitude = -122.6784}, PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
Location{.city = "Munich", .city_regexes = {R"(^972\d{2}$)"},
.state_province = "Bavaria", .examples = {"97201", "97202", "97209"}}},
.iso3166_2 = "DE-BY", City{.city = "Munich",
.country = "Germany", .state_province = "Bavaria",
.iso3166_1 = "DE", .iso3166_2 = "DE-BY",
.local_languages = {"de"}, .country = "Germany",
.latitude = 48.1351, .iso3166_1 = "DE",
.longitude = 11.5820}, .local_languages = {"de"},
Location{.city = "Lyon", .postal_code =
.state_province = "Auvergne-Rhone-Alpes", PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.iso3166_2 = "FR-ARA", .city_regexes = {R"(^8\d{4}$)"},
.country = "France", .examples = {"80331", "80333", "81667"}}},
.iso3166_1 = "FR", City{.city = "Lyon",
.local_languages = {"fr"}, .state_province = "Auvergne-Rhone-Alpes",
.latitude = 45.7640, .iso3166_2 = "FR-ARA",
.longitude = 4.8357}, .country = "France",
Location{.city = "Brussels", .iso3166_1 = "FR",
.state_province = "Brussels-Capital", .local_languages = {"fr"},
.iso3166_2 = "BE-BRU", .postal_code =
.country = "Belgium", PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.iso3166_1 = "BE", .city_regexes = {R"(^690\d{2}$)"},
.local_languages = {"nl", "fr"}, .examples = {"69001", "69002", "69007"}}},
.latitude = 50.8503, City{.city = "Brussels",
.longitude = 4.3517}, .state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU",
.country = "Belgium",
.iso3166_1 = "BE",
.local_languages = {"nl", "fr"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{4}$)",
.city_regexes = {R"(^1\d{3}$)"},
.examples = {"1000", "1050", "1180"}}},
}, },
personas_{ personas_{
UserPersona{.name = "Hophead Explorer", UserPersona{.name = "Hophead Explorer",

View File

@@ -11,7 +11,7 @@
#include "services/enrichment/wikipedia_service.h" #include "services/enrichment/wikipedia_service.h"
std::string WikipediaEnrichmentService::GetLocationContext( std::string WikipediaEnrichmentService::GetLocationContext(
const Location& loc) { const City& loc) {
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
if (!this->client_) { if (!this->client_) {
if (logger_) { if (logger_) {

View File

@@ -14,9 +14,11 @@ void SqliteExportService::Finalize() {
} }
try { try {
insert_user_address_stmt_.reset();
insert_user_stmt_.reset(); insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset(); insert_brewery_stmt_.reset();
insert_location_stmt_.reset(); insert_city_stmt_.reset();
if (transaction_open_) { if (transaction_open_) {
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction"); db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
@@ -24,7 +26,7 @@ void SqliteExportService::Finalize() {
} }
db_handle_.reset(); db_handle_.reset();
location_cache_.clear(); city_cache_.clear();
} catch (...) { } catch (...) {
RollbackAndCloseNoThrow(); RollbackAndCloseNoThrow();
throw; throw;

View File

@@ -88,7 +88,8 @@ sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) {
} }
std::string SerializeVector(const std::vector<std::string>& str_vec) { std::string SerializeVector(const std::vector<std::string>& str_vec) {
boost::json::array array(str_vec.size()); boost::json::array array;
array.reserve(str_vec.size());
for (const auto& s : str_vec) { for (const auto& s : str_vec) {
array.emplace_back(s); array.emplace_back(s);
} }

View File

@@ -28,26 +28,40 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
void SqliteExportService::InitializeSchema() const { void SqliteExportService::InitializeSchema() const {
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql, db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql,
"Failed to create SQLite locations table"); "Failed to create SQLite cities table");
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql, db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
"Failed to create SQLite breweries table"); "Failed to create SQLite breweries table");
sqlite_export_service_internal::ExecSql(
db_handle_,
sqlite_export_service_internal::kCreateBreweryAddressesTableSql,
"Failed to create SQLite brewery_addresses table");
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql, db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
"Failed to create SQLite users table"); "Failed to create SQLite users table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUserAddressesTableSql,
"Failed to create SQLite user_addresses table");
} }
void SqliteExportService::PrepareStatements() { void SqliteExportService::PrepareStatements() {
insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement( insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertLocationSql, db_handle_, sqlite_export_service_internal::kInsertCitySql,
"Failed to prepare SQLite location insert statement"); "Failed to prepare SQLite city insert statement");
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement( insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBrewerySql, db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
"Failed to prepare SQLite brewery insert statement"); "Failed to prepare SQLite brewery insert statement");
insert_brewery_address_stmt_ =
sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql,
"Failed to prepare SQLite brewery address insert statement");
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement( insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserSql, db_handle_, sqlite_export_service_internal::kInsertUserSql,
"Failed to prepare SQLite user insert statement"); "Failed to prepare SQLite user insert statement");
insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserAddressSql,
"Failed to prepare SQLite user address insert statement");
} }
void SqliteExportService::RollbackAndCloseNoThrow() noexcept { void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
@@ -60,11 +74,13 @@ void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
transaction_open_ = false; transaction_open_ = false;
} }
insert_user_address_stmt_.reset();
insert_user_stmt_.reset(); insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset(); insert_brewery_stmt_.reset();
insert_location_stmt_.reset(); insert_city_stmt_.reset();
db_handle_.reset(); db_handle_.reset();
location_cache_.clear(); city_cache_.clear();
} }
void SqliteExportService::Initialize() { void SqliteExportService::Initialize() {

View File

@@ -1,10 +1,9 @@
/** /**
* @file services/sqlite/process_record.cc * @file services/sqlite/process_record.cc
* @brief SqliteExportService::ProcessRecord() implementation * @brief SqliteExportService::ProcessRecord() implementation
* and the shared location-resolution helper. * and the shared city-resolution helper.
*/ */
#include <iomanip>
#include <sstream> #include <sstream>
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
@@ -12,94 +11,91 @@
#include "services/database/sqlite_export_service.h" #include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h" #include "services/database/sqlite_export_service_helpers.h"
constexpr int kLocationPrecision = 17; std::string SqliteExportService::BuildCityKey(const City& city) {
std::string SqliteExportService::BuildLocationKey(const Location& location) {
std::ostringstream key_stream; std::ostringstream key_stream;
key_stream << location.city << '\n' key_stream << city.city << '\n'
<< location.state_province << '\n' << city.state_province << '\n'
<< location.iso3166_2 << '\n' << city.iso3166_2 << '\n'
<< location.country << '\n' << city.country << '\n'
<< location.iso3166_1 << '\n' << city.iso3166_1 << '\n'
<< std::setprecision(kLocationPrecision) << location.latitude
<< '\n'
<< std::setprecision(kLocationPrecision) << location.longitude
<< '\n'
<< sqlite_export_service_internal::SerializeVector( << sqlite_export_service_internal::SerializeVector(
location.local_languages); city.local_languages);
return key_stream.str(); return key_stream.str();
} }
sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) { sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) {
const std::string location_key = BuildLocationKey(location); const std::string city_key = BuildCityKey(city);
const auto cached_location = location_cache_.find(location_key); const auto cached_city = city_cache_.find(city_key);
if (cached_location != location_cache_.end()) { if (cached_city != city_cache_.end()) {
return cached_location->second; return cached_city->second;
} }
const std::string local_languages_json = const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(location.local_languages); sqlite_export_service_internal::SerializeVector(city.local_languages);
const std::string city_regex_json =
sqlite_export_service_internal::SerializeVector(
city.postal_code.city_regexes);
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCityBindIndex, .index = sqlite_export_service_internal::kCityCityBindIndex,
.value = location.city, .value = city.city,
.action = "Failed to bind SQLite location city"}); .action = "Failed to bind SQLite city name"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = .index = sqlite_export_service_internal::kCityStateProvinceBindIndex,
sqlite_export_service_internal::kLocationStateProvinceBindIndex, .value = city.state_province,
.value = location.state_province, .action = "Failed to bind SQLite city state/province"});
.action = "Failed to bind SQLite location state/province"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31662BindIndex, .index = sqlite_export_service_internal::kCityIso31662BindIndex,
.value = location.iso3166_2, .value = city.iso3166_2,
.action = "Failed to bind SQLite location ISO 3166-2 code"}); .action = "Failed to bind SQLite city ISO 3166-2 code"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCountryBindIndex, .index = sqlite_export_service_internal::kCityCountryBindIndex,
.value = location.country, .value = city.country,
.action = "Failed to bind SQLite location country"}); .action = "Failed to bind SQLite city country"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31661BindIndex, .index = sqlite_export_service_internal::kCityIso31661BindIndex,
.value = location.iso3166_1, .value = city.iso3166_1,
.action = "Failed to bind SQLite location ISO 3166-1 code"}); .action = "Failed to bind SQLite city ISO 3166-1 code"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationLanguagesBindIndex, .index = sqlite_export_service_internal::kCityLanguagesBindIndex,
.value = local_languages_json, .value = local_languages_json,
.action = "Failed to bind SQLite location languages"}); .action = "Failed to bind SQLite city languages"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationLatitudeBindIndex, .index = sqlite_export_service_internal::
.value = location.latitude, kCityPostalCodeCountryFormatRegexBindIndex,
.action = "Failed to bind SQLite location latitude"}); .value = city.postal_code.country_format_regex,
.action = "Failed to bind SQLite city postal-code format regex"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_location_stmt_, insert_city_stmt_,
sqlite_export_service_internal::BoundParam{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationLongitudeBindIndex, .index = sqlite_export_service_internal::
.value = location.longitude, kCityPostalCodeCityRegexJsonBindIndex,
.action = "Failed to bind SQLite location longitude"}); .value = city_regex_json,
.action = "Failed to bind SQLite city postal-code city regexes"});
sqlite_export_service_internal::StepStatement( sqlite_export_service_internal::StepStatement(
db_handle_, insert_location_stmt_, db_handle_, insert_city_stmt_, "Failed to insert SQLite city row");
"Failed to insert SQLite location row");
const sqlite3_int64 location_id = const sqlite3_int64 city_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_); sqlite_export_service_internal::LastInsertRowId(db_handle_);
location_cache_.emplace(location_key, location_id); city_cache_.emplace(city_key, city_id);
sqlite_export_service_internal::ResetStatement(insert_location_stmt_); sqlite_export_service_internal::ResetStatement(insert_city_stmt_);
return location_id; return city_id;
} }
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) { uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
@@ -107,14 +103,7 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
throw std::runtime_error("SQLite export service is not initialized"); throw std::runtime_error("SQLite export service is not initialized");
} }
const sqlite3_int64 location_id = ResolveLocationId(brewery.location); const sqlite3_int64 city_id = ResolveCityId(brewery.address.city);
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kBreweryLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite brewery location id"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_brewery_stmt_, insert_brewery_stmt_,
@@ -149,7 +138,38 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
sqlite_export_service_internal::StepStatement( sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row"); db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
const sqlite3_int64 brewery_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_); sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
return sqlite_export_service_internal::LastInsertRowId(db_handle_); sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressBreweryIdBindIndex,
.value = brewery_id,
.action = "Failed to bind SQLite brewery address brewery id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_address_stmt_,
"Failed to insert SQLite brewery address row");
sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_);
return static_cast<uint64_t>(brewery_id);
} }

View File

@@ -13,14 +13,8 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
throw std::runtime_error("SQLite export service is not initialized"); throw std::runtime_error("SQLite export service is not initialized");
} }
const sqlite3_int64 location_id = ResolveLocationId(user.location); const sqlite3_int64 city_id = ResolveCityId(user.address.city);
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite user location id"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_user_stmt_, insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
@@ -73,7 +67,36 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
sqlite_export_service_internal::StepStatement( sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row"); db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
const sqlite3_int64 user_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_user_stmt_); sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
return sqlite_export_service_internal::LastInsertRowId(db_handle_); sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressUserIdBindIndex,
.value = user_id,
.action = "Failed to bind SQLite user address user id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
.value = user.address.postal_code,
.action = "Failed to bind SQLite user address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_,
"Failed to insert SQLite user address row");
sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_);
return static_cast<uint64_t>(user_id);
} }