AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 17:47:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pipeline: rename Location to City, wire postal code into breweries

Browse Source
This commit is contained in:
Aaron Po
2026-07-13 02:26:44 -04:00
parent fbcf438381
commit d52dba904c
48 changed files with 2261 additions and 1227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -7,16 +7,20 @@
#include <utility>
#include "services/postal_code/postal_code_service.h"
BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::shared_ptr<ILogger> logger,
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)),
postal_code_service_(std::move(postal_code_service)),
application_options_(application_options) {}

8
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc
View File

@@ -22,12 +22,16 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
const auto generate_record =
[this, &skipped_count](
const Location& location,
const City& location,
const std::string& region_context) -> std::optional<BreweryRecord> {
try {
const BreweryResult brewery =
generator_->GenerateBrewery(location, region_context);
return BreweryRecord{.location = location, .brewery = brewery};
const std::string postal_code =
postal_code_service_->GeneratePostalCode(location);
return BreweryRecord{.location = location,
.address = Address{.postal_code = postal_code},
.brewery = brewery};
} catch (const std::exception& e) {
++skipped_count;

3
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -14,12 +14,13 @@
void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array brewery_output;
for (const auto& [location, brewery] : generated_breweries_) {
for (const auto& [location, address, brewery] : generated_breweries_) {
brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"postal_code", address.postal_code},
{"location", boost::json::object{
{"city", location.city},
{"country", location.country},

8
tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
View File

@@ -14,14 +14,14 @@
#include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h"
std::vector<Location>
std::vector<City>
BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
const std::vector<Location>& all_locations =
curated_data_service_->LoadLocations();
const std::vector<City>& all_locations =
curated_data_service_->LoadCities();
const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count),
@@ -31,7 +31,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count);
std::vector<Location> sampled_locations;
std::vector<City> sampled_locations;
sampled_locations.reserve(sample_count);
std::random_device random_generator;

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc
View File

@@ -14,7 +14,7 @@ bool BiergartenPipelineOrchestrator::Run() {
try {
exporter_->Initialize();
std::vector<Location> cities = QueryCitiesWithCountries();
std::vector<City> cities = QueryCitiesWithCountries();
std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size());

2
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -36,7 +36,7 @@ static std::string FormatLocalLanguageCodes(
static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery(
const Location& location, const std::string& region_context) {
const City& location, const std::string& region_context) {
/**
* Preprocess and truncate region context to manageable size
*/

4
tooling/pipeline/src/data_generation/mock/deterministic_hash.cc
View File

@@ -8,14 +8,14 @@
#include "data_generation/mock_generator.h"
size_t MockGenerator::DeterministicHash(const Location& location) {
size_t MockGenerator::DeterministicHash(const City& location) {
size_t seed = 0;
boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country);
return seed;
}
size_t MockGenerator::DeterministicHash(const Location& location,
size_t MockGenerator::DeterministicHash(const City& location,
const UserPersona& persona,
const Name& name) {
size_t seed = DeterministicHash(location);

2
tooling/pipeline/src/data_generation/mock/generate_brewery.cc
View File

@@ -11,7 +11,7 @@
#include "data_generation/mock_generator.h"
BreweryResult MockGenerator::GenerateBrewery(
const Location& location, const std::string& /*region_context*/) {
const City& location, const std::string& /*region_context*/) {
const size_t hash = DeterministicHash(location);
const std::string_view adjective =

6
tooling/pipeline/src/main.cc
View File

@@ -102,12 +102,14 @@ int main(const int argc, char** argv) {
return std::make_unique<CuratedJsonDataService>(
CuratedDataFilePaths{
.locations_path = "locations.json",
.cities_path = "cities.json",
.personas_path = "personas.json",
.forenames_path = "forenames-by-country.json",
.surnames_path = "surnames-by-country.json",
});
}),
di::bind<IPostalCodeService>().to<MockPostalCodeService>(),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
@@ -226,4 +228,4 @@ int main(const int argc, char** argv) {
return shutdown(EXIT_FAILURE);
}
}
}

36
tooling/pipeline/src/services/curated_data/curated_json_data_service.cc
View File

@@ -1,6 +1,6 @@
/**
* @file json_handling/json_loader.cc
* @brief Parses curated location JSON input into strongly typed Location
* @brief Parses curated location JSON input into strongly typed City
* records with strict field validation and descriptive error reporting.
*/
@@ -109,31 +109,38 @@ std::string ReadFirstOfStringArray(const boost::json::object& object,
CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths)
: filepaths_(std::move(filepaths)) {}
const LocationsList& CuratedJsonDataService::LoadLocations() {
const CityList& CuratedJsonDataService::LoadCities() {
if (!cache_.locations.empty()) {
return cache_.locations;
}
const boost::json::value root =
ParseJsonFile(filepaths_.locations_path, "locations");
ParseJsonFile(filepaths_.cities_path, "cities");
if (!root.is_array()) {
throw std::runtime_error(
"Invalid locations JSON: root element must be an array");
"Invalid cities JSON: root element must be an array");
}
LocationsList locations;
CityList locations;
const auto& items = root.as_array();
locations.reserve(items.size());
for (const auto& item : items) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid locations JSON: each entry must be an object");
"Invalid cities JSON: each entry must be an object");
}
const auto& object = item.as_object();
locations.push_back(Location{
const boost::json::value* postal_code = object.if_contains("postal_code");
if (postal_code == nullptr || !postal_code->is_object()) {
throw std::runtime_error(
"Missing or invalid object field: postal_code");
}
locations.push_back(City{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
@@ -142,8 +149,23 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.latitude = ReadRequiredNumber(object, "latitude"),
.longitude = ReadRequiredNumber(object, "longitude"),
.postal_regex =
ReadRequiredStringArray(postal_code->as_object(), "city_regex"),
.postal_code_examples =
ReadRequiredStringArray(postal_code->as_object(), "examples"),
});
}
for (auto location : locations) {
std::cout << "Location: " << location.city << ", "
<< location.state_province << ", " << location.iso3166_2 << ", "
<< location.country << ", " << location.iso3166_1 << ", "
<< location.latitude << ", " << location.longitude << std::endl;
for (const auto& regex : location.postal_regex) {
std::cout << " Postal regex: " << regex << std::endl;
}
}
cache_.locations = std::move(locations);
return cache_.locations;
}

26
tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc
View File

@@ -7,38 +7,46 @@
MockCuratedDataService::MockCuratedDataService()
: locations_{
Location{.city = "Portland",
City{.city = "Portland",
.state_province = "Oregon",
.iso3166_2 = "US-OR",
.country = "United States",
.iso3166_1 = "US",
.local_languages = {"en"},
.latitude = 45.5152,
.longitude = -122.6784},
Location{.city = "Munich",
.longitude = -122.6784,
.postal_regex = {"^972[0-9]{2}(?:-[0-9]{4})?$"},
.postal_code_examples = {"97201", "97294"}},
City{.city = "Munich",
.state_province = "Bavaria",
.iso3166_2 = "DE-BY",
.country = "Germany",
.iso3166_1 = "DE",
.local_languages = {"de"},
.latitude = 48.1351,
.longitude = 11.5820},
Location{.city = "Lyon",
.longitude = 11.5820,
.postal_regex = {"^8[01][0-9]{3}$"},
.postal_code_examples = {"80331", "81929"}},
City{.city = "Lyon",
.state_province = "Auvergne-Rhone-Alpes",
.iso3166_2 = "FR-ARA",
.country = "France",
.iso3166_1 = "FR",
.local_languages = {"fr"},
.latitude = 45.7640,
.longitude = 4.8357},
Location{.city = "Brussels",
.longitude = 4.8357,
.postal_regex = {"^6900[1-9]$"},
.postal_code_examples = {"69001", "69009"}},
City{.city = "Brussels",
.state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU",
.country = "Belgium",
.iso3166_1 = "BE",
.local_languages = {"nl", "fr"},
.latitude = 50.8503,
.longitude = 4.3517},
.longitude = 4.3517,
.postal_regex = {"^1(?:0[0-9]{2}|1[0-9]{2}|20[0-9]|210)$"},
.postal_code_examples = {"1000", "1210"}},
},
personas_{
UserPersona{.name = "Hophead Explorer",
@@ -83,7 +91,7 @@ MockCuratedDataService::MockCuratedDataService()
{"BE", SurnameList{"Peeters", "Janssens"}},
} {}
const LocationsList& MockCuratedDataService::LoadLocations() {
const CityList& MockCuratedDataService::LoadCities() {
return locations_;
}

2
tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc
View File

@@ -11,7 +11,7 @@
#include "services/enrichment/wikipedia_service.h"
std::string WikipediaEnrichmentService::GetLocationContext(
const Location& loc) {
const City& loc) {
using namespace std::literals::chrono_literals;
if (!this->client_) {
if (logger_) {

4
tooling/pipeline/src/services/sqlite/finalize.cc
View File

@@ -16,7 +16,7 @@ void SqliteExportService::Finalize() {
try {
insert_user_stmt_.reset();
insert_brewery_stmt_.reset();
insert_location_stmt_.reset();
insert_city_stmt_.reset();
if (transaction_open_) {
sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
@@ -24,7 +24,7 @@ void SqliteExportService::Finalize() {
}
db_handle_.reset();
location_cache_.clear();
city_cache_.clear();
} catch (...) {
RollbackAndCloseNoThrow();
throw;

14
tooling/pipeline/src/services/sqlite/initialize.cc
View File

@@ -28,8 +28,8 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
void SqliteExportService::InitializeSchema() const {
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql,
"Failed to create SQLite locations table");
db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql,
"Failed to create SQLite cities table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
"Failed to create SQLite breweries table");
@@ -39,9 +39,9 @@ void SqliteExportService::InitializeSchema() const {
}
void SqliteExportService::PrepareStatements() {
insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertLocationSql,
"Failed to prepare SQLite location insert statement");
insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertCitySql,
"Failed to prepare SQLite city insert statement");
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
"Failed to prepare SQLite brewery insert statement");
@@ -62,9 +62,9 @@ void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
insert_user_stmt_.reset();
insert_brewery_stmt_.reset();
insert_location_stmt_.reset();
insert_city_stmt_.reset();
db_handle_.reset();
location_cache_.clear();
city_cache_.clear();
}
void SqliteExportService::Initialize() {

89
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -1,7 +1,7 @@
/**
* @file services/sqlite/process_record.cc
* @brief SqliteExportService::ProcessRecord() implementation
* and the shared location-resolution helper.
* and the shared city-resolution helper.
*/
#include <iomanip>
@@ -14,7 +14,7 @@
constexpr int kLocationPrecision = 17;
std::string SqliteExportService::BuildLocationKey(const Location& location) {
std::string SqliteExportService::BuildCityKey(const City& location) {
std::ostringstream key_stream;
key_stream << location.city << '\n'
<< location.state_province << '\n'
@@ -30,76 +30,76 @@ std::string SqliteExportService::BuildLocationKey(const Location& location) {
return key_stream.str();
}
sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
const std::string location_key = BuildLocationKey(location);
const auto cached_location = location_cache_.find(location_key);
if (cached_location != location_cache_.end()) {
return cached_location->second;
sqlite3_int64 SqliteExportService::ResolveCityId(const City& location) {
const std::string city_key = BuildCityKey(location);
const auto cached_city = city_cache_.find(city_key);
if (cached_city != city_cache_.end()) {
return cached_city->second;
}
const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(location.local_languages);
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCityBindIndex,
.index = sqlite_export_service_internal::kCityNameBindIndex,
.value = location.city,
.action = "Failed to bind SQLite location city"});
.action = "Failed to bind SQLite city name"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kLocationStateProvinceBindIndex,
sqlite_export_service_internal::kCityStateProvinceBindIndex,
.value = location.state_province,
.action = "Failed to bind SQLite location state/province"});
.action = "Failed to bind SQLite city state/province"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31662BindIndex,
.index = sqlite_export_service_internal::kCityIso31662BindIndex,
.value = location.iso3166_2,
.action = "Failed to bind SQLite location ISO 3166-2 code"});
.action = "Failed to bind SQLite city ISO 3166-2 code"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationCountryBindIndex,
.index = sqlite_export_service_internal::kCityCountryBindIndex,
.value = location.country,
.action = "Failed to bind SQLite location country"});
.action = "Failed to bind SQLite city country"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationIso31661BindIndex,
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
.value = location.iso3166_1,
.action = "Failed to bind SQLite location ISO 3166-1 code"});
.action = "Failed to bind SQLite city ISO 3166-1 code"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kLocationLanguagesBindIndex,
.index = sqlite_export_service_internal::kCityLanguagesBindIndex,
.value = local_languages_json,
.action = "Failed to bind SQLite location languages"});
.action = "Failed to bind SQLite city languages"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLatitudeBindIndex,
.index = sqlite_export_service_internal::kCityLatitudeBindIndex,
.value = location.latitude,
.action = "Failed to bind SQLite location latitude"});
.action = "Failed to bind SQLite city latitude"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
insert_city_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLongitudeBindIndex,
.index = sqlite_export_service_internal::kCityLongitudeBindIndex,
.value = location.longitude,
.action = "Failed to bind SQLite location longitude"});
.action = "Failed to bind SQLite city longitude"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_location_stmt_,
"Failed to insert SQLite location row");
db_handle_, insert_city_stmt_,
"Failed to insert SQLite city row");
const sqlite3_int64 location_id =
const sqlite3_int64 city_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
location_cache_.emplace(location_key, location_id);
sqlite_export_service_internal::ResetStatement(insert_location_stmt_);
city_cache_.emplace(city_key, city_id);
sqlite_export_service_internal::ResetStatement(insert_city_stmt_);
return location_id;
return city_id;
}
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
@@ -107,14 +107,14 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 location_id = ResolveLocationId(brewery.location);
const sqlite3_int64 city_id = ResolveCityId(brewery.location);
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kBreweryLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite brewery location id"});
.index = sqlite_export_service_internal::kBreweryCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery city id"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
@@ -146,6 +146,13 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
.value = brewery.brewery.description_local,
.action = "Failed to bind SQLite brewery local description"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kBreweryPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");

8
tooling/pipeline/src/services/sqlite/process_user_record.cc
View File

@@ -13,14 +13,14 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 location_id = ResolveLocationId(user.location);
const sqlite3_int64 city_id = ResolveCityId(user.location);
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite user location id"});
.index = sqlite_export_service_internal::kUserCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite user city id"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{