diff --git a/tooling/pipeline/includes/data_model/generated_models.h b/tooling/pipeline/includes/data_model/generated_models.h index c2e62377..89b2cc9d 100644 --- a/tooling/pipeline/includes/data_model/generated_models.h +++ b/tooling/pipeline/includes/data_model/generated_models.h @@ -94,6 +94,8 @@ struct EnrichedCity { struct BreweryAddress { City city; std::string postal_code{}; + double longitude{}; + double latitude{}; }; /** diff --git a/tooling/pipeline/includes/data_model/models.h b/tooling/pipeline/includes/data_model/models.h index 3705e134..478804e5 100644 --- a/tooling/pipeline/includes/data_model/models.h +++ b/tooling/pipeline/includes/data_model/models.h @@ -66,6 +66,9 @@ struct City { */ std::string iso3166_1{}; + double longitude{}; + double latitude{}; + /** * @brief Local language codes in priority order. */ diff --git a/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h b/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h index fde53882..8b7a2507 100644 --- a/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h +++ b/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h @@ -26,6 +26,8 @@ CREATE TABLE IF NOT EXISTS cities ( iso3166_2 TEXT NOT NULL, country TEXT NOT NULL, iso3166_1 TEXT NOT NULL, + longitude REAL NOT NULL, + latitude REAL NOT NULL, local_languages_json TEXT NOT NULL, postal_code_country_format_regex TEXT NOT NULL, postal_code_city_regex_json TEXT NOT NULL, @@ -53,6 +55,8 @@ CREATE TABLE IF NOT EXISTS brewery_addresses ( brewery_id INTEGER NOT NULL, city_id INTEGER NOT NULL, postal_code TEXT NOT NULL, + longitude REAL NOT NULL, + latitude REAL NOT NULL, FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE, FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE ); @@ -105,10 +109,12 @@ INSERT INTO cities ( iso3166_2, country, iso3166_1, + longitude, + latitude, local_languages_json, postal_code_country_format_regex, postal_code_city_regex_json -) VALUES (?, ?, ?, ?, ?, ?, ?, ?); +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?); )sql"; inline constexpr std::string_view kInsertBrewerySql = R"sql( @@ -124,8 +130,10 @@ inline constexpr std::string_view kInsertBreweryAddressSql = R"sql( INSERT INTO brewery_addresses ( brewery_id, city_id, - postal_code -) VALUES (?, ?, ?); + postal_code, + longitude, + latitude +) VALUES (?, ?, ?, ?, ?); )sql"; inline constexpr std::string_view kInsertUserSql = R"sql( @@ -157,6 +165,8 @@ enum CityBindIndex { kCityIso31662BindIndex, kCityCountryBindIndex, kCityIso31661BindIndex, + kCityLongitudeBindIndex, + kCityLatitudeBindIndex, kCityLanguagesBindIndex, kCityPostalCodeCountryFormatRegexBindIndex, kCityPostalCodeCityRegexJsonBindIndex, @@ -173,6 +183,8 @@ enum BreweryAddressBindIndex { kBreweryAddressBreweryIdBindIndex = 1, kBreweryAddressCityIdBindIndex, kBreweryAddressPostalCodeBindIndex, + kBreweryAddressLongitudeBindIndex, + kBreweryAddressLatitudeBindIndex, }; enum UserBindIndex { diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc index d2e273b1..3a5d24a1 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc @@ -4,12 +4,55 @@ */ #include +#include #include +#include #include +#include #include "biergarten_pipeline_orchestrator.h" #include "services/logging/logger.h" +struct Coords { + double longitude; + double latitude; +}; + +// @todo find a better home for this +namespace { +constexpr double kEarthRadiusKm = 6371.0; + +double DegreesToRadians(double degrees) { + return degrees * std::numbers::pi / 180.0; +} + +double RadiansToDegrees(double radians) { + return radians * 180.0 / std::numbers::pi; +} +} // namespace + +static Coords GetRandomCoordsWithinRange(Coords const& centre, + int const distance_km, + std::mt19937& rng) { + std::uniform_real_distribution angle_dist(0.0, + 2.0 * std::numbers::pi); + std::uniform_real_distribution unit_dist(0.0, 1.0); + + const double angle_radians = angle_dist(rng); + // sqrt() keeps points uniformly distributed over the disc's area rather + // than clustering near the centre. + const double radius_km = distance_km * std::sqrt(unit_dist(rng)); + + const double delta_latitude_degrees = + RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm); + const double delta_longitude_degrees = RadiansToDegrees( + radius_km * std::sin(angle_radians) / + (kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude)))); + + return Coords{.longitude = centre.longitude + delta_longitude_degrees, + .latitude = centre.latitude + delta_latitude_degrees}; +} + void BiergartenPipelineOrchestrator::GenerateBreweries( std::span cities) { logger_->Log({.level = LogLevel::Info, @@ -19,18 +62,28 @@ void BiergartenPipelineOrchestrator::GenerateBreweries( generated_breweries_.clear(); size_t skipped_count = 0; size_t export_failed_count = 0; + std::mt19937 rng(std::random_device{}()); const auto generate_record = - [this, &skipped_count]( + [this, &skipped_count, &rng]( const EnrichedCity& enriched_city) -> std::optional { try { const BreweryResult brewery = generator_->GenerateBrewery(enriched_city); const std::string postal_code = postal_code_service_->GeneratePostalCode(enriched_city.location); + + constexpr int kMaxDistanceFromCentreKm = 5; + const Coords city_centre{.longitude = enriched_city.location.longitude, + .latitude = enriched_city.location.latitude}; + const Coords brewery_coords = GetRandomCoordsWithinRange( + city_centre, kMaxDistanceFromCentreKm, rng); + return BreweryRecord{ .address = BreweryAddress{.city = enriched_city.location, - .postal_code = postal_code}, + .postal_code = postal_code, + .longitude = brewery_coords.longitude, + .latitude = brewery_coords.latitude}, .brewery = brewery}; } catch (const std::exception& e) { ++skipped_count; @@ -91,4 +144,4 @@ void BiergartenPipelineOrchestrator::GenerateBreweries( "brewery/breweries to SQLite", export_failed_count)}); } -} +} \ No newline at end of file diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc index 5dd3c209..1b27752a 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc @@ -17,17 +17,19 @@ void BiergartenPipelineOrchestrator::LogResults() const { for (const auto& [address, brewery] : generated_breweries_) { const City& city = address.city; brewery_output.push_back(boost::json::object{ - {"name_en", brewery.name_en}, - {"description_en", brewery.description_en}, - {"name_local", brewery.name_local}, - {"description_local", brewery.description_local}, - {"address", boost::json::object{ - {"city", city.city}, - {"country", city.country}, - {"state_province", city.state_province}, - {"iso3166_2", city.iso3166_2}, - {"postal_code", address.postal_code}, - }}}); + {"name_en", brewery.name_en}, + {"description_en", brewery.description_en}, + {"name_local", brewery.name_local}, + {"description_local", brewery.description_local}, + {"address", boost::json::object{ + {"city", city.city}, + {"country", city.country}, + {"state_province", city.state_province}, + {"iso3166_2", city.iso3166_2}, + {"postal_code", address.postal_code}, + {"longitude", address.longitude}, + {"latitude", address.latitude}, + }}}); } std::ostringstream brewery_oss; @@ -40,14 +42,14 @@ void BiergartenPipelineOrchestrator::LogResults() const { for (const auto& generated_user : generated_users_) { user_output.push_back(boost::json::object{ - {"first_name", generated_user.user.first_name}, - {"last_name", generated_user.user.last_name}, - {"gender", generated_user.user.gender}, - {"username", generated_user.user.username}, - {"bio", generated_user.user.bio}, - {"activity_weight", generated_user.user.activity_weight}, - {"email", generated_user.email}, - {"date_of_birth", generated_user.date_of_birth}, + {"first_name", generated_user.user.first_name}, + {"last_name", generated_user.user.last_name}, + {"gender", generated_user.user.gender}, + {"username", generated_user.user.username}, + {"bio", generated_user.user.bio}, + {"activity_weight", generated_user.user.activity_weight}, + {"email", generated_user.email}, + {"date_of_birth", generated_user.date_of_birth}, }); } @@ -56,4 +58,4 @@ void BiergartenPipelineOrchestrator::LogResults() const { logger_->Log({.level = LogLevel::Info, .phase = PipelinePhase::Teardown, .message = user_oss.str()}); -} +} \ No newline at end of file diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc index 95e9c096..9d419a23 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc @@ -39,11 +39,17 @@ std::vector BiergartenPipelineOrchestrator::QueryLocations() { logger_->Log({.level = LogLevel::Info, .phase = PipelinePhase::Startup, - .message = std::format(" Locations available: {}", - all_locations.size())}); - logger_->Log({.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = std::format(" Sampled locations: {}", - sampled_locations.size())}); + .message = std::format( + " Locations available: {}\n Sampled Locations: {}", + all_locations.size(), sampled_locations.size())}); + + for (const auto& location : all_locations) { + logger_->Log({.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = std::format("{} {} {} {}", location.city, + location.country, location.longitude, + location.latitude)}); + } + return sampled_locations; -} +} \ No newline at end of file diff --git a/tooling/pipeline/src/main.cc b/tooling/pipeline/src/main.cc index 5154058a..5b182713 100644 --- a/tooling/pipeline/src/main.cc +++ b/tooling/pipeline/src/main.cc @@ -110,13 +110,13 @@ int main(const int argc, char** argv) { di::bind().to( [options, &log_producer]() -> std::unique_ptr { - if (options.generator.use_mocked) { - log_producer->Log({.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = "Curated data: mock"}); - - return std::make_unique(); - } + // if (options.generator.use_mocked) { + // log_producer->Log({.level = LogLevel::Info, + // .phase = PipelinePhase::Startup, + // .message = "Curated data: mock"}); + // + // return std::make_unique(); + // } log_producer->Log({.level = LogLevel::Info, .phase = PipelinePhase::Startup, diff --git a/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc b/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc index c85a7820..eb17df45 100644 --- a/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc +++ b/tooling/pipeline/src/services/curated_data/curated_json_data_service.cc @@ -30,6 +30,16 @@ static std::string ReadRequiredString(const boost::json::object& object, return std::string(text); } +static double ReadRequiredDouble(const boost::json::object& object, + const char* key) { + const boost::json::value* value = object.if_contains(key); + if (value == nullptr || !value->is_double()) { + throw std::runtime_error( + std::format("Missing or invalid double field: {}", key)); + } + return value->as_double(); +} + static std::vector ReadRequiredStringArray( const boost::json::object& object, const char* key) { const boost::json::value* value = object.if_contains(key); @@ -67,7 +77,6 @@ static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) { } namespace { - boost::json::value ParseJsonFile(const std::filesystem::path& filepath, const char* what) { std::ifstream input(filepath); @@ -108,7 +117,6 @@ std::string ReadFirstOfStringArray(const boost::json::object& object, throw std::runtime_error( std::format("Missing or invalid string array field: {}", key)); } - } // namespace CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths) @@ -144,6 +152,8 @@ const LocationsList& CuratedJsonDataService::LoadLocations() { .iso3166_2 = ReadRequiredString(object, "iso3166_2"), .country = ReadRequiredString(object, "country"), .iso3166_1 = ReadRequiredString(object, "iso3166_1"), + .latitude = ReadRequiredDouble(object, "latitude"), + .longitude = ReadRequiredDouble(object, "longitude"), .local_languages = ReadRequiredStringArray(object, "local_languages"), .postal_code = ReadPostalCodeSpec(object), }); @@ -267,4 +277,4 @@ const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() { cache_.surnames_by_country = std::move(surnames_by_country); return cache_.surnames_by_country; -} +} \ No newline at end of file diff --git a/tooling/pipeline/src/services/sqlite/process_record.cc b/tooling/pipeline/src/services/sqlite/process_record.cc index 1c860aee..964892a7 100644 --- a/tooling/pipeline/src/services/sqlite/process_record.cc +++ b/tooling/pipeline/src/services/sqlite/process_record.cc @@ -66,6 +66,18 @@ sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) { .index = sqlite_export_service_internal::kCityIso31661BindIndex, .value = city.iso3166_1, .action = "Failed to bind SQLite city ISO 3166-1 code"}); + sqlite_export_service_internal::Bind( + insert_city_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal::kCityLongitudeBindIndex, + .value = city.longitude, + .action = "Failed to bind SQLite city longitude"}); + sqlite_export_service_internal::Bind( + insert_city_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal::kCityLatitudeBindIndex, + .value = city.latitude, + .action = "Failed to bind SQLite city latitude"}); sqlite_export_service_internal::Bind( insert_city_stmt_, sqlite_export_service_internal::BoundParam{ @@ -165,6 +177,20 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) { kBreweryAddressPostalCodeBindIndex, .value = brewery.address.postal_code, .action = "Failed to bind SQLite brewery address postal code"}); + sqlite_export_service_internal::Bind( + insert_brewery_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal:: + kBreweryAddressLongitudeBindIndex, + .value = brewery.address.longitude, + .action = "Failed to bind SQLite brewery address longitude"}); + sqlite_export_service_internal::Bind( + insert_brewery_address_stmt_, + sqlite_export_service_internal::BoundParam{ + .index = sqlite_export_service_internal:: + kBreweryAddressLatitudeBindIndex, + .value = brewery.address.latitude, + .action = "Failed to bind SQLite brewery address latitude"}); sqlite_export_service_internal::StepStatement( db_handle_, insert_brewery_address_stmt_,