mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-15 01:54:43 +00:00
Formatted src/biergarten_pipeline_orchestrator
This commit is contained in:
@@ -12,80 +12,83 @@
|
||||
|
||||
void BiergartenPipelineOrchestrator::GenerateBreweries(
|
||||
std::span<const EnrichedCity> cities) {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = "=== SAMPLE BREWERY GENERATION ==="});
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = "=== SAMPLE BREWERY GENERATION ==="});
|
||||
|
||||
generated_breweries_.clear();
|
||||
size_t skipped_count = 0;
|
||||
size_t export_failed_count = 0;
|
||||
generated_breweries_.clear();
|
||||
size_t skipped_count = 0;
|
||||
size_t export_failed_count = 0;
|
||||
|
||||
const auto generate_record =
|
||||
[this, &skipped_count](
|
||||
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
|
||||
try {
|
||||
const BreweryResult brewery = generator_->GenerateBrewery(enriched_city);
|
||||
const std::string postal_code =
|
||||
postal_code_service_->GeneratePostalCode(enriched_city.location);
|
||||
return BreweryRecord{
|
||||
.address = BreweryAddress{.city = enriched_city.location,
|
||||
.postal_code = postal_code},
|
||||
.brewery = brewery};
|
||||
} catch (const std::exception& e) {
|
||||
++skipped_count;
|
||||
const auto generate_record =
|
||||
[this, &skipped_count](
|
||||
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
|
||||
try {
|
||||
const BreweryResult brewery =
|
||||
generator_->GenerateBrewery(enriched_city);
|
||||
const std::string postal_code =
|
||||
postal_code_service_->GeneratePostalCode(enriched_city.location);
|
||||
return BreweryRecord{
|
||||
.address = BreweryAddress{.city = enriched_city.location,
|
||||
.postal_code = postal_code},
|
||||
.brewery = brewery};
|
||||
} catch (const std::exception& e) {
|
||||
++skipped_count;
|
||||
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipping city '{}' ({}): brewery "
|
||||
"generation failed: {}",
|
||||
enriched_city.location.city,
|
||||
enriched_city.location.country, e.what())});
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
const auto export_record =
|
||||
[this, &export_failed_count](const BreweryRecord& record) {
|
||||
try {
|
||||
exporter_->ProcessRecord(record);
|
||||
} catch (const std::exception& export_exception) {
|
||||
++export_failed_count;
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Generated brewery for '{}' ({}) "
|
||||
"but SQLite export failed: {}",
|
||||
record.address.city.city, record.address.city.country,
|
||||
export_exception.what())});
|
||||
}
|
||||
};
|
||||
|
||||
for (const EnrichedCity& enriched_city : cities) {
|
||||
const std::optional<BreweryRecord> record =
|
||||
generate_record(enriched_city);
|
||||
if (!record.has_value()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
generated_breweries_.push_back(*record);
|
||||
export_record(*record);
|
||||
}
|
||||
|
||||
if (skipped_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
|
||||
"generation failed: {}",
|
||||
enriched_city.location.city,
|
||||
enriched_city.location.country, e.what())});
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipped {} city/cities due to generation errors",
|
||||
skipped_count)});
|
||||
}
|
||||
|
||||
const auto export_record = [this, &export_failed_count](
|
||||
const BreweryRecord& record) {
|
||||
try {
|
||||
exporter_->ProcessRecord(record);
|
||||
} catch (const std::exception& export_exception) {
|
||||
++export_failed_count;
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Generated brewery for '{}' ({}) "
|
||||
"but SQLite export failed: {}",
|
||||
record.address.city.city, record.address.city.country,
|
||||
export_exception.what())});
|
||||
}
|
||||
};
|
||||
|
||||
for (const EnrichedCity& enriched_city : cities) {
|
||||
const std::optional<BreweryRecord> record = generate_record(enriched_city);
|
||||
if (!record.has_value()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
generated_breweries_.push_back(*record);
|
||||
export_record(*record);
|
||||
}
|
||||
|
||||
if (skipped_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipped {} city/cities due to generation errors",
|
||||
skipped_count)});
|
||||
}
|
||||
|
||||
if (export_failed_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = std::format("[Pipeline] Failed to export {} generated "
|
||||
"brewery/breweries to SQLite",
|
||||
export_failed_count)});
|
||||
}
|
||||
if (export_failed_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = std::format("[Pipeline] Failed to export {} generated "
|
||||
"brewery/breweries to SQLite",
|
||||
export_failed_count)});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,203 +20,205 @@
|
||||
|
||||
namespace {
|
||||
std::string Sanitize(std::string_view value) {
|
||||
std::string out;
|
||||
out.reserve(value.size());
|
||||
for (const char character : value) {
|
||||
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
|
||||
out.push_back(static_cast<char>(
|
||||
std::tolower(static_cast<unsigned char>(character))));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
std::string out;
|
||||
out.reserve(value.size());
|
||||
for (const char character : value) {
|
||||
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
|
||||
out.push_back(static_cast<char>(
|
||||
std::tolower(static_cast<unsigned char>(character))));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string BuildEmail(const Name& name,
|
||||
std::unordered_set<std::string>& used_local_parts) {
|
||||
const std::string base =
|
||||
std::format("{}.{}", Sanitize(name.first_name), Sanitize(name.last_name));
|
||||
const std::string base = std::format("{}.{}", Sanitize(name.first_name),
|
||||
Sanitize(name.last_name));
|
||||
|
||||
std::string local_part = base;
|
||||
uint32_t suffix = 1;
|
||||
std::string local_part = base;
|
||||
uint32_t suffix = 1;
|
||||
|
||||
while (used_local_parts.contains(local_part)) {
|
||||
local_part = std::format("{}{}", base, suffix);
|
||||
++suffix;
|
||||
}
|
||||
while (used_local_parts.contains(local_part)) {
|
||||
local_part = std::format("{}{}", base, suffix);
|
||||
++suffix;
|
||||
}
|
||||
|
||||
used_local_parts.insert(local_part);
|
||||
used_local_parts.insert(local_part);
|
||||
|
||||
return std::format("{}@thebiergarten.app", local_part);
|
||||
return std::format("{}@thebiergarten.app", local_part);
|
||||
}
|
||||
|
||||
std::string GenerateDateOfBirth(std::mt19937& rng) {
|
||||
using namespace std::chrono;
|
||||
using namespace std::chrono;
|
||||
|
||||
constexpr int kMinAge = 19;
|
||||
constexpr int kMaxAge = 48;
|
||||
constexpr int kMaxDayOffset = 364;
|
||||
constexpr int kMinAge = 19;
|
||||
constexpr int kMaxAge = 48;
|
||||
constexpr int kMaxDayOffset = 364;
|
||||
|
||||
std::uniform_int_distribution<int> age_dist(kMinAge, kMaxAge);
|
||||
std::uniform_int_distribution<int> day_offset_dist(0, kMaxDayOffset);
|
||||
std::uniform_int_distribution<int> age_dist(kMinAge, kMaxAge);
|
||||
std::uniform_int_distribution<int> day_offset_dist(0, kMaxDayOffset);
|
||||
|
||||
const year_month_day today{floor<days>(system_clock::now())};
|
||||
const year_month_day birth_year_anchor{today.year() - years{age_dist(rng)},
|
||||
today.month(), today.day()};
|
||||
const sys_days birth_date =
|
||||
sys_days{birth_year_anchor} - days{day_offset_dist(rng)};
|
||||
const year_month_day birth_ymd{birth_date};
|
||||
const year_month_day today{floor<days>(system_clock::now())};
|
||||
const year_month_day birth_year_anchor{today.year() - years{age_dist(rng)},
|
||||
today.month(), today.day()};
|
||||
const sys_days birth_date =
|
||||
sys_days{birth_year_anchor} - days{day_offset_dist(rng)};
|
||||
const year_month_day birth_ymd{birth_date};
|
||||
|
||||
return std::format("{:04}-{:02}-{:02}", static_cast<int>(birth_ymd.year()),
|
||||
static_cast<unsigned>(birth_ymd.month()),
|
||||
static_cast<unsigned>(birth_ymd.day()));
|
||||
return std::format("{:04}-{:02}-{:02}", static_cast<int>(birth_ymd.year()),
|
||||
static_cast<unsigned>(birth_ymd.month()),
|
||||
static_cast<unsigned>(birth_ymd.day()));
|
||||
}
|
||||
|
||||
std::optional<Name> SampleName(
|
||||
const ForenamesByCountryMap& forenames_by_country,
|
||||
const SurnamesByCountryMap& surnames_by_country,
|
||||
const std::string& iso3166_1, std::mt19937& rng) {
|
||||
const auto forenames_it = forenames_by_country.find(iso3166_1);
|
||||
const auto surnames_it = surnames_by_country.find(iso3166_1);
|
||||
const auto forenames_it = forenames_by_country.find(iso3166_1);
|
||||
const auto surnames_it = surnames_by_country.find(iso3166_1);
|
||||
|
||||
if (forenames_it == forenames_by_country.end() ||
|
||||
surnames_it == surnames_by_country.end() ||
|
||||
forenames_it->second.empty() || surnames_it->second.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
if (forenames_it == forenames_by_country.end() ||
|
||||
surnames_it == surnames_by_country.end() ||
|
||||
forenames_it->second.empty() || surnames_it->second.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const ForenameList& forenames = forenames_it->second;
|
||||
const SurnameList& surnames = surnames_it->second;
|
||||
const ForenameList& forenames = forenames_it->second;
|
||||
const SurnameList& surnames = surnames_it->second;
|
||||
|
||||
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
|
||||
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
|
||||
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
|
||||
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
|
||||
|
||||
auto forename_it = forenames.begin();
|
||||
std::advance(forename_it, forename_dist(rng));
|
||||
auto forename_it = forenames.begin();
|
||||
std::advance(forename_it, forename_dist(rng));
|
||||
|
||||
auto surname_it = surnames.begin();
|
||||
std::advance(surname_it, surname_dist(rng));
|
||||
auto surname_it = surnames.begin();
|
||||
std::advance(surname_it, surname_dist(rng));
|
||||
|
||||
return Name{.first_name = forename_it->name,
|
||||
.last_name = *surname_it,
|
||||
.gender = forename_it->gender};
|
||||
return Name{.first_name = forename_it->name,
|
||||
.last_name = *surname_it,
|
||||
.gender = forename_it->gender};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
std::span<const EnrichedCity> cities) {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = "=== SAMPLE USER GENERATION ==="});
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = "=== SAMPLE USER GENERATION ==="});
|
||||
|
||||
const PersonasList& personas = curated_data_service_->LoadPersonas();
|
||||
const PersonasList& personas = curated_data_service_->LoadPersonas();
|
||||
|
||||
if (personas.empty()) {
|
||||
throw std::runtime_error(
|
||||
"No personas available in personas.json for user generation");
|
||||
}
|
||||
if (personas.empty()) {
|
||||
throw std::runtime_error(
|
||||
"No personas available in personas.json for user generation");
|
||||
}
|
||||
|
||||
const ForenamesByCountryMap& forenames_by_country =
|
||||
curated_data_service_->LoadForenamesByCountry();
|
||||
const SurnamesByCountryMap& surnames_by_country =
|
||||
curated_data_service_->LoadSurnamesByCountry();
|
||||
const ForenamesByCountryMap& forenames_by_country =
|
||||
curated_data_service_->LoadForenamesByCountry();
|
||||
const SurnamesByCountryMap& surnames_by_country =
|
||||
curated_data_service_->LoadSurnamesByCountry();
|
||||
|
||||
std::mt19937 rng(std::random_device{}());
|
||||
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
|
||||
std::mt19937 rng(std::random_device{}());
|
||||
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
|
||||
|
||||
generated_users_.clear();
|
||||
size_t skipped_count = 0;
|
||||
size_t export_failed_count = 0;
|
||||
std::unordered_set<std::string> used_email_local_parts;
|
||||
generated_users_.clear();
|
||||
size_t skipped_count = 0;
|
||||
size_t export_failed_count = 0;
|
||||
std::unordered_set<std::string> used_email_local_parts;
|
||||
|
||||
const auto generate_record =
|
||||
[this, &rng, &skipped_count, &used_email_local_parts](
|
||||
const EnrichedCity& city, const UserPersona& persona,
|
||||
const Name& sampled_name) -> std::optional<UserRecord> {
|
||||
try {
|
||||
const UserResult user =
|
||||
generator_->GenerateUser(city, persona, sampled_name);
|
||||
const std::string postal_code =
|
||||
postal_code_service_->GeneratePostalCode(city.location);
|
||||
const auto generate_record =
|
||||
[this, &rng, &skipped_count, &used_email_local_parts](
|
||||
const EnrichedCity& city, const UserPersona& persona,
|
||||
const Name& sampled_name) -> std::optional<UserRecord> {
|
||||
try {
|
||||
const UserResult user =
|
||||
generator_->GenerateUser(city, persona, sampled_name);
|
||||
const std::string postal_code =
|
||||
postal_code_service_->GeneratePostalCode(city.location);
|
||||
|
||||
return UserRecord{
|
||||
.address =
|
||||
UserAddress{.city = city.location, .postal_code = postal_code},
|
||||
.user = user,
|
||||
.email = BuildEmail(sampled_name, used_email_local_parts),
|
||||
.date_of_birth = GenerateDateOfBirth(rng),
|
||||
};
|
||||
} catch (const std::exception& e) {
|
||||
++skipped_count;
|
||||
return UserRecord{
|
||||
.address =
|
||||
UserAddress{.city = city.location, .postal_code = postal_code},
|
||||
.user = user,
|
||||
.email = BuildEmail(sampled_name, used_email_local_parts),
|
||||
.date_of_birth = GenerateDateOfBirth(rng),
|
||||
};
|
||||
} catch (const std::exception& e) {
|
||||
++skipped_count;
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format("[Pipeline] Skipping city '{}' ({}): "
|
||||
"user generation failed: {}",
|
||||
city.location.city, city.location.country,
|
||||
e.what())});
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
const auto export_record = [this,
|
||||
&export_failed_count](const UserRecord& record) {
|
||||
try {
|
||||
exporter_->ProcessRecord(record);
|
||||
} catch (const std::exception& export_exception) {
|
||||
++export_failed_count;
|
||||
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Generated user for '{}' ({}) but "
|
||||
"SQLite export failed: {}",
|
||||
record.address.city.city, record.address.city.country,
|
||||
export_exception.what())});
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto& city : cities) {
|
||||
const std::optional<Name> sampled_name =
|
||||
SampleName(forenames_by_country, surnames_by_country,
|
||||
city.location.iso3166_1, rng);
|
||||
|
||||
if (!sampled_name.has_value()) {
|
||||
++skipped_count;
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipping city '{}' ({}): no names "
|
||||
"available for country '{}'",
|
||||
city.location.city, city.location.country,
|
||||
city.location.iso3166_1)});
|
||||
continue;
|
||||
}
|
||||
|
||||
const UserPersona& persona = personas[persona_dist(rng)];
|
||||
|
||||
const std::optional<UserRecord> record =
|
||||
generate_record(city, persona, *sampled_name);
|
||||
if (!record.has_value()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
generated_users_.push_back(*record);
|
||||
export_record(*record);
|
||||
}
|
||||
|
||||
if (skipped_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipped {} city/cities during user generation",
|
||||
skipped_count)});
|
||||
}
|
||||
|
||||
if (export_failed_count > 0) {
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipping city '{}' ({}): "
|
||||
"user generation failed: {}",
|
||||
city.location.city, city.location.country, e.what())});
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
const auto export_record = [this,
|
||||
&export_failed_count](const UserRecord& record) {
|
||||
try {
|
||||
exporter_->ProcessRecord(record);
|
||||
} catch (const std::exception& export_exception) {
|
||||
++export_failed_count;
|
||||
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Generated user for '{}' ({}) but "
|
||||
"SQLite export failed: {}",
|
||||
record.address.city.city, record.address.city.country,
|
||||
export_exception.what())});
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto& city : cities) {
|
||||
const std::optional<Name> sampled_name =
|
||||
SampleName(forenames_by_country, surnames_by_country,
|
||||
city.location.iso3166_1, rng);
|
||||
|
||||
if (!sampled_name.has_value()) {
|
||||
++skipped_count;
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipping city '{}' ({}): no names "
|
||||
"available for country '{}'",
|
||||
city.location.city, city.location.country,
|
||||
city.location.iso3166_1)});
|
||||
continue;
|
||||
}
|
||||
|
||||
const UserPersona& persona = personas[persona_dist(rng)];
|
||||
|
||||
const std::optional<UserRecord> record =
|
||||
generate_record(city, persona, *sampled_name);
|
||||
if (!record.has_value()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
generated_users_.push_back(*record);
|
||||
export_record(*record);
|
||||
}
|
||||
|
||||
if (skipped_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::UserGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipped {} city/cities during user generation",
|
||||
skipped_count)});
|
||||
}
|
||||
|
||||
if (export_failed_count > 0) {
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = std::format("[Pipeline] Failed to export {} "
|
||||
"generated user/users to SQLite",
|
||||
export_failed_count)});
|
||||
}
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = std::format("[Pipeline] Failed to export {} "
|
||||
"generated user/users to SQLite",
|
||||
export_failed_count)});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,48 +12,48 @@
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
void BiergartenPipelineOrchestrator::LogResults() const {
|
||||
boost::json::array brewery_output;
|
||||
boost::json::array brewery_output;
|
||||
|
||||
for (const auto& [address, brewery] : generated_breweries_) {
|
||||
const City& city = address.city;
|
||||
brewery_output.push_back(boost::json::object{
|
||||
{"name_en", brewery.name_en},
|
||||
{"description_en", brewery.description_en},
|
||||
{"name_local", brewery.name_local},
|
||||
{"description_local", brewery.description_local},
|
||||
{"address", boost::json::object{
|
||||
{"city", city.city},
|
||||
{"country", city.country},
|
||||
{"state_province", city.state_province},
|
||||
{"iso3166_2", city.iso3166_2},
|
||||
{"postal_code", address.postal_code},
|
||||
}}});
|
||||
}
|
||||
for (const auto& [address, brewery] : generated_breweries_) {
|
||||
const City& city = address.city;
|
||||
brewery_output.push_back(boost::json::object{
|
||||
{"name_en", brewery.name_en},
|
||||
{"description_en", brewery.description_en},
|
||||
{"name_local", brewery.name_local},
|
||||
{"description_local", brewery.description_local},
|
||||
{"address", boost::json::object{
|
||||
{"city", city.city},
|
||||
{"country", city.country},
|
||||
{"state_province", city.state_province},
|
||||
{"iso3166_2", city.iso3166_2},
|
||||
{"postal_code", address.postal_code},
|
||||
}}});
|
||||
}
|
||||
|
||||
std::ostringstream brewery_oss;
|
||||
PrettyPrint(brewery_oss, brewery_output);
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = brewery_oss.str()});
|
||||
std::ostringstream brewery_oss;
|
||||
PrettyPrint(brewery_oss, brewery_output);
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = brewery_oss.str()});
|
||||
|
||||
boost::json::array user_output;
|
||||
boost::json::array user_output;
|
||||
|
||||
for (const auto& generated_user : generated_users_) {
|
||||
user_output.push_back(boost::json::object{
|
||||
{"first_name", generated_user.user.first_name},
|
||||
{"last_name", generated_user.user.last_name},
|
||||
{"gender", generated_user.user.gender},
|
||||
{"username", generated_user.user.username},
|
||||
{"bio", generated_user.user.bio},
|
||||
{"activity_weight", generated_user.user.activity_weight},
|
||||
{"email", generated_user.email},
|
||||
{"date_of_birth", generated_user.date_of_birth},
|
||||
});
|
||||
}
|
||||
for (const auto& generated_user : generated_users_) {
|
||||
user_output.push_back(boost::json::object{
|
||||
{"first_name", generated_user.user.first_name},
|
||||
{"last_name", generated_user.user.last_name},
|
||||
{"gender", generated_user.user.gender},
|
||||
{"username", generated_user.user.username},
|
||||
{"bio", generated_user.user.bio},
|
||||
{"activity_weight", generated_user.user.activity_weight},
|
||||
{"email", generated_user.email},
|
||||
{"date_of_birth", generated_user.date_of_birth},
|
||||
});
|
||||
}
|
||||
|
||||
std::ostringstream user_oss;
|
||||
PrettyPrint(user_oss, user_output);
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = user_oss.str()});
|
||||
std::ostringstream user_oss;
|
||||
PrettyPrint(user_oss, user_output);
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = user_oss.str()});
|
||||
}
|
||||
|
||||
@@ -14,37 +14,36 @@
|
||||
#include "services/curated_data/curated_json_data_service.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
std::vector<City>
|
||||
BiergartenPipelineOrchestrator::QueryLocations() {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
|
||||
std::vector<City> BiergartenPipelineOrchestrator::QueryLocations() {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
|
||||
|
||||
const std::vector<City>& all_locations =
|
||||
curated_data_service_->LoadLocations();
|
||||
const std::vector<City>& all_locations =
|
||||
curated_data_service_->LoadLocations();
|
||||
|
||||
const size_t sample_count = std::min(
|
||||
static_cast<size_t>(application_options_.pipeline.location_count),
|
||||
all_locations.size());
|
||||
const size_t sample_count = std::min(
|
||||
static_cast<size_t>(application_options_.pipeline.location_count),
|
||||
all_locations.size());
|
||||
|
||||
const auto sample_count_signed =
|
||||
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
|
||||
sample_count);
|
||||
const auto sample_count_signed =
|
||||
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
|
||||
sample_count);
|
||||
|
||||
std::vector<City> sampled_locations;
|
||||
sampled_locations.reserve(sample_count);
|
||||
std::vector<City> sampled_locations;
|
||||
sampled_locations.reserve(sample_count);
|
||||
|
||||
std::random_device random_generator;
|
||||
std::ranges::sample(all_locations, std::back_inserter(sampled_locations),
|
||||
sample_count_signed, random_generator);
|
||||
std::random_device random_generator;
|
||||
std::ranges::sample(all_locations, std::back_inserter(sampled_locations),
|
||||
sample_count_signed, random_generator);
|
||||
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format(" Locations available: {}",
|
||||
all_locations.size())});
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format(" Sampled locations: {}",
|
||||
sampled_locations.size())});
|
||||
return sampled_locations;
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format(" Locations available: {}",
|
||||
all_locations.size())});
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = std::format(" Sampled locations: {}",
|
||||
sampled_locations.size())});
|
||||
return sampled_locations;
|
||||
}
|
||||
|
||||
@@ -11,51 +11,52 @@
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
bool BiergartenPipelineOrchestrator::Run() {
|
||||
try {
|
||||
exporter_->Initialize();
|
||||
try {
|
||||
exporter_->Initialize();
|
||||
|
||||
std::vector<City> cities = QueryLocations();
|
||||
std::vector<EnrichedCity> enriched;
|
||||
enriched.reserve(cities.size());
|
||||
std::vector<City> cities = QueryLocations();
|
||||
std::vector<EnrichedCity> enriched;
|
||||
enriched.reserve(cities.size());
|
||||
|
||||
size_t skipped_count = 0;
|
||||
for (auto& city : cities) {
|
||||
try {
|
||||
std::string region_context = context_service_->GetLocationContext(city);
|
||||
size_t skipped_count = 0;
|
||||
for (auto& city : cities) {
|
||||
try {
|
||||
std::string region_context =
|
||||
context_service_->GetLocationContext(city);
|
||||
|
||||
enriched.push_back(
|
||||
EnrichedCity{.location = std::move(city),
|
||||
.region_context = std::move(region_context)});
|
||||
} catch (const std::exception& exception) {
|
||||
++skipped_count;
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipping city '{}' ({}): context "
|
||||
"lookup failed: {}",
|
||||
city.city, city.country, exception.what())});
|
||||
enriched.push_back(
|
||||
EnrichedCity{.location = std::move(city),
|
||||
.region_context = std::move(region_context)});
|
||||
} catch (const std::exception& exception) {
|
||||
++skipped_count;
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format(
|
||||
"[Pipeline] Skipping city '{}' ({}): context "
|
||||
"lookup failed: {}",
|
||||
city.city, city.country, exception.what())});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (skipped_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format("[Pipeline] Skipped {} city/cities due "
|
||||
"to context lookup errors",
|
||||
skipped_count)});
|
||||
}
|
||||
if (skipped_count > 0) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format("[Pipeline] Skipped {} city/cities due "
|
||||
"to context lookup errors",
|
||||
skipped_count)});
|
||||
}
|
||||
|
||||
this->GenerateUsers(enriched);
|
||||
this->GenerateBreweries(enriched);
|
||||
exporter_->Finalize();
|
||||
this->LogResults();
|
||||
return true;
|
||||
} catch (const std::exception& e) {
|
||||
logger_->Log({.level = LogLevel::Error,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = std::format(
|
||||
"Pipeline execution failed with error: {}", e.what())});
|
||||
return false;
|
||||
}
|
||||
this->GenerateUsers(enriched);
|
||||
this->GenerateBreweries(enriched);
|
||||
exporter_->Finalize();
|
||||
this->LogResults();
|
||||
return true;
|
||||
} catch (const std::exception& e) {
|
||||
logger_->Log({.level = LogLevel::Error,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
.message = std::format(
|
||||
"Pipeline execution failed with error: {}", e.what())});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user