AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 17:47:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
Aaron Po
2026-06-21 13:13:02 -04:00
parent 4de0ea6638
commit ad97b0ea6c
27 changed files with 313 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
tooling/pipeline/src/application_options/parse_arguments.cc
View File

@@ -51,7 +51,8 @@ std::optional<ApplicationOptions> ParseArguments(
prog_opts::value<std::string>()->default_value("pipeline.log"),
"Path for application logs");
opt("prompt-dir", prog_opts::value<std::string>()->default_value(""),
"Directory containing named prompt files (e.g. BREWERY_GENERATION.md)."
"Directory containing named prompt files (e.g. "
"BREWERY_GENERATION.md)."
" Required when not using --mocked.");
opt("location-count", prog_opts::value<uint32_t>()->default_value(10));
};
@@ -70,11 +71,11 @@ std::optional<ApplicationOptions> ParseArguments(
})();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = title});
.phase = PipelinePhase::Startup,
.message = title});
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = usage});
.phase = PipelinePhase::Startup,
.message = usage});
}
return std::nullopt;
}
@@ -89,8 +90,8 @@ std::optional<ApplicationOptions> ParseArguments(
help_stream << "\n" << desc;
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = help_stream.str()});
.phase = PipelinePhase::Startup,
.message = help_stream.str()});
}
return std::nullopt;
}
@@ -106,14 +107,16 @@ std::optional<ApplicationOptions> ParseArguments(
const std::string model_path = var_map["model"].as<std::string>();
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
// Enforce mutual exclusivity before any further configuration is applied.
// Enforce mutual exclusivity before any further configuration is
// applied.
if (use_mocked && !model_path.empty()) {
const std::string msg =
"Invalid arguments: --mocked and --model are mutually exclusive";
"Invalid arguments: --mocked and --model are mutually "
"exclusive";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
@@ -122,11 +125,12 @@ std::optional<ApplicationOptions> ParseArguments(
if (!use_mocked && model_path.empty()) {
const std::string msg =
"Invalid arguments: either --mocked or --model must be specified";
"Invalid arguments: either --mocked or --model must be "
"specified";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
@@ -137,7 +141,8 @@ std::optional<ApplicationOptions> ParseArguments(
// generator has no use for it and should not require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using --mocked";
"Invalid arguments: --prompt-dir is required when not using "
"--mocked";
if (logger) {
logger->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
@@ -168,8 +173,8 @@ std::optional<ApplicationOptions> ParseArguments(
"Sampling parameters are ignored when using --mocked";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Warn,
.phase = PipelinePhase::Startup,
.message = msg});
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
@@ -194,8 +199,8 @@ std::optional<ApplicationOptions> ParseArguments(
exception.what();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
} catch (...) {
@@ -203,8 +208,8 @@ std::optional<ApplicationOptions> ParseArguments(
"Failed to parse command-line arguments: unknown error";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
}

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -12,7 +12,7 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
const ApplicationOptions &app_options)
const ApplicationOptions& app_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),

37
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc
View File

@@ -36,33 +36,38 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message =
std::format("[Pipeline] Generated brewery for '{}' ({}) but SQLite export failed: {}",
.message = std::format(
"[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}",
location.city, location.country, export_exception.what())});
}
} catch (const std::exception& e) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery generation failed: {}",
location.city, location.country, e.what())});
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
"generation failed: {}",
location.city, location.country, e.what())});
}
}
if (skipped_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to generation errors",
skipped_count)});
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to generation errors",
skipped_count)});
}
if (export_failed_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format(
"[Pipeline] Failed to export {} generated brewery/breweries to SQLite",
export_failed_count)});
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} generated "
"brewery/breweries to SQLite",
export_failed_count)});
}
}

63
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -24,8 +24,8 @@ std::string Sanitize(std::string_view value) {
out.reserve(value.size());
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(
static_cast<char>(std::tolower(static_cast<unsigned char>(character))));
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
@@ -62,8 +62,7 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
sys_days{birth_year_anchor} - days{day_offset_dist(rng)};
const year_month_day birth_ymd{birth_date};
return std::format("{:04}-{:02}-{:02}",
static_cast<int>(birth_ymd.year()),
return std::format("{:04}-{:02}-{:02}", static_cast<int>(birth_ymd.year()),
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
}
@@ -71,7 +70,8 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
std::string GenerateRandomPassword(std::mt19937& rng) {
constexpr size_t kPasswordLength = 32;
constexpr std::string_view kCharset =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*";
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&"
"*";
std::uniform_int_distribution<size_t> char_dist(0, kCharset.size() - 1);
@@ -115,14 +115,14 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
if (!sampled_name.has_value()) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): no names available for "
"country '{}'",
city.location.city, city.location.country,
city.location.iso3166_1)});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): no names "
"available for "
"country '{}'",
city.location.city, city.location.country,
city.location.iso3166_1)});
continue;
}
@@ -147,22 +147,22 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Generated user for '{}' ({}) but SQLite export failed: {}",
city.location.city, city.location.country,
export_exception.what())});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
city.location.city, city.location.country,
export_exception.what())});
}
} catch (const std::exception& e) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): user generation failed: {}",
city.location.city, city.location.country, e.what())});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): "
"user generation failed: {}",
city.location.city, city.location.country, e.what())});
}
}
@@ -176,11 +176,10 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
}
if (export_failed_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format(
"[Pipeline] Failed to export {} generated user/users to SQLite",
export_failed_count)});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} "
"generated user/users to SQLite",
export_failed_count)});
}
}

1
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -38,7 +38,6 @@ void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array user_output;
for (const auto& generated_user : generated_users_) {
user_output.push_back(boost::json::object{
{"first_name", generated_user.user.first_name},

32
tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc
View File

@@ -28,21 +28,22 @@ bool BiergartenPipelineOrchestrator::Run() {
.region_context = std::move(region_context)});
} catch (const std::exception& exception) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): context lookup failed: {}",
city.city, city.country, exception.what())});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): context "
"lookup failed: {}",
city.city, city.country, exception.what())});
}
}
if (skipped_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to context lookup errors",
skipped_count)});
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("[Pipeline] Skipped {} city/cities due "
"to context lookup errors",
skipped_count)});
}
this->GenerateUsers(enriched);
@@ -51,11 +52,10 @@ bool BiergartenPipelineOrchestrator::Run() {
this->LogResults();
return true;
} catch (const std::exception& e) {
logger_->Log(
{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message =
std::format("Pipeline execution failed with error: {}", e.what())});
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = std::format(
"Pipeline execution failed with error: {}", e.what())});
return false;
}
}

42
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -83,8 +83,8 @@ BreweryResult LlamaGenerator::GenerateBrewery(
/**
* RETRY LOOP with validation and error correction
* Attempts to generate valid brewery data up to 3 times, with feedback-based
* refinement
* Attempts to generate valid brewery data up to 3 times, with
* feedback-based refinement
*/
constexpr int max_attempts = 3;
std::string raw;
@@ -103,7 +103,7 @@ BreweryResult LlamaGenerator::GenerateBrewery(
{.level = LogLevel::Debug,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
attempt + 1, raw)});
}
// Validate output: parse JSON and check required fields
@@ -119,8 +119,9 @@ BreweryResult LlamaGenerator::GenerateBrewery(
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: successfully generated brewery data on attempt {}",
attempt + 1)});
.message = std::format("LlamaGenerator: successfully generated "
"brewery data on attempt {}",
attempt + 1)});
}
return brewery;
@@ -133,33 +134,38 @@ BreweryResult LlamaGenerator::GenerateBrewery(
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message =
std::format("LlamaGenerator: malformed brewery JSON (attempt {}): {}",
.message = std::format(
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
// Update prompt with error details to guide LLM toward correct output.
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON with "
"exactly these keys, in this exact order: {{\"name_en\": \"<English "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"name_en\": "
"\"<English "
"brewery name>\", \"description_en\": \"<English single-paragraph "
"description>\", \"name_local\": \"<local-language brewery name>\", "
"description>\", \"name_local\": \"<local-language brewery "
"name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\nKeep the JSON strings concise enough "
"description>\"}}.\nDo not include markdown, comments, extra keys, "
"or "
"literal placeholder values.\n\nKeep the JSON strings concise "
"enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
// All retry attempts exhausted: log failure and throw exception
if (logger_) {
logger_->Log(
{.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery "
"response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed brewery response");
}

52
tooling/pipeline/src/data_generation/llama/generate_user.cc
View File

@@ -41,17 +41,17 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
std::string user_prompt = std::format(
"## NAME:\n{} {}\n\n## GENDER:\n{}\n\n## CITY:\n{}\n\n## COUNTRY:\n{}\n\n"
"## NAME:\n{} {}\n\n## GENDER:\n{}\n\n## CITY:\n{}\n\n## "
"COUNTRY:\n{}\n\n"
"## PERSONA:\n{}\n\n## PERSONA DESCRIPTION:\n{}\n\n## STYLE "
"AFFINITIES:\n{}",
name.first_name, name.last_name, name.gender, city.location.city,
city.location.country, persona.name, persona.description,
style_affinities);
const std::string retry_context =
std::format("Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
name.last_name, city.location.city, city.location.country,
persona.name);
const std::string retry_context = std::format(
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name, name.last_name,
city.location.city, city.location.country, persona.name);
constexpr int max_attempts = 3;
std::string raw;
@@ -59,14 +59,13 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
int max_tokens = kUserInitialMaxTokens;
for (int attempt = 0; attempt < max_attempts; ++attempt) {
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kUserJsonGrammar);
raw = this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
attempt + 1, raw)});
}
UserResult user;
@@ -78,8 +77,9 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: successfully generated user data on attempt {}",
attempt + 1)});
.message = std::format("LlamaGenerator: successfully "
"generated user data on attempt {}",
attempt + 1)});
}
user.first_name = name.first_name;
@@ -90,32 +90,34 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message =
std::format("LlamaGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON with "
"exactly these keys, in this exact order: {{\"username\": \"<handle "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"username\": "
"\"<handle "
"derived from the given name>\", \"bio\": \"<first-person bio "
"grounded in the persona>\", \"activity_weight\": <number between 0 "
"grounded in the persona>\", \"activity_weight\": <number between "
"0 "
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user response "
"after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed user response");
}

3
tooling/pipeline/src/data_model/names_by_country.cc
View File

@@ -29,8 +29,7 @@ std::optional<Name> NamesByCountry::SampleName(const std::string& iso3166_1,
const std::vector<ForenameEntry>& forenames = forenames_it->second;
const std::vector<std::string>& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0,
forenames.size() - 1);
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
const ForenameEntry& forename = forenames[forename_dist(rng)];

27
tooling/pipeline/src/json_handling/json_loader.cc
View File

@@ -6,12 +6,10 @@
#include "json_handling/json_loader.h"
#include <format>
#include "services/logging/logger.h"
#include <iostream>
#include <boost/json.hpp>
#include <format>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
@@ -19,6 +17,8 @@
#include <unordered_map>
#include <utility>
#include "services/logging/logger.h"
static std::string ReadRequiredString(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
@@ -45,7 +45,7 @@ static std::vector<std::string> ReadRequiredStringArray(
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_array()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
std::format("Missing or invalid string array field: {}", key));
}
const auto& array = value->as_array();
@@ -67,8 +67,8 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
const char* what) {
std::ifstream input(filepath);
if (!input.is_open()) {
throw std::runtime_error(std::format("Failed to open {} file: {}", what,
filepath.string()));
throw std::runtime_error(
std::format("Failed to open {} file: {}", what, filepath.string()));
}
std::stringstream buffer;
@@ -87,8 +87,7 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
/// `fallback_key` if `key` is missing/empty (some source entries only have a
/// "localized" form and no "romanized" form).
std::string ReadFirstOfStringArray(const boost::json::object& object,
const char* key,
const char* fallback_key) {
const char* key, const char* fallback_key) {
for (const char* candidate_key : {key, fallback_key}) {
const boost::json::value* value = object.if_contains(candidate_key);
if (value == nullptr || !value->is_array()) {
@@ -163,8 +162,7 @@ std::vector<UserPersona> JsonLoader::LoadPersonas(
personas.push_back(UserPersona{
.name = ReadRequiredString(object, "name"),
.description = ReadRequiredString(object, "description"),
.style_affinities =
ReadRequiredStringArray(object, "style_affinities"),
.style_affinities = ReadRequiredStringArray(object, "style_affinities"),
});
}
@@ -207,8 +205,8 @@ NamesByCountry JsonLoader::LoadNamesByCountry(
}
const auto& name_object = name_value.as_object();
entries.push_back(ForenameEntry{
.name = ReadFirstOfStringArray(name_object, "romanized",
"localized"),
.name =
ReadFirstOfStringArray(name_object, "romanized", "localized"),
.gender = ReadRequiredString(name_object, "gender"),
});
}
@@ -216,8 +214,7 @@ NamesByCountry JsonLoader::LoadNamesByCountry(
forenames_by_country.emplace(country_code, std::move(entries));
}
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country;
std::unordered_map<std::string, std::vector<std::string>> surnames_by_country;
for (const auto& [country_code, name_entries] : surnames_root.as_object()) {
if (!name_entries.is_array()) {
continue;

49
tooling/pipeline/src/main.cc
View File

@@ -67,8 +67,8 @@ int main(const int argc, char** argv) {
const LlamaBackendState llama_backend_state;
#endif
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "STARTING PIPELINE"});
const std::optional<ApplicationOptions> parsed_options =
@@ -90,8 +90,8 @@ int main(const int argc, char** argv) {
prompt_directory = std::make_unique<PromptDirectory>(
options.pipeline.prompt_dir, log_producer);
} catch (const std::exception& dir_error) {
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = std::format("Invalid --prompt-dir: {}",
dir_error.what())});
@@ -109,7 +109,7 @@ int main(const int argc, char** argv) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::unique_ptr<IPromptFormatter>(nullptr);
@@ -117,7 +117,7 @@ int main(const int argc, char** argv) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
@@ -126,15 +126,15 @@ int main(const int argc, char** argv) {
di::bind<WebClient>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: none (mock mode)"});
}
return std::unique_ptr<WebClient>(nullptr);
}
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: HttpWebClient"});
}
return std::unique_ptr<WebClient>(
@@ -145,15 +145,15 @@ int main(const int argc, char** argv) {
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: mock"});
}
return std::make_unique<MockEnrichmentService>();
}
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: Wikipedia"});
}
return std::make_unique<WikipediaEnrichmentService>(
@@ -165,8 +165,8 @@ int main(const int argc, char** argv) {
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Generator: mock"});
}
return std::make_unique<MockGenerator>();
@@ -174,9 +174,10 @@ int main(const int argc, char** argv) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | temp={:.2f} "
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} "
"top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
@@ -191,14 +192,14 @@ int main(const int argc, char** argv) {
injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
if (!orchestrator->Run()) {
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = "Pipeline execution failed"});
return shutdown(EXIT_FAILURE);
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = std::format("Pipeline complete in {} ms",
timer.Elapsed())});
@@ -206,8 +207,8 @@ int main(const int argc, char** argv) {
} catch (const std::exception& exception) {
const LogDTO log_entry{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = exception.what()};
.phase = PipelinePhase::Teardown,
.message = exception.what()};
if (log_producer) {
log_producer->Log(log_entry);
} else {

92
tooling/pipeline/src/services/enrichment/wikipedia/fetch_extract.cc
View File

@@ -20,9 +20,10 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (const auto cache_it = this->extract_cache_.find(cache_key);
cache_it != this->extract_cache_.end()) {
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format("Wikipedia: Cache hit for {}!", cache_key)});
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format("Wikipedia: Cache hit for {}!", cache_key)});
}
return cache_it->second;
}
@@ -30,7 +31,8 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
const std::string encoded = this->client_->EncodeURL(cache_key);
const std::string url = std::format(
"https://en.wikipedia.org/w/"
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format=json",
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format="
"json",
encoded);
const std::string body = this->client_->Get(url);
@@ -45,11 +47,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (ec) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: JSON parse error for '{}': {}",
std::string(query), ec.message())});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: JSON parse error for '{}': {}",
std::string(query), ec.message())});
}
return {};
}
@@ -58,12 +60,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
const json::object* obj = doc.if_object();
if (obj == nullptr) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: Expected root object for '{}'",
std::string(query))});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Expected root object for '{}'",
std::string(query))});
}
return {};
}
@@ -76,12 +77,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if ((pages_ptr == nullptr) || !pages_ptr->is_object()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: Missing query.pages for '{}'",
std::string(query))});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Missing query.pages for '{}'",
std::string(query))});
}
return {};
}
@@ -90,11 +90,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (pages.empty()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: No pages returned for '{}'",
std::string(query))});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No pages returned for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
@@ -106,12 +106,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (!page_val.is_object()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: Unexpected page format for '{}'",
std::string(query))});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Unexpected page format for '{}'",
std::string(query))});
}
return {};
}
@@ -121,10 +120,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
// Handle 404/Missing status
if (page.contains("missing")) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Page '{}' does not exist",
std::string(query))});
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Page '{}' does not exist",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
@@ -134,12 +134,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if ((extract_ptr == nullptr) || !extract_ptr->is_string()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: No extract string found for '{}'",
std::string(query))});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No extract string found for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
@@ -148,10 +147,11 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
// 4. Success
std::string extract(extract_ptr->as_string());
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Fetched {} chars for '{}'",
extract.size(), std::string(query))});
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Fetched {} chars for '{}'",
extract.size(), std::string(query))});
}
this->extract_cache_.insert_or_assign(cache_key, extract);

7
tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc
View File

@@ -45,8 +45,9 @@ std::string WikipediaEnrichmentService::GetLocationContext(
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("Done fetching for {}. Sleeping for 10 seconds.",
location_query)});
.message = std::format(
"Done fetching for {}. Sleeping for 10 seconds.",
location_query)});
}
std::this_thread::sleep_for(10s);
@@ -56,7 +57,7 @@ std::string WikipediaEnrichmentService::GetLocationContext(
{.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService lookup failed for '{}': {}",
location_query, e.what())});
location_query, e.what())});
}
}
return result;

21
tooling/pipeline/src/services/prompt_directory.cc
View File

@@ -44,8 +44,8 @@ PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir,
// Scenario 4: directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(
std::format("PromptDirectory: prompt directory is not readable: {} ({})",
throw std::runtime_error(std::format(
"PromptDirectory: prompt directory is not readable: {} ({})",
prompt_dir_.string(), ec.message()));
}
@@ -76,7 +76,7 @@ std::string PromptDirectory::Load(std::string_view key) {
if (!file.is_open()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file not found for key '{}': {}",
key_str, file_path.string()));
key_str, file_path.string()));
}
std::string content((std::istreambuf_iterator<char>(file)),
@@ -84,15 +84,18 @@ std::string PromptDirectory::Load(std::string_view key) {
file.close();
if (content.empty()) {
throw std::runtime_error(std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
key_str, file_path.string()));
throw std::runtime_error(
std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
key_str, file_path.string()));
}
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format("[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
key_str, file_path.string(), content.size())});
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
key_str, file_path.string(), content.size())});
}
cache_.emplace(key_str, content);

6
tooling/pipeline/src/services/sqlite/initialize.cc
View File

@@ -18,9 +18,9 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
std::filesystem::path candidate = output_path_ / base_filename;
for (int suffix = 1; std::filesystem::exists(candidate); ++suffix) {
candidate = output_path_ /
std::filesystem::path(std::format("biergarten_seed_{}-{}.sqlite",
run_timestamp_utc_, suffix));
candidate = output_path_ / std::filesystem::path(
std::format("biergarten_seed_{}-{}.sqlite",
run_timestamp_utc_, suffix));
}
return candidate;

6
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -38,8 +38,7 @@ sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
}
const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(
location.local_languages);
sqlite_export_service_internal::SerializeVector(location.local_languages);
sqlite_export_service_internal::Bind(
insert_location_stmt_,
@@ -92,7 +91,8 @@ sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
.action = "Failed to bind SQLite location longitude"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_location_stmt_, "Failed to insert SQLite location row");
db_handle_, insert_location_stmt_,
"Failed to insert SQLite location row");
const sqlite3_int64 location_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);

17
tooling/pipeline/src/web_client/http_web_client.cc
View File

@@ -48,21 +48,20 @@ std::string HttpWebClient::Get(const std::string& url) {
const httplib::Result result = client.Get(path);
if (!result) {
throw std::runtime_error(std::format(
"[HttpWebClient] Request failed for URL: {} — {}", url,
httplib::to_string(result.error())));
throw std::runtime_error(
std::format("[HttpWebClient] Request failed for URL: {} — {}", url,
httplib::to_string(result.error())));
}
if (result->status < kSuccessMin || result->status >= kSuccessMax) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Error,
.phase = PipelinePhase::Enrichment,
.message =
std::format("[HttpWebClient] Request failed for URL: {}", url)});
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[HttpWebClient] Request failed for URL: {}", url)});
}
throw std::runtime_error(std::format("[HttpWebClient] HTTP {} for URL: {}",
result->status, url));
result->status, url));
}
return result->body;