AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-10-15 01:54:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Formatting

Browse Source
This commit is contained in:
Aaron Po
2026-07-19 04:05:30 -04:00
parent 8e7ecf2fc3
commit ec770aa724
78 changed files with 13388 additions and 32143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

373
tooling/pipeline/src/application_options/parse_arguments.cc
View File

@@ -10,207 +10,212 @@
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger) {
prog_opts::options_description desc("Pipeline Options");
auto opt = desc.add_options();
prog_opts::options_description desc("Pipeline Options");
auto opt = desc.add_options();
opt("help,h", "Produce help message");
opt("help,h", "Produce help message");
auto add_sampling_options = [&]() -> void {
const SamplingOptions sampling_defaults{};
opt("temperature",
prog_opts::value<float>()->default_value(sampling_defaults.temperature),
"Sampling temperature (higher = more random)");
opt("top-p",
prog_opts::value<float>()->default_value(sampling_defaults.top_p),
"Nucleus sampling top-p in (0,1] (higher = more random)");
opt("top-k",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.top_k),
"Top-k sampling parameter (higher = more candidate tokens)");
opt("n-ctx",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.n_ctx),
"Context window size in tokens");
opt("seed", prog_opts::value<int>()->default_value(sampling_defaults.seed),
"Sampler seed: -1 for random, otherwise non-negative integer");
opt("n-gpu-layers", prog_opts::value<int>()->default_value(0),
"Number of layers to offload to GPU");
};
auto add_sampling_options = [&]() -> void {
const SamplingOptions sampling_defaults{};
opt("temperature",
prog_opts::value<float>()->default_value(
sampling_defaults.temperature),
"Sampling temperature (higher = more random)");
opt("top-p",
prog_opts::value<float>()->default_value(sampling_defaults.top_p),
"Nucleus sampling top-p in (0,1] (higher = more random)");
opt("top-k",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.top_k),
"Top-k sampling parameter (higher = more candidate tokens)");
opt("n-ctx",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.n_ctx),
"Context window size in tokens");
opt("seed",
prog_opts::value<int>()->default_value(sampling_defaults.seed),
"Sampler seed: -1 for random, otherwise non-negative integer");
opt("n-gpu-layers", prog_opts::value<int>()->default_value(0),
"Number of layers to offload to GPU");
};
// --mocked and --model are mutually exclusive; validation is enforced below
// rather than at registration to produce a clear diagnostic message.
auto add_generator_options = [&]() -> void {
opt("mocked", prog_opts::bool_switch(),
"Use mocked generator for brewery/user data");
opt("model,m", prog_opts::value<std::string>()->default_value(""),
"Path to LLM model (gguf)");
};
// --mocked and --model are mutually exclusive; validation is enforced below
// rather than at registration to produce a clear diagnostic message.
auto add_generator_options = [&]() -> void {
opt("mocked", prog_opts::bool_switch(),
"Use mocked generator for brewery/user data");
opt("model,m", prog_opts::value<std::string>()->default_value(""),
"Path to LLM model (gguf)");
};
auto add_pipeline_options = [&]() -> void {
opt("output,o", prog_opts::value<std::string>()->default_value("output"),
"Directory for generated artifacts");
opt("log-path",
prog_opts::value<std::string>()->default_value("pipeline.log"),
"Path for application logs");
opt("prompt-dir", prog_opts::value<std::string>()->default_value(""),
"Directory containing named prompt files (e.g. "
"BREWERY_GENERATION.md)."
" Required when not using --mocked.");
opt("location-count", prog_opts::value<uint32_t>()->default_value(10));
};
auto add_pipeline_options = [&]() -> void {
opt("output,o", prog_opts::value<std::string>()->default_value("output"),
"Directory for generated artifacts");
opt("log-path",
prog_opts::value<std::string>()->default_value("pipeline.log"),
"Path for application logs");
opt("prompt-dir", prog_opts::value<std::string>()->default_value(""),
"Directory containing named prompt files (e.g. "
"BREWERY_GENERATION.md)."
" Required when not using --mocked.");
opt("location-count", prog_opts::value<uint32_t>()->default_value(10));
};
add_sampling_options();
add_generator_options();
add_pipeline_options();
add_sampling_options();
add_generator_options();
add_pipeline_options();
// No flags provided — treat as a help request rather than an error.
if (argc == 1) {
const std::string title = "Biergarten Pipeline";
const std::string usage = ([&] {
std::stringstream usage_stream;
usage_stream << "\nUsage: biergarten-pipeline [options]\n\n" << desc;
return usage_stream.str();
})();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = title});
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = usage});
}
return std::nullopt;
}
try {
prog_opts::variables_map var_map;
prog_opts::store(prog_opts::parse_command_line(argc, argv, desc), var_map);
prog_opts::notify(var_map);
if (var_map.contains("help")) {
std::stringstream help_stream;
help_stream << "\n" << desc;
// No flags provided — treat as a help request rather than an error.
if (argc == 1) {
const std::string title = "Biergarten Pipeline";
const std::string usage = ([&] {
std::stringstream usage_stream;
usage_stream << "\nUsage: biergarten-pipeline [options]\n\n" << desc;
return usage_stream.str();
})();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = help_stream.str()});
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = title});
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = usage});
}
return std::nullopt;
}
}
ApplicationOptions options;
try {
prog_opts::variables_map var_map;
prog_opts::store(prog_opts::parse_command_line(argc, argv, desc),
var_map);
prog_opts::notify(var_map);
options.pipeline.output_path = var_map["output"].as<std::string>();
options.pipeline.log_path = var_map["log-path"].as<std::string>();
options.pipeline.prompt_dir = var_map["prompt-dir"].as<std::string>();
options.pipeline.location_count = var_map["location-count"].as<uint32_t>();
const bool use_mocked = var_map["mocked"].as<bool>();
const std::string model_path = var_map["model"].as<std::string>();
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
// Enforce mutual exclusivity before any further configuration is
// applied.
if (use_mocked && !model_path.empty()) {
const std::string msg =
"Invalid arguments: --mocked and --model are mutually "
"exclusive";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
if (var_map.contains("help")) {
std::stringstream help_stream;
help_stream << "\n" << desc;
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = help_stream.str()});
}
return std::nullopt;
}
return std::nullopt;
}
if (!use_mocked && model_path.empty()) {
const std::string msg =
"Invalid arguments: either --mocked or --model must be "
"specified";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
ApplicationOptions options;
options.pipeline.output_path = var_map["output"].as<std::string>();
options.pipeline.log_path = var_map["log-path"].as<std::string>();
options.pipeline.prompt_dir = var_map["prompt-dir"].as<std::string>();
options.pipeline.location_count =
var_map["location-count"].as<uint32_t>();
const bool use_mocked = var_map["mocked"].as<bool>();
const std::string model_path = var_map["model"].as<std::string>();
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
// Enforce mutual exclusivity before any further configuration is
// applied.
if (use_mocked && !model_path.empty()) {
const std::string msg =
"Invalid arguments: --mocked and --model are mutually "
"exclusive";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
return std::nullopt;
}
return std::nullopt;
}
// Prompt directory is only meaningful for live inference — the mock
// generator has no use for it and should not require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using "
"--mocked";
if (logger) {
logger->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
if (!use_mocked && model_path.empty()) {
const std::string msg =
"Invalid arguments: either --mocked or --model must be "
"specified";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
return std::nullopt;
}
return std::nullopt;
}
options.generator.use_mocked = use_mocked;
options.generator.model_path = model_path;
// Only populate sampling config when the user explicitly overrides at
// least one value. Leaving it as std::nullopt lets LlamaGenerator fall
// back to its own SamplingOptions{} defaults, keeping the two paths
// consistent without redundant copies.
const bool user_provided_sampling =
!var_map["temperature"].defaulted() || !var_map["top-p"].defaulted() ||
!var_map["top-k"].defaulted() || !var_map["n-ctx"].defaulted() ||
!var_map["seed"].defaulted() || !var_map["n_gpu_layers"].defaulted();
if (user_provided_sampling) {
// Warn but do not fail — the run is still valid, the flags are just
// silently irrelevant when no model is loaded.
if (use_mocked) {
const std::string msg =
"Sampling parameters are ignored when using --mocked";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Warn,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
} else {
SamplingOptions sampling;
sampling.temperature = var_map["temperature"].as<float>();
sampling.top_p = var_map["top-p"].as<float>();
sampling.top_k = var_map["top-k"].as<uint32_t>();
sampling.n_ctx = var_map["n-ctx"].as<uint32_t>();
sampling.seed = var_map["seed"].as<int>();
sampling.n_gpu_layers = var_map["n-gpu-layers"].as<int>();
options.generator.sampling = sampling;
}
}
return options;
} catch (const std::exception& exception) {
const std::string msg =
std::string("Failed to parse command-line arguments: ") +
exception.what();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
// Prompt directory is only meaningful for live inference — the mock
// generator has no use for it and should not require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using "
"--mocked";
if (logger) {
logger->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
} catch (...) {
const std::string msg =
"Failed to parse command-line arguments: unknown error";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
}
} else {
std::cerr << msg << std::endl;
}
return std::nullopt;
}
options.generator.use_mocked = use_mocked;
options.generator.model_path = model_path;
// Only populate sampling config when the user explicitly overrides at
// least one value. Leaving it as std::nullopt lets LlamaGenerator fall
// back to its own SamplingOptions{} defaults, keeping the two paths
// consistent without redundant copies.
const bool user_provided_sampling =
!var_map["temperature"].defaulted() ||
!var_map["top-p"].defaulted() || !var_map["top-k"].defaulted() ||
!var_map["n-ctx"].defaulted() || !var_map["seed"].defaulted() ||
!var_map["n_gpu_layers"].defaulted();
if (user_provided_sampling) {
// Warn but do not fail — the run is still valid, the flags are just
// silently irrelevant when no model is loaded.
if (use_mocked) {
const std::string msg =
"Sampling parameters are ignored when using --mocked";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Warn,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
} else {
SamplingOptions sampling;
sampling.temperature = var_map["temperature"].as<float>();
sampling.top_p = var_map["top-p"].as<float>();
sampling.top_k = var_map["top-k"].as<uint32_t>();
sampling.n_ctx = var_map["n-ctx"].as<uint32_t>();
sampling.seed = var_map["seed"].as<int>();
sampling.n_gpu_layers = var_map["n-gpu-layers"].as<int>();
options.generator.sampling = sampling;
}
}
return options;
} catch (const std::exception& exception) {
const std::string msg =
std::string("Failed to parse command-line arguments: ") +
exception.what();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
} catch (...) {
const std::string msg =
"Failed to parse command-line arguments: unknown error";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
}
}

141
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc
View File

@@ -12,80 +12,83 @@
void BiergartenPipelineOrchestrator::GenerateBreweries(
std::span<const EnrichedCity> cities) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = "=== SAMPLE BREWERY GENERATION ==="});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = "=== SAMPLE BREWERY GENERATION ==="});
generated_breweries_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
generated_breweries_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
const auto generate_record =
[this, &skipped_count](
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
try {
const BreweryResult brewery = generator_->GenerateBrewery(enriched_city);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(enriched_city.location);
return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code},
.brewery = brewery};
} catch (const std::exception& e) {
++skipped_count;
const auto generate_record =
[this, &skipped_count](
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
try {
const BreweryResult brewery =
generator_->GenerateBrewery(enriched_city);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(enriched_city.location);
return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code},
.brewery = brewery};
} catch (const std::exception& e) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): brewery "
"generation failed: {}",
enriched_city.location.city,
enriched_city.location.country, e.what())});
return std::nullopt;
}
};
const auto export_record =
[this, &export_failed_count](const BreweryRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};
for (const EnrichedCity& enriched_city : cities) {
const std::optional<BreweryRecord> record =
generate_record(enriched_city);
if (!record.has_value()) {
continue;
}
generated_breweries_.push_back(*record);
export_record(*record);
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
"generation failed: {}",
enriched_city.location.city,
enriched_city.location.country, e.what())});
return std::nullopt;
}
};
.message = std::format(
"[Pipeline] Skipped {} city/cities due to generation errors",
skipped_count)});
}
const auto export_record = [this, &export_failed_count](
const BreweryRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};
for (const EnrichedCity& enriched_city : cities) {
const std::optional<BreweryRecord> record = generate_record(enriched_city);
if (!record.has_value()) {
continue;
}
generated_breweries_.push_back(*record);
export_record(*record);
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to generation errors",
skipped_count)});
}
if (export_failed_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} generated "
"brewery/breweries to SQLite",
export_failed_count)});
}
if (export_failed_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} generated "
"brewery/breweries to SQLite",
export_failed_count)});
}
}

320
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -20,203 +20,205 @@
namespace {
std::string Sanitize(std::string_view value) {
std::string out;
out.reserve(value.size());
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
std::string out;
out.reserve(value.size());
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
}
std::string BuildEmail(const Name& name,
std::unordered_set<std::string>& used_local_parts) {
const std::string base =
std::format("{}.{}", Sanitize(name.first_name), Sanitize(name.last_name));
const std::string base = std::format("{}.{}", Sanitize(name.first_name),
Sanitize(name.last_name));
std::string local_part = base;
uint32_t suffix = 1;
std::string local_part = base;
uint32_t suffix = 1;
while (used_local_parts.contains(local_part)) {
local_part = std::format("{}{}", base, suffix);
++suffix;
}
while (used_local_parts.contains(local_part)) {
local_part = std::format("{}{}", base, suffix);
++suffix;
}
used_local_parts.insert(local_part);
used_local_parts.insert(local_part);
return std::format("{}@thebiergarten.app", local_part);
return std::format("{}@thebiergarten.app", local_part);
}
std::string GenerateDateOfBirth(std::mt19937& rng) {
using namespace std::chrono;
using namespace std::chrono;
constexpr int kMinAge = 19;
constexpr int kMaxAge = 48;
constexpr int kMaxDayOffset = 364;
constexpr int kMinAge = 19;
constexpr int kMaxAge = 48;
constexpr int kMaxDayOffset = 364;
std::uniform_int_distribution<int> age_dist(kMinAge, kMaxAge);
std::uniform_int_distribution<int> day_offset_dist(0, kMaxDayOffset);
std::uniform_int_distribution<int> age_dist(kMinAge, kMaxAge);
std::uniform_int_distribution<int> day_offset_dist(0, kMaxDayOffset);
const year_month_day today{floor<days>(system_clock::now())};
const year_month_day birth_year_anchor{today.year() - years{age_dist(rng)},
today.month(), today.day()};
const sys_days birth_date =
sys_days{birth_year_anchor} - days{day_offset_dist(rng)};
const year_month_day birth_ymd{birth_date};
const year_month_day today{floor<days>(system_clock::now())};
const year_month_day birth_year_anchor{today.year() - years{age_dist(rng)},
today.month(), today.day()};
const sys_days birth_date =
sys_days{birth_year_anchor} - days{day_offset_dist(rng)};
const year_month_day birth_ymd{birth_date};
return std::format("{:04}-{:02}-{:02}", static_cast<int>(birth_ymd.year()),
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
return std::format("{:04}-{:02}-{:02}", static_cast<int>(birth_ymd.year()),
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
}
std::optional<Name> SampleName(
const ForenamesByCountryMap& forenames_by_country,
const SurnamesByCountryMap& surnames_by_country,
const std::string& iso3166_1, std::mt19937& rng) {
const auto forenames_it = forenames_by_country.find(iso3166_1);
const auto surnames_it = surnames_by_country.find(iso3166_1);
const auto forenames_it = forenames_by_country.find(iso3166_1);
const auto surnames_it = surnames_by_country.find(iso3166_1);
if (forenames_it == forenames_by_country.end() ||
surnames_it == surnames_by_country.end() ||
forenames_it->second.empty() || surnames_it->second.empty()) {
return std::nullopt;
}
if (forenames_it == forenames_by_country.end() ||
surnames_it == surnames_by_country.end() ||
forenames_it->second.empty() || surnames_it->second.empty()) {
return std::nullopt;
}
const ForenameList& forenames = forenames_it->second;
const SurnameList& surnames = surnames_it->second;
const ForenameList& forenames = forenames_it->second;
const SurnameList& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
auto forename_it = forenames.begin();
std::advance(forename_it, forename_dist(rng));
auto forename_it = forenames.begin();
std::advance(forename_it, forename_dist(rng));
auto surname_it = surnames.begin();
std::advance(surname_it, surname_dist(rng));
auto surname_it = surnames.begin();
std::advance(surname_it, surname_dist(rng));
return Name{.first_name = forename_it->name,
.last_name = *surname_it,
.gender = forename_it->gender};
return Name{.first_name = forename_it->name,
.last_name = *surname_it,
.gender = forename_it->gender};
}
} // namespace
void BiergartenPipelineOrchestrator::GenerateUsers(
std::span<const EnrichedCity> cities) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = "=== SAMPLE USER GENERATION ==="});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = "=== SAMPLE USER GENERATION ==="});
const PersonasList& personas = curated_data_service_->LoadPersonas();
const PersonasList& personas = curated_data_service_->LoadPersonas();
if (personas.empty()) {
throw std::runtime_error(
"No personas available in personas.json for user generation");
}
if (personas.empty()) {
throw std::runtime_error(
"No personas available in personas.json for user generation");
}
const ForenamesByCountryMap& forenames_by_country =
curated_data_service_->LoadForenamesByCountry();
const SurnamesByCountryMap& surnames_by_country =
curated_data_service_->LoadSurnamesByCountry();
const ForenamesByCountryMap& forenames_by_country =
curated_data_service_->LoadForenamesByCountry();
const SurnamesByCountryMap& surnames_by_country =
curated_data_service_->LoadSurnamesByCountry();
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
generated_users_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
std::unordered_set<std::string> used_email_local_parts;
generated_users_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
std::unordered_set<std::string> used_email_local_parts;
const auto generate_record =
[this, &rng, &skipped_count, &used_email_local_parts](
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
try {
const UserResult user =
generator_->GenerateUser(city, persona, sampled_name);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(city.location);
const auto generate_record =
[this, &rng, &skipped_count, &used_email_local_parts](
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
try {
const UserResult user =
generator_->GenerateUser(city, persona, sampled_name);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(city.location);
return UserRecord{
.address =
UserAddress{.city = city.location, .postal_code = postal_code},
.user = user,
.email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng),
};
} catch (const std::exception& e) {
++skipped_count;
return UserRecord{
.address =
UserAddress{.city = city.location, .postal_code = postal_code},
.user = user,
.email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng),
};
} catch (const std::exception& e) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): "
"user generation failed: {}",
city.location.city, city.location.country,
e.what())});
return std::nullopt;
}
};
const auto export_record = [this,
&export_failed_count](const UserRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};
for (const auto& city : cities) {
const std::optional<Name> sampled_name =
SampleName(forenames_by_country, surnames_by_country,
city.location.iso3166_1, rng);
if (!sampled_name.has_value()) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): no names "
"available for country '{}'",
city.location.city, city.location.country,
city.location.iso3166_1)});
continue;
}
const UserPersona& persona = personas[persona_dist(rng)];
const std::optional<UserRecord> record =
generate_record(city, persona, *sampled_name);
if (!record.has_value()) {
continue;
}
generated_users_.push_back(*record);
export_record(*record);
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
}
if (export_failed_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): "
"user generation failed: {}",
city.location.city, city.location.country, e.what())});
return std::nullopt;
}
};
const auto export_record = [this,
&export_failed_count](const UserRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};
for (const auto& city : cities) {
const std::optional<Name> sampled_name =
SampleName(forenames_by_country, surnames_by_country,
city.location.iso3166_1, rng);
if (!sampled_name.has_value()) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): no names "
"available for country '{}'",
city.location.city, city.location.country,
city.location.iso3166_1)});
continue;
}
const UserPersona& persona = personas[persona_dist(rng)];
const std::optional<UserRecord> record =
generate_record(city, persona, *sampled_name);
if (!record.has_value()) {
continue;
}
generated_users_.push_back(*record);
export_record(*record);
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
}
if (export_failed_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} "
"generated user/users to SQLite",
export_failed_count)});
}
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} "
"generated user/users to SQLite",
export_failed_count)});
}
}

78
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -12,48 +12,48 @@
#include "services/logging/logger.h"
void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array brewery_output;
boost::json::array brewery_output;
for (const auto& [address, brewery] : generated_breweries_) {
const City& city = address.city;
brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"address", boost::json::object{
{"city", city.city},
{"country", city.country},
{"state_province", city.state_province},
{"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
}}});
}
for (const auto& [address, brewery] : generated_breweries_) {
const City& city = address.city;
brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"address", boost::json::object{
{"city", city.city},
{"country", city.country},
{"state_province", city.state_province},
{"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
}}});
}
std::ostringstream brewery_oss;
PrettyPrint(brewery_oss, brewery_output);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = brewery_oss.str()});
std::ostringstream brewery_oss;
PrettyPrint(brewery_oss, brewery_output);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = brewery_oss.str()});
boost::json::array user_output;
boost::json::array user_output;
for (const auto& generated_user : generated_users_) {
user_output.push_back(boost::json::object{
{"first_name", generated_user.user.first_name},
{"last_name", generated_user.user.last_name},
{"gender", generated_user.user.gender},
{"username", generated_user.user.username},
{"bio", generated_user.user.bio},
{"activity_weight", generated_user.user.activity_weight},
{"email", generated_user.email},
{"date_of_birth", generated_user.date_of_birth},
});
}
for (const auto& generated_user : generated_users_) {
user_output.push_back(boost::json::object{
{"first_name", generated_user.user.first_name},
{"last_name", generated_user.user.last_name},
{"gender", generated_user.user.gender},
{"username", generated_user.user.username},
{"bio", generated_user.user.bio},
{"activity_weight", generated_user.user.activity_weight},
{"email", generated_user.email},
{"date_of_birth", generated_user.date_of_birth},
});
}
std::ostringstream user_oss;
PrettyPrint(user_oss, user_output);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = user_oss.str()});
std::ostringstream user_oss;
PrettyPrint(user_oss, user_output);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = user_oss.str()});
}

53
tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
View File

@@ -14,37 +14,36 @@
#include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h"
std::vector<City>
BiergartenPipelineOrchestrator::QueryLocations() {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
std::vector<City> BiergartenPipelineOrchestrator::QueryLocations() {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
const std::vector<City>& all_locations =
curated_data_service_->LoadLocations();
const std::vector<City>& all_locations =
curated_data_service_->LoadLocations();
const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count),
all_locations.size());
const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count),
all_locations.size());
const auto sample_count_signed =
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count);
const auto sample_count_signed =
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count);
std::vector<City> sampled_locations;
sampled_locations.reserve(sample_count);
std::vector<City> sampled_locations;
sampled_locations.reserve(sample_count);
std::random_device random_generator;
std::ranges::sample(all_locations, std::back_inserter(sampled_locations),
sample_count_signed, random_generator);
std::random_device random_generator;
std::ranges::sample(all_locations, std::back_inserter(sampled_locations),
sample_count_signed, random_generator);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Locations available: {}",
all_locations.size())});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Sampled locations: {}",
sampled_locations.size())});
return sampled_locations;
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Locations available: {}",
all_locations.size())});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Sampled locations: {}",
sampled_locations.size())});
return sampled_locations;
}

83
tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc
View File

@@ -11,51 +11,52 @@
#include "services/logging/logger.h"
bool BiergartenPipelineOrchestrator::Run() {
try {
exporter_->Initialize();
try {
exporter_->Initialize();
std::vector<City> cities = QueryLocations();
std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size());
std::vector<City> cities = QueryLocations();
std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size());
size_t skipped_count = 0;
for (auto& city : cities) {
try {
std::string region_context = context_service_->GetLocationContext(city);
size_t skipped_count = 0;
for (auto& city : cities) {
try {
std::string region_context =
context_service_->GetLocationContext(city);
enriched.push_back(
EnrichedCity{.location = std::move(city),
.region_context = std::move(region_context)});
} catch (const std::exception& exception) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): context "
"lookup failed: {}",
city.city, city.country, exception.what())});
enriched.push_back(
EnrichedCity{.location = std::move(city),
.region_context = std::move(region_context)});
} catch (const std::exception& exception) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): context "
"lookup failed: {}",
city.city, city.country, exception.what())});
}
}
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("[Pipeline] Skipped {} city/cities due "
"to context lookup errors",
skipped_count)});
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("[Pipeline] Skipped {} city/cities due "
"to context lookup errors",
skipped_count)});
}
this->GenerateUsers(enriched);
this->GenerateBreweries(enriched);
exporter_->Finalize();
this->LogResults();
return true;
} catch (const std::exception& e) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = std::format(
"Pipeline execution failed with error: {}", e.what())});
return false;
}
this->GenerateUsers(enriched);
this->GenerateBreweries(enriched);
exporter_->Finalize();
this->LogResults();
return true;
} catch (const std::exception& e) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = std::format(
"Pipeline execution failed with error: {}", e.what())});
return false;
}
}

236
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -18,144 +18,144 @@
static std::string FormatLocalLanguageCodes(
const std::vector<std::string>& codes) {
if (codes.empty()) {
return "Not provided";
}
if (codes.empty()) {
return "Not provided";
}
std::string formatted;
for (const std::string& code : codes) {
if (!formatted.empty()) {
formatted += ", ";
}
formatted += code;
}
std::string formatted;
for (const std::string& code : codes) {
if (!formatted.empty()) {
formatted += ", ";
}
formatted += code;
}
return formatted;
return formatted;
}
static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery(
const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const std::string& region_context = enriched_city.region_context;
/**
* Preprocess and truncate region context to manageable size
*/
const std::string safe_region_context = PrepareRegionContext(region_context);
const City& location = enriched_city.location;
const std::string& region_context = enriched_city.region_context;
/**
* Preprocess and truncate region context to manageable size
*/
const std::string safe_region_context = PrepareRegionContext(region_context);
const std::string local_language_codes =
FormatLocalLanguageCodes(location.local_languages);
const std::string local_language_codes =
FormatLocalLanguageCodes(location.local_languages);
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
/**
* Load brewery system prompt via the injected prompt directory.
* The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside
* the configured --prompt-dir. Throws on missing or empty file.
*/
const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION");
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
/**
* Load brewery system prompt via the injected prompt directory.
* The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside
* the configured --prompt-dir. Throws on missing or empty file.
*/
const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION");
std::string user_prompt = std::format(
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
"CONTEXT:\n{}",
location.city, location.country, local_language_codes,
safe_region_context);
std::string user_prompt = std::format(
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
"CONTEXT:\n{}",
location.city, location.country, local_language_codes,
safe_region_context);
/**
* Store location context for retry prompts (without repeating full context)
*/
const std::string retry_location =
std::format("City: {}{}\nLocal language codes: {}", location.city,
country_suffix, local_language_codes);
/**
* Store location context for retry prompts (without repeating full context)
*/
const std::string retry_location =
std::format("City: {}{}\nLocal language codes: {}", location.city,
country_suffix, local_language_codes);
/**
* RETRY LOOP with validation and error correction
* Attempts to generate valid brewery data up to 3 times, with
* feedback-based refinement
*/
constexpr int max_attempts = 3;
std::string raw;
std::string last_error;
/**
* RETRY LOOP with validation and error correction
* Attempts to generate valid brewery data up to 3 times, with
* feedback-based refinement
*/
constexpr int max_attempts = 3;
std::string raw;
std::string last_error;
// Token budget: too small risks truncating valid JSON mid-string.
int max_tokens = kBreweryInitialMaxTokens;
// Limit output length to keep it concise and focused
for (int attempt = 0; attempt < max_attempts; ++attempt) {
// Generate brewery data from LLM
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kBreweryJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
// Validate output: parse JSON and check required fields
BreweryResult brewery;
const std::optional<std::string> validation_error =
ValidateBreweryJson(raw, brewery);
if (!validation_error.has_value()) {
// Success: return parsed brewery data
// Token budget: too small risks truncating valid JSON mid-string.
int max_tokens = kBreweryInitialMaxTokens;
// Limit output length to keep it concise and focused
for (int attempt = 0; attempt < max_attempts; ++attempt) {
// Generate brewery data from LLM
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kBreweryJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: successfully generated "
"brewery data on attempt {}",
attempt + 1)});
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
return brewery;
}
// Validate output: parse JSON and check required fields
// Validation failed: log error and prepare corrective feedback
BreweryResult brewery;
const std::optional<std::string> validation_error =
ValidateBreweryJson(raw, brewery);
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
if (!validation_error.has_value()) {
// Success: return parsed brewery data
// Update prompt with error details to guide LLM toward correct output.
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"name_en\": "
"\"<English "
"brewery name>\", \"description_en\": \"<English single-paragraph "
"description>\", \"name_local\": \"<local-language brewery "
"name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, "
"or "
"literal placeholder values.\n\nKeep the JSON strings concise "
"enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: successfully generated "
"brewery data on attempt {}",
attempt + 1)});
}
// All retry attempts exhausted: log failure and throw exception
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery "
"response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed brewery response");
return brewery;
}
// Validation failed: log error and prepare corrective feedback
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
// Update prompt with error details to guide LLM toward correct output.
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"name_en\": "
"\"<English "
"brewery name>\", \"description_en\": \"<English single-paragraph "
"description>\", \"name_local\": \"<local-language brewery "
"name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, "
"or "
"literal placeholder values.\n\nKeep the JSON strings concise "
"enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
// All retry attempts exhausted: log failure and throw exception
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery "
"response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed brewery response");
}

166
tooling/pipeline/src/data_generation/llama/generate_user.cc
View File

@@ -19,97 +19,99 @@ static constexpr int kUserInitialMaxTokens = 1200;
UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
const UserPersona& persona,
const Name& name) {
std::string style_affinities;
for (const std::string& style : persona.style_affinities) {
if (!style_affinities.empty()) {
style_affinities += ", ";
}
style_affinities += style;
}
std::string style_affinities;
for (const std::string& style : persona.style_affinities) {
if (!style_affinities.empty()) {
style_affinities += ", ";
}
style_affinities += style;
}
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
std::string user_prompt = std::format(
"## NAME:\n\n{} {}\n\n"
"## GENDER:\n\n{}\n\n"
"## CITY:\n\n{}\n\n"
"## COUNTRY:\n\n{}\n\n"
"## PERSONA:\n\n{}\n\n"
"## PERSONA DESCRIPTION:\n\n{}\n\n"
"## STYLE AFFINITIES:\n\n{}",
name.first_name, name.last_name, name.gender, city.location.city,
city.location.country, persona.name, persona.description,
style_affinities);
std::string user_prompt = std::format(
"## NAME:\n\n{} {}\n\n"
"## GENDER:\n\n{}\n\n"
"## CITY:\n\n{}\n\n"
"## COUNTRY:\n\n{}\n\n"
"## PERSONA:\n\n{}\n\n"
"## PERSONA DESCRIPTION:\n\n{}\n\n"
"## STYLE AFFINITIES:\n\n{}",
name.first_name, name.last_name, name.gender, city.location.city,
city.location.country, persona.name, persona.description,
style_affinities);
const std::string retry_context = std::format(
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name, name.last_name,
city.location.city, city.location.country, persona.name);
const std::string retry_context = std::format(
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
name.last_name, city.location.city, city.location.country, persona.name);
constexpr int max_attempts = 3;
std::string raw;
std::string last_error;
int max_tokens = kUserInitialMaxTokens;
constexpr int max_attempts = 3;
std::string raw;
std::string last_error;
int max_tokens = kUserInitialMaxTokens;
for (int attempt = 0; attempt < max_attempts; ++attempt) {
raw = this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
UserResult user;
const std::optional<std::string> validation_error =
ValidateUserJson(raw, user);
if (!validation_error.has_value()) {
for (int attempt = 0; attempt < max_attempts; ++attempt) {
raw =
this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: successfully "
"generated user data on attempt {}",
attempt + 1)});
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
user.first_name = name.first_name;
user.last_name = name.last_name;
user.gender = name.gender;
return user;
}
UserResult user;
const std::optional<std::string> validation_error =
ValidateUserJson(raw, user);
last_error = *validation_error;
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
if (!validation_error.has_value()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: successfully "
"generated user data on attempt {}",
attempt + 1)});
}
user.first_name = name.first_name;
user.last_name = name.last_name;
user.gender = name.gender;
return user;
}
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"username\": "
"\"<handle "
"derived from the given name>\", \"bio\": \"<first-person bio "
"grounded in the persona>\", \"activity_weight\": <number between "
"0 "
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"username\": "
"\"<handle "
"derived from the given name>\", \"bio\": \"<first-person bio "
"grounded in the persona>\", \"activity_weight\": <number between "
"0 "
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user response "
"after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed user response");
"LlamaGenerator: malformed user response "
"after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed user response");
}

329
tooling/pipeline/src/data_generation/llama/helpers.cc
View File

@@ -4,6 +4,8 @@
* parsing, token decoding, and JSON validation helpers for Llama modules.
*/
#include <llama.h>
#include <algorithm>
#include <array>
#include <boost/json.hpp>
@@ -15,8 +17,6 @@
#include <string_view>
#include <vector>
#include <llama.h>
#include "data_generation/llama_generator_helpers.h"
namespace {
@@ -24,14 +24,14 @@ namespace {
* String trimming: removes leading and trailing whitespace
*/
std::string Trim(std::string_view value) {
constexpr std::string_view whitespace = " \t\n\r\f\v";
const size_t first_index = value.find_first_not_of(whitespace);
if (first_index == std::string_view::npos) {
return {};
}
constexpr std::string_view whitespace = " \t\n\r\f\v";
const size_t first_index = value.find_first_not_of(whitespace);
if (first_index == std::string_view::npos) {
return {};
}
const size_t last_index = value.find_last_not_of(whitespace);
return std::string(value.substr(first_index, last_index - first_index + 1));
const size_t last_index = value.find_last_not_of(whitespace);
return std::string(value.substr(first_index, last_index - first_index + 1));
}
/**
@@ -39,26 +39,26 @@ std::string Trim(std::string_view value) {
* spaces
*/
std::string CondenseWhitespace(std::string_view text) {
std::string out;
out.reserve(text.size());
std::string out;
out.reserve(text.size());
bool pending_space = false;
for (const char chr : text) {
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
if (!out.empty()) {
pending_space = true;
bool pending_space = false;
for (const char chr : text) {
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
if (!out.empty()) {
pending_space = true;
}
continue;
}
continue;
}
if (pending_space) {
out.push_back(' ');
pending_space = false;
}
out.push_back(chr);
}
if (pending_space) {
out.push_back(' ');
pending_space = false;
}
out.push_back(chr);
}
return out;
return out;
}
// Guard against truncating in the first half of the string.
@@ -69,30 +69,30 @@ constexpr size_t kTruncationGuardDivisor = 2;
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
std::string_view key, std::string& out,
std::string* error_out) {
const boost::json::value* field = obj.if_contains(key);
if (field == nullptr || !field->is_string()) {
return false;
}
const boost::json::value* field = obj.if_contains(key);
if (field == nullptr || !field->is_string()) {
return false;
}
const auto& string_value = field->as_string();
out = Trim(std::string_view(string_value.data(), string_value.size()));
return !out.empty();
const auto& string_value = field->as_string();
out = Trim(std::string_view(string_value.data(), string_value.size()));
return !out.empty();
}
bool HasSchemaPlaceholder(std::span<std::string* const> values) {
for (const std::string* value : values) {
std::string lowered = *value;
std::ranges::transform(lowered, lowered.begin(),
[](const unsigned char character) {
return static_cast<char>(std::tolower(character));
});
for (const std::string* value : values) {
std::string lowered = *value;
std::ranges::transform(
lowered, lowered.begin(), [](const unsigned char character) {
return static_cast<char>(std::tolower(character));
});
if (lowered == "string") {
return true;
}
}
if (lowered == "string") {
return true;
}
}
return false;
return false;
}
} // namespace
@@ -102,166 +102,167 @@ bool HasSchemaPlaceholder(std::span<std::string* const> values) {
*/
std::string PrepareRegionContext(std::string_view region_context,
const size_t max_chars) {
std::string normalized = CondenseWhitespace(region_context);
if (normalized.size() <= max_chars) {
return normalized;
}
std::string normalized = CondenseWhitespace(region_context);
if (normalized.size() <= max_chars) {
return normalized;
}
normalized.resize(max_chars);
const size_t last_space = normalized.find_last_of(' ');
if (last_space != std::string::npos &&
last_space > max_chars / kTruncationGuardDivisor) {
normalized.resize(last_space);
}
normalized.resize(max_chars);
const size_t last_space = normalized.find_last_of(' ');
if (last_space != std::string::npos &&
last_space > max_chars / kTruncationGuardDivisor) {
normalized.resize(last_space);
}
normalized += "...";
return normalized;
normalized += "...";
return normalized;
}
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
std::string& output) {
constexpr size_t initial_buffer_size = 256;
constexpr size_t initial_buffer_size = 256;
std::array<char, initial_buffer_size> buffer{};
std::array<char, initial_buffer_size> buffer{};
// serialize the sampled token into UTF-8 bytes
// serialize the sampled token into UTF-8 bytes
auto buffer_too_small = [](int32_t result) -> bool { return result < 0; };
auto buffer_too_small = [](int32_t result) -> bool { return result < 0; };
int32_t bytes =
llama_token_to_piece(vocab, token, buffer.data(), buffer.size(), 0, true);
int32_t bytes = llama_token_to_piece(vocab, token, buffer.data(),
buffer.size(), 0, true);
if (!buffer_too_small(bytes)) {
// Append the decoded bytes from the stack buffer.
output.append(buffer.data(), static_cast<size_t>(bytes));
return;
}
if (!buffer_too_small(bytes)) {
// Append the decoded bytes from the stack buffer.
output.append(buffer.data(), static_cast<size_t>(bytes));
return;
}
const int32_t required_size = -bytes;
std::vector<char> dynamic_buffer(static_cast<size_t>(required_size));
const int32_t required_size = -bytes;
std::vector<char> dynamic_buffer(static_cast<size_t>(required_size));
// Retry token decoding against the larger heap buffer.
bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(),
static_cast<int32_t>(dynamic_buffer.size()), 0,
true);
// Retry token decoding against the larger heap buffer.
bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(),
static_cast<int32_t>(dynamic_buffer.size()), 0,
true);
if (!buffer_too_small(bytes)) {
output.append(dynamic_buffer.data(), static_cast<size_t>(bytes));
return;
}
if (!buffer_too_small(bytes)) {
output.append(dynamic_buffer.data(), static_cast<size_t>(bytes));
return;
}
throw std::runtime_error(
"LlamaGenerator: failed to decode sampled token piece");
throw std::runtime_error(
"LlamaGenerator: failed to decode sampled token piece");
}
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 4) {
return "JSON object must contain exactly four keys";
}
const auto& obj = json_value.get_object();
if (obj.size() != 4) {
return "JSON object must contain exactly four keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
&validation_error)) {
return validation_error;
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_en",
brewery_out.description_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_en",
brewery_out.description_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "name_local", brewery_out.name_local,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(
obj, "name_local", brewery_out.name_local, &validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_local",
brewery_out.description_local,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_local",
brewery_out.description_local,
&validation_error)) {
return validation_error;
}
const std::array schema_placeholders = {
&brewery_out.name_en, &brewery_out.description_en,
&brewery_out.name_local, &brewery_out.description_local};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
const std::array schema_placeholders = {
&brewery_out.name_en, &brewery_out.description_en,
&brewery_out.name_local, &brewery_out.description_local};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
return std::nullopt;
}
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 3) {
return "JSON object must contain exactly three keys";
}
const auto& obj = json_value.get_object();
if (obj.size() != 3) {
return "JSON object must contain exactly three keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
&validation_error)) {
return validation_error;
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
&validation_error)) {
return validation_error;
}
const boost::json::value* activity_weight_field =
obj.if_contains("activity_weight");
if (activity_weight_field == nullptr || !activity_weight_field->is_number()) {
return "Missing or invalid numeric field: activity_weight";
}
const boost::json::value* activity_weight_field =
obj.if_contains("activity_weight");
if (activity_weight_field == nullptr ||
!activity_weight_field->is_number()) {
return "Missing or invalid numeric field: activity_weight";
}
const double activity_weight = activity_weight_field->to_number<double>();
if (activity_weight < 0.0 || activity_weight > 1.0) {
return "activity_weight must be between 0 and 1";
}
user_out.activity_weight = static_cast<float>(activity_weight);
const double activity_weight = activity_weight_field->to_number<double>();
if (activity_weight < 0.0 || activity_weight > 1.0) {
return "activity_weight must be between 0 and 1";
}
user_out.activity_weight = static_cast<float>(activity_weight);
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
return std::nullopt;
}

342
tooling/pipeline/src/data_generation/llama/infer.cc
View File

@@ -5,6 +5,8 @@
* output tokens back to text for system+user chat prompts.
*/
#include <llama.h>
#include <algorithm>
#include <chrono>
#include <format>
@@ -14,8 +16,6 @@
#include <string_view>
#include <vector>
#include <llama.h>
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
@@ -30,48 +30,49 @@ using SamplerHandle =
std::unique_ptr<llama_sampler, decltype(&llama_sampler_free)>;
struct SamplerConfig {
float temperature;
uint32_t top_k;
float top_p;
uint32_t seed;
float temperature;
uint32_t top_k;
float top_p;
uint32_t seed;
};
SamplerHandle MakeSamplerChain(const llama_vocab* vocab,
const SamplerConfig& config,
std::string_view grammar) {
const llama_sampler_chain_params sampler_params =
llama_sampler_chain_default_params();
const llama_sampler_chain_params sampler_params =
llama_sampler_chain_default_params();
SamplerHandle chain(llama_sampler_chain_init(sampler_params),
llama_sampler_free);
if (!chain) {
throw std::runtime_error("LlamaGenerator: failed to initialize sampler");
}
SamplerHandle chain(llama_sampler_chain_init(sampler_params),
llama_sampler_free);
if (!chain) {
throw std::runtime_error("LlamaGenerator: failed to initialize sampler");
}
auto add_sampler = [&](llama_sampler* sampler, const char* error_message) {
if (sampler == nullptr) {
throw std::runtime_error(error_message);
}
auto add_sampler = [&](llama_sampler* sampler, const char* error_message) {
if (sampler == nullptr) {
throw std::runtime_error(error_message);
}
llama_sampler_chain_add(chain.get(), sampler);
};
llama_sampler_chain_add(chain.get(), sampler);
};
if (!grammar.empty()) {
const std::string grammar_text(grammar);
add_sampler(llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"),
"LlamaGenerator: failed to initialize grammar sampler");
}
if (!grammar.empty()) {
const std::string grammar_text(grammar);
add_sampler(
llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"),
"LlamaGenerator: failed to initialize grammar sampler");
}
add_sampler(llama_sampler_init_temp(config.temperature),
"LlamaGenerator: failed to initialize temperature sampler");
add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)),
"LlamaGenerator: failed to initialize top-k sampler");
add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep),
"LlamaGenerator: failed to initialize top-p sampler");
add_sampler(llama_sampler_init_dist(config.seed),
"LlamaGenerator: failed to initialize distribution sampler");
add_sampler(llama_sampler_init_temp(config.temperature),
"LlamaGenerator: failed to initialize temperature sampler");
add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)),
"LlamaGenerator: failed to initialize top-k sampler");
add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep),
"LlamaGenerator: failed to initialize top-p sampler");
add_sampler(llama_sampler_init_dist(config.seed),
"LlamaGenerator: failed to initialize distribution sampler");
return chain;
return chain;
}
} // namespace
@@ -80,163 +81,164 @@ std::string LlamaGenerator::Infer(const std::string& system_prompt,
const std::string& prompt,
const int max_tokens,
std::string_view grammar) {
return InferFormatted(prompt_formatter_->Format(system_prompt, prompt),
max_tokens, grammar);
return InferFormatted(prompt_formatter_->Format(system_prompt, prompt),
max_tokens, grammar);
}
std::string LlamaGenerator::InferFormatted(const std::string& formatted_prompt,
const int max_tokens,
std::string_view grammar) {
/**
* Validate that model and context are loaded
*/
if (!model_ || !context_) {
throw std::runtime_error("LlamaGenerator: model not loaded");
}
/**
* Validate that model and context are loaded
*/
if (!model_ || !context_) {
throw std::runtime_error("LlamaGenerator: model not loaded");
}
/**
* Get vocabulary for tokenization and token-to-text conversion
*/
const llama_vocab* vocab = llama_model_get_vocab(model_.get());
if (vocab == nullptr) {
throw std::runtime_error("LlamaGenerator: vocab unavailable");
}
/**
* Get vocabulary for tokenization and token-to-text conversion
*/
const llama_vocab* vocab = llama_model_get_vocab(model_.get());
if (vocab == nullptr) {
throw std::runtime_error("LlamaGenerator: vocab unavailable");
}
const SamplerConfig sampler_config{
.temperature = sampling_temperature_,
.top_k = sampling_top_k_,
.top_p = sampling_top_p_,
.seed = static_cast<uint32_t>(rng_()),
};
const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar);
const SamplerConfig sampler_config{
.temperature = sampling_temperature_,
.top_k = sampling_top_k_,
.top_p = sampling_top_p_,
.seed = static_cast<uint32_t>(rng_()),
};
const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar);
/**
* Clear KV cache to ensure clean inference state (no residual context)
*/
llama_memory_clear(llama_get_memory(context_.get()), true);
/**
* Clear KV cache to ensure clean inference state (no residual context)
*/
llama_memory_clear(llama_get_memory(context_.get()), true);
/**
* TOKENIZATION PHASE
* Convert text prompt into token IDs (integers) that the model understands
*/
std::vector<llama_token> prompt_tokens(formatted_prompt.size() +
kPromptTokenSlack);
/**
* TOKENIZATION PHASE
* Convert text prompt into token IDs (integers) that the model understands
*/
std::vector<llama_token> prompt_tokens(formatted_prompt.size() +
kPromptTokenSlack);
int32_t token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
int32_t token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
/**
* If buffer too small, negative return indicates required size
*/
if (token_count < 0) {
prompt_tokens.resize(static_cast<size_t>(-token_count));
/**
* If buffer too small, negative return indicates required size
*/
if (token_count < 0) {
prompt_tokens.resize(static_cast<size_t>(-token_count));
token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
}
token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
}
if (token_count < 0) {
throw std::runtime_error("LlamaGenerator: prompt tokenization failed");
}
if (token_count < 0) {
throw std::runtime_error("LlamaGenerator: prompt tokenization failed");
}
/**
* CONTEXT SIZE VALIDATION
* Validate and compute effective token budgets based on context window
* constraints
*/
const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get()));
const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get()));
if (n_ctx <= 1 || n_batch <= 0) {
throw std::runtime_error("LlamaGenerator: invalid context or batch size");
}
/**
* CONTEXT SIZE VALIDATION
* Validate and compute effective token budgets based on context window
* constraints
*/
const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get()));
const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get()));
if (n_ctx <= 1 || n_batch <= 0) {
throw std::runtime_error("LlamaGenerator: invalid context or batch size");
}
/**
* Clamp generation limit to available context window, reserve space for
* output
*/
const int32_t effective_max_tokens =
std::max(1, std::min(max_tokens, n_ctx - 1));
/**
* Clamp generation limit to available context window, reserve space for
* output
*/
const int32_t effective_max_tokens =
std::max(1, std::min(max_tokens, n_ctx - 1));
/**
* Prompt can use remaining context after reserving space for generation
*/
int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens);
prompt_budget = std::max<int32_t>(1, prompt_budget);
/**
* Prompt can use remaining context after reserving space for generation
*/
int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens);
prompt_budget = std::max<int32_t>(1, prompt_budget);
/**
* Truncate prompt if necessary to fit within constraints
*/
prompt_tokens.resize(static_cast<size_t>(token_count));
if (token_count > prompt_budget) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: prompt too long ({} tokens), "
"truncating to {} tokens to fit n_batch/n_ctx limits",
token_count, prompt_budget)});
}
prompt_tokens.resize(static_cast<size_t>(prompt_budget));
token_count = prompt_budget;
}
/**
* Truncate prompt if necessary to fit within constraints
*/
prompt_tokens.resize(static_cast<size_t>(token_count));
if (token_count > prompt_budget) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: prompt too long ({} tokens), "
"truncating to {} tokens to fit n_batch/n_ctx limits",
token_count, prompt_budget)});
}
prompt_tokens.resize(static_cast<size_t>(prompt_budget));
token_count = prompt_budget;
}
/**
* PROMPT PROCESSING PHASE
* Create a batch containing all prompt tokens and feed through the model
* This computes internal representations and fills the KV cache
*/
const llama_batch prompt_batch = llama_batch_get_one(
prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size()));
if (llama_decode(context_.get(), prompt_batch) != 0) {
throw std::runtime_error("LlamaGenerator: prompt decode failed");
}
/**
* PROMPT PROCESSING PHASE
* Create a batch containing all prompt tokens and feed through the model
* This computes internal representations and fills the KV cache
*/
const llama_batch prompt_batch = llama_batch_get_one(
prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size()));
if (llama_decode(context_.get(), prompt_batch) != 0) {
throw std::runtime_error("LlamaGenerator: prompt decode failed");
}
/**
* TOKEN GENERATION LOOP
* Iteratively generate tokens one at a time until max_tokens or
* end-of-sequence
*/
std::vector<llama_token> generated_tokens;
generated_tokens.reserve(static_cast<size_t>(effective_max_tokens));
/**
* TOKEN GENERATION LOOP
* Iteratively generate tokens one at a time until max_tokens or
* end-of-sequence
*/
std::vector<llama_token> generated_tokens;
generated_tokens.reserve(static_cast<size_t>(effective_max_tokens));
for (int i = 0; i < effective_max_tokens; ++i) {
/**
* Sample next token using configured sampler chain and model logits
* Index -1 means use the last output position from previous batch
*/
const llama_token next =
llama_sampler_sample(sampler.get(), context_.get(), -1);
/**
* Stop if model predicts end-of-generation token (EOS/EOT)
*/
if (llama_vocab_is_eog(vocab, next)) {
break;
}
generated_tokens.push_back(next);
/**
* Feed the sampled token back into model for next iteration
* (autoregressive)
*/
llama_token decode_token = next;
const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1);
if (llama_decode(context_.get(), one_token_batch) != 0) {
throw std::runtime_error(
"LlamaGenerator: decode failed during generation");
}
}
for (int i = 0; i < effective_max_tokens; ++i) {
/**
* Sample next token using configured sampler chain and model logits
* Index -1 means use the last output position from previous batch
*/
const llama_token next =
llama_sampler_sample(sampler.get(), context_.get(), -1);
/**
* Stop if model predicts end-of-generation token (EOS/EOT)
*/
if (llama_vocab_is_eog(vocab, next)) {
break;
}
generated_tokens.push_back(next);
/**
* Feed the sampled token back into model for next iteration
* (autoregressive)
*/
llama_token decode_token = next;
const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1);
if (llama_decode(context_.get(), one_token_batch) != 0) {
throw std::runtime_error(
"LlamaGenerator: decode failed during generation");
}
}
/**
* DETOKENIZATION PHASE
* Convert generated token IDs back to text using vocabulary
*/
std::string output;
for (const llama_token token : generated_tokens) {
AppendTokenPiece(vocab, token, output);
}
/**
* DETOKENIZATION PHASE
* Convert generated token IDs back to text using vocabulary
*/
std::string output;
for (const llama_token token : generated_tokens) {
AppendTokenPiece(vocab, token, output);
}
return output;
return output;
}

102
tooling/pipeline/src/data_generation/llama/llama_generator.cc
View File

@@ -5,30 +5,30 @@
#include "data_generation/llama_generator.h"
#include <llama.h>
#include <filesystem>
#include <memory>
#include <random>
#include <stdexcept>
#include <string>
#include <llama.h>
#include "data_model/models.h"
static constexpr uint32_t kMaxContextSize = 32768U;
void LlamaGenerator::ModelDeleter::operator()(
llama_model* model) const noexcept {
if (model != nullptr) {
llama_model_free(model);
}
if (model != nullptr) {
llama_model_free(model);
}
}
void LlamaGenerator::ContextDeleter::operator()(
llama_context* context) const noexcept {
if (context != nullptr) {
llama_free(context);
}
if (context != nullptr) {
llama_free(context);
}
}
LlamaGenerator::LlamaGenerator(
@@ -40,61 +40,61 @@ LlamaGenerator::LlamaGenerator(
logger_(std::move(logger)),
prompt_formatter_(std::move(prompt_formatter)),
prompt_directory_(std::move(prompt_directory)) {
if (model_path.empty()) {
throw std::runtime_error("LlamaGenerator: model path must not be empty");
}
if (model_path.empty()) {
throw std::runtime_error("LlamaGenerator: model path must not be empty");
}
if (!prompt_formatter_) {
throw std::runtime_error(
"LlamaGenerator: prompt formatter dependency must not be null");
}
if (!prompt_formatter_) {
throw std::runtime_error(
"LlamaGenerator: prompt formatter dependency must not be null");
}
if (!prompt_directory_) {
throw std::runtime_error(
"LlamaGenerator: prompt directory dependency must not be null");
}
if (!prompt_directory_) {
throw std::runtime_error(
"LlamaGenerator: prompt directory dependency must not be null");
}
const auto sampling = options.generator.sampling.value_or(SamplingOptions{});
const auto sampling = options.generator.sampling.value_or(SamplingOptions{});
if (sampling.temperature < 0.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling temperature must be >= 0");
}
if (sampling.temperature < 0.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling temperature must be >= 0");
}
if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling top-p must be in (0, 1]");
}
if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling top-p must be in (0, 1]");
}
if (sampling.top_k == 0U) {
throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0");
}
if (sampling.top_k == 0U) {
throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0");
}
if (sampling.seed < -1) {
throw std::runtime_error(
"LlamaGenerator: seed must be >= 0, or -1 for random");
}
if (sampling.seed < -1) {
throw std::runtime_error(
"LlamaGenerator: seed must be >= 0, or -1 for random");
}
if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) {
throw std::runtime_error(
"LlamaGenerator: context size must be in range [1, 32768]");
}
if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) {
throw std::runtime_error(
"LlamaGenerator: context size must be in range [1, 32768]");
}
sampling_temperature_ = sampling.temperature;
sampling_top_p_ = sampling.top_p;
sampling_top_k_ = sampling.top_k;
sampling_temperature_ = sampling.temperature;
sampling_top_p_ = sampling.top_p;
sampling_top_k_ = sampling.top_k;
if (sampling.seed == -1) {
std::random_device random_device;
rng_.seed(random_device());
} else {
rng_.seed(static_cast<uint32_t>(sampling.seed));
}
if (sampling.seed == -1) {
std::random_device random_device;
rng_.seed(random_device());
} else {
rng_.seed(static_cast<uint32_t>(sampling.seed));
}
n_ctx_ = sampling.n_ctx;
n_gpu_layers_ = sampling.n_gpu_layers;
n_ctx_ = sampling.n_ctx;
n_gpu_layers_ = sampling.n_gpu_layers;
this->Load(model_path);
this->Load(model_path);
}
LlamaGenerator::~LlamaGenerator() = default;

64
tooling/pipeline/src/data_generation/llama/load.cc
View File

@@ -4,15 +4,15 @@
* context, and resets prior resources during model initialization.
*/
#include <ggml-backend.h>
#include <llama.h>
#include <algorithm>
#include <chrono>
#include <stdexcept>
#include <string>
#include <utility>
#include <ggml-backend.h>
#include <llama.h>
#include "data_generation/llama_generator.h"
// Maximum batch size for decode operations. Capping the batch prevents
@@ -20,42 +20,42 @@
static constexpr uint32_t kMaxBatchSize = 5000U;
void LlamaGenerator::Load(const std::string& model_path) {
context_.reset();
model_.reset();
context_.reset();
model_.reset();
// Specifically load dynamic ggml backends (like CUDA) that are provided
// externally before attempting to load a model.
ggml_backend_load_all();
// Specifically load dynamic ggml backends (like CUDA) that are provided
// externally before attempting to load a model.
ggml_backend_load_all();
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = n_gpu_layers_;
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = n_gpu_layers_;
ModelHandle loaded_model(
llama_model_load_from_file(model_path.c_str(), model_params));
ModelHandle loaded_model(
llama_model_load_from_file(model_path.c_str(), model_params));
if (!loaded_model) {
throw std::runtime_error(
"LlamaGenerator: failed to load model from path: " + model_path);
}
if (!loaded_model) {
throw std::runtime_error(
"LlamaGenerator: failed to load model from path: " + model_path);
}
llama_context_params context_params = llama_context_default_params();
context_params.n_ctx = n_ctx_;
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
llama_context_params context_params = llama_context_default_params();
context_params.n_ctx = n_ctx_;
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
ContextHandle loaded_context(
llama_init_from_model(loaded_model.get(), context_params));
ContextHandle loaded_context(
llama_init_from_model(loaded_model.get(), context_params));
if (!loaded_context) {
throw std::runtime_error("LlamaGenerator: failed to create context");
}
if (!loaded_context) {
throw std::runtime_error("LlamaGenerator: failed to create context");
}
model_ = std::move(loaded_model);
context_ = std::move(loaded_context);
model_ = std::move(loaded_model);
context_ = std::move(loaded_context);
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format("[LlamaGenerator] Loaded model: {} ",
model_path)});
}
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format("[LlamaGenerator] Loaded model: {} ",
model_path)});
}
}

18
tooling/pipeline/src/data_generation/mock/deterministic_hash.cc
View File

@@ -9,18 +9,18 @@
#include "data_generation/mock_generator.h"
size_t MockGenerator::DeterministicHash(const City& location) {
size_t seed = 0;
boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country);
return seed;
size_t seed = 0;
boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country);
return seed;
}
size_t MockGenerator::DeterministicHash(const City& location,
const UserPersona& persona,
const Name& name) {
size_t seed = DeterministicHash(location);
boost::hash_combine(seed, persona.name);
boost::hash_combine(seed, name.first_name);
boost::hash_combine(seed, name.last_name);
return seed;
size_t seed = DeterministicHash(location);
boost::hash_combine(seed, persona.name);
boost::hash_combine(seed, name.first_name);
boost::hash_combine(seed, name.last_name);
return seed;
}

52
tooling/pipeline/src/data_generation/mock/generate_brewery.cc
View File

@@ -12,34 +12,34 @@
BreweryResult MockGenerator::GenerateBrewery(
const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const size_t hash = DeterministicHash(location);
const City& location = enriched_city.location;
const size_t hash = DeterministicHash(location);
const std::string_view adjective =
kBreweryAdjectives.at(hash % kBreweryAdjectives.size());
const std::string_view noun =
kBreweryNouns.at(hash / kNounHashStride % kBreweryNouns.size());
const std::string_view base_description = kBreweryDescriptions.at(
(hash / kDescriptionHashStride) % kBreweryDescriptions.size());
const std::string_view adjective =
kBreweryAdjectives.at(hash % kBreweryAdjectives.size());
const std::string_view noun =
kBreweryNouns.at(hash / kNounHashStride % kBreweryNouns.size());
const std::string_view base_description = kBreweryDescriptions.at(
(hash / kDescriptionHashStride) % kBreweryDescriptions.size());
const std::string name =
std::format("{} {} {}", location.city, adjective, noun);
const std::string name =
std::format("{} {} {}", location.city, adjective, noun);
const std::string state_suffix =
location.state_province.empty()
? std::string{}
: std::format(", {}", location.state_province);
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
const std::string description =
std::format("{} Located in {}{}{}.", base_description, location.city,
state_suffix, country_suffix);
const std::string state_suffix =
location.state_province.empty()
? std::string{}
: std::format(", {}", location.state_province);
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
const std::string description =
std::format("{} Located in {}{}{}.", base_description, location.city,
state_suffix, country_suffix);
return {
.name_en = name,
.description_en = description,
.name_local = name,
.description_local = description,
};
return {
.name_en = name,
.description_en = description,
.name_local = name,
.description_local = description,
};
}

30
tooling/pipeline/src/data_generation/mock/generate_user.cc
View File

@@ -11,21 +11,21 @@
UserResult MockGenerator::GenerateUser(const EnrichedCity& city,
const UserPersona& persona,
const Name& name) {
const size_t hash = DeterministicHash(city.location, persona, name);
const size_t hash = DeterministicHash(city.location, persona, name);
const std::string_view username = kUsernames[hash % kUsernames.size()];
const std::string_view bio = kBios[hash / kBioHashStride % kBios.size()];
const float activity_weight =
static_cast<float>(hash / kActivityWeightHashStride %
kActivityWeightHashRange) /
static_cast<float>(kActivityWeightHashRange);
const std::string_view username = kUsernames[hash % kUsernames.size()];
const std::string_view bio = kBios[hash / kBioHashStride % kBios.size()];
const float activity_weight =
static_cast<float>(hash / kActivityWeightHashStride %
kActivityWeightHashRange) /
static_cast<float>(kActivityWeightHashRange);
return {
.first_name = name.first_name,
.last_name = name.last_name,
.gender = name.gender,
.username = std::string(username),
.bio = std::string(bio),
.activity_weight = activity_weight,
};
return {
.first_name = name.first_name,
.last_name = name.last_name,
.gender = name.gender,
.username = std::string(username),
.bio = std::string(bio),
.activity_weight = activity_weight,
};
}

28
tooling/pipeline/src/data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.cc
View File

@@ -8,25 +8,25 @@ static constexpr std::string_view kWhitespace = " \t\n\r\f\v";
// Strips leading and trailing whitespace to ensure clean prompt injection.
static std::string_view Trim(std::string_view value) {
const size_t first_index = value.find_first_not_of(kWhitespace);
const size_t first_index = value.find_first_not_of(kWhitespace);
const bool is_all_whitespace = (first_index == std::string_view::npos);
if (is_all_whitespace) {
return "";
}
const bool is_all_whitespace = (first_index == std::string_view::npos);
if (is_all_whitespace) {
return "";
}
const size_t last_index = value.find_last_not_of(kWhitespace);
return value.substr(first_index, last_index - first_index + 1);
const size_t last_index = value.find_last_not_of(kWhitespace);
return value.substr(first_index, last_index - first_index + 1);
}
std::string Gemma4JinjaPromptFormatter::Format(
std::string_view system_prompt, std::string_view user_prompt) const {
std::string_view trimmed_system = Trim(system_prompt);
std::string_view trimmed_user = Trim(user_prompt);
std::string_view trimmed_system = Trim(system_prompt);
std::string_view trimmed_user = Trim(user_prompt);
return std::format(
"<|turn|>system\n<|think|>\n{}\n<|turn|>\n"
"<|turn|>user\n{}\n<|turn|>\n"
"<|turn|>model\n<|channel>thought\n",
trimmed_system, trimmed_user);
return std::format(
"<|turn|>system\n<|think|>\n{}\n<|turn|>\n"
"<|turn|>user\n{}\n<|turn|>\n"
"<|turn|>model\n<|channel>thought\n",
trimmed_system, trimmed_user);
}

385
tooling/pipeline/src/main.cc
View File

@@ -25,223 +25,230 @@ namespace di = boost::di;
static constexpr size_t kLogMaxCount = 512;
int main(const int argc, char** argv) {
spdlog::set_level(spdlog::level::debug);
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
spdlog::set_level(spdlog::level::debug);
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel);
auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel);
std::thread log_thread([&log_dispatcher] { log_dispatcher->Run(); });
std::thread log_thread([&log_dispatcher] { log_dispatcher->Run(); });
auto shutdown = [&](const int exit_code) {
log_channel.Close();
log_thread.join();
return exit_code;
};
auto shutdown = [&](const int exit_code) {
log_channel.Close();
log_thread.join();
return exit_code;
};
try {
Timer timer;
try {
Timer timer;
#ifndef BIERGARTEN_MOCK_ONLY
const LlamaBackendState llama_backend_state;
const LlamaBackendState llama_backend_state;
#endif
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "STARTING PIPELINE"});
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "STARTING PIPELINE"});
const std::optional<ApplicationOptions> parsed_options =
ParseArguments(argc, argv, log_producer);
const std::optional<ApplicationOptions> parsed_options =
ParseArguments(argc, argv, log_producer);
if (!parsed_options.has_value()) {
return shutdown(EXIT_FAILURE);
}
const auto options = *parsed_options;
const std::string model_path = options.generator.model_path.string();
const auto sampling =
options.generator.sampling.value_or(SamplingOptions{});
std::unique_ptr<IPromptDirectory> prompt_directory;
if (!options.generator.use_mocked) {
try {
prompt_directory = std::make_unique<PromptDirectory>(
options.pipeline.prompt_dir, log_producer);
} catch (const std::exception& dir_error) {
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = std::format("Invalid --prompt-dir: {}",
dir_error.what())});
return shutdown(EXIT_FAILURE);
if (!parsed_options.has_value()) {
return shutdown(EXIT_FAILURE);
}
}
const auto injector = di::make_injector(
di::bind<ILogger>().to(log_producer),
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPostalCodeService>().to(
[options, &log_producer]() -> std::unique_ptr<IPostalCodeService> {
if (options.generator.use_mocked) {
const auto options = *parsed_options;
const std::string model_path = options.generator.model_path.string();
const auto sampling =
options.generator.sampling.value_or(SamplingOptions{});
std::unique_ptr<IPromptDirectory> prompt_directory;
if (!options.generator.use_mocked) {
try {
prompt_directory = std::make_unique<PromptDirectory>(
options.pipeline.prompt_dir, log_producer);
} catch (const std::exception& dir_error) {
log_producer->Log(
{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = std::format("Invalid --prompt-dir: {}",
dir_error.what())});
return shutdown(EXIT_FAILURE);
}
}
const auto injector = di::make_injector(
di::bind<ILogger>().to(log_producer),
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPostalCodeService>().to(
[options,
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
if (options.generator.use_mocked) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: mock (curated examples)"});
return std::make_unique<MockPostalCodeService>();
}
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
"Postal code: Xeger (regex-based generation)"});
return std::make_unique<XegerPostalCodeService>();
}),
di::bind<ICuratedDataService>().to(
[options,
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: mock"});
return std::make_unique<MockCuratedDataService>();
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: JsonLoader"});
return std::make_unique<CuratedJsonDataService>(
CuratedDataFilePaths{
.locations_path = "locations.json",
.personas_path = "personas.json",
.forenames_path = "forenames-by-country.json",
.surnames_path = "surnames-by-country.json",
});
}),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: mock (curated examples)"});
.message =
"Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>());
}),
di::bind<WebClient>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: none (mock mode)"});
}
return std::unique_ptr<WebClient>(nullptr);
}
return std::make_unique<MockPostalCodeService>();
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: HttpWebClient"});
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: Xeger (regex-based generation)"});
return std::unique_ptr<WebClient>(
std::make_unique<HttpWebClient>(log_producer));
}),
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: mock"});
return std::make_unique<XegerPostalCodeService>();
}),
di::bind<ICuratedDataService>().to(
[options, &log_producer]() -> std::unique_ptr<ICuratedDataService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: mock"});
return std::make_unique<MockEnrichmentService>();
}
return std::make_unique<MockCuratedDataService>();
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: Wikipedia"});
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: JsonLoader"});
return std::make_unique<WikipediaEnrichmentService>(
inj.template create<std::unique_ptr<WebClient>>(),
log_producer);
}),
di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory,
&log_producer](
const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Generator: mock"});
return std::make_unique<CuratedJsonDataService>(
CuratedDataFilePaths{
.locations_path = "locations.json",
.personas_path = "personas.json",
.forenames_path = "forenames-by-country.json",
.surnames_path = "surnames-by-country.json",
});
}),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>());
}),
di::bind<WebClient>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: none (mock mode)"});
}
return std::unique_ptr<WebClient>(nullptr);
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: HttpWebClient"});
return std::unique_ptr<WebClient>(
std::make_unique<HttpWebClient>(log_producer));
}),
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: mock"});
return std::make_unique<MockEnrichmentService>();
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: Wikipedia"});
return std::make_unique<WikipediaEnrichmentService>(
inj.template create<std::unique_ptr<WebClient>>(),
log_producer);
}),
di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory,
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Generator: mock"});
return std::make_unique<MockGenerator>();
}
return std::make_unique<MockGenerator>();
}
#ifdef BIERGARTEN_MOCK_ONLY
// Unreachable: ParseArguments requires --mocked when
// BIERGARTEN_MOCK_ONLY is compiled in, so LlamaGenerator is
// never constructed and is not linked into this binary.
throw std::runtime_error(
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
"build");
// Unreachable: ParseArguments requires --mocked when
// BIERGARTEN_MOCK_ONLY is compiled in, so LlamaGenerator is
// never constructed and is not linked into this binary.
throw std::runtime_error(
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
"build");
#else
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
return std::make_unique<LlamaGenerator>(
options, model_path, log_producer,
inj.template create<std::unique_ptr<IPromptFormatter>>(),
std::move(prompt_directory));
return std::make_unique<LlamaGenerator>(
options, model_path, log_producer,
inj.template create<std::unique_ptr<IPromptFormatter>>(),
std::move(prompt_directory));
#endif
}));
}));
const auto orchestrator =
injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
const auto orchestrator =
injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
if (!orchestrator->Run()) {
log_producer->Log({.level = LogLevel::Error,
if (!orchestrator->Run()) {
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = "Pipeline execution failed"});
return shutdown(EXIT_FAILURE);
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = "Pipeline execution failed"});
.message = std::format("Pipeline complete in {} ms",
timer.Elapsed())});
return shutdown(EXIT_SUCCESS);
} catch (const std::exception& exception) {
const LogDTO log_entry{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = exception.what()};
if (log_producer) {
log_producer->Log(log_entry);
} else {
std::cerr << log_entry.message << std::endl;
}
return shutdown(EXIT_FAILURE);
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = std::format("Pipeline complete in {} ms",
timer.Elapsed())});
return shutdown(EXIT_SUCCESS);
} catch (const std::exception& exception) {
const LogDTO log_entry{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = exception.what()};
if (log_producer) {
log_producer->Log(log_entry);
} else {
std::cerr << log_entry.message << std::endl;
}
return shutdown(EXIT_FAILURE);
}
}
}

360
tooling/pipeline/src/services/curated_data/curated_json_data_service.cc
View File

@@ -21,71 +21,71 @@
static std::string ReadRequiredString(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_string()) {
throw std::runtime_error(
std::format("Missing or invalid string field: {}", key));
}
const std::string_view text = value->as_string();
return std::string(text);
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_string()) {
throw std::runtime_error(
std::format("Missing or invalid string field: {}", key));
}
const std::string_view text = value->as_string();
return std::string(text);
}
static std::vector<std::string> ReadRequiredStringArray(
const boost::json::object& object, const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_array()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
const auto& array = value->as_array();
std::vector<std::string> items;
items.reserve(array.size());
for (const auto& item : array) {
if (!item.is_string()) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_array()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
items.emplace_back(item.as_string());
}
return items;
}
const auto& array = value->as_array();
std::vector<std::string> items;
items.reserve(array.size());
for (const auto& item : array) {
if (!item.is_string()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
items.emplace_back(item.as_string());
}
return items;
}
static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
const boost::json::value* value = object.if_contains("postal_code");
if (value == nullptr || !value->is_object()) {
throw std::runtime_error("Missing or invalid object field: postal_code");
}
const boost::json::value* value = object.if_contains("postal_code");
if (value == nullptr || !value->is_object()) {
throw std::runtime_error("Missing or invalid object field: postal_code");
}
const auto& postal_code = value->as_object();
return PostalCodeSpec{
.country_format_regex =
ReadRequiredString(postal_code, "country_format_regex"),
.city_regexes = ReadRequiredStringArray(postal_code, "city_regex"),
.examples = ReadRequiredStringArray(postal_code, "examples"),
};
const auto& postal_code = value->as_object();
return PostalCodeSpec{
.country_format_regex =
ReadRequiredString(postal_code, "country_format_regex"),
.city_regexes = ReadRequiredStringArray(postal_code, "city_regex"),
.examples = ReadRequiredStringArray(postal_code, "examples"),
};
}
namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
const char* what) {
std::ifstream input(filepath);
if (!input.is_open()) {
throw std::runtime_error(
std::format("Failed to open {} file: {}", what, filepath.string()));
}
std::ifstream input(filepath);
if (!input.is_open()) {
throw std::runtime_error(
std::format("Failed to open {} file: {}", what, filepath.string()));
}
std::stringstream buffer;
buffer << input.rdbuf();
std::stringstream buffer;
buffer << input.rdbuf();
boost::system::error_code error;
boost::json::value root = boost::json::parse(buffer.str(), error);
if (error) {
throw std::runtime_error(
std::format("Failed to parse {} JSON: {}", what, error.message()));
}
return root;
boost::system::error_code error;
boost::json::value root = boost::json::parse(buffer.str(), error);
if (error) {
throw std::runtime_error(
std::format("Failed to parse {} JSON: {}", what, error.message()));
}
return root;
}
/**
@@ -95,18 +95,18 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
*/
std::string ReadFirstOfStringArray(const boost::json::object& object,
const char* key, const char* fallback_key) {
for (const char* candidate_key : {key, fallback_key}) {
const boost::json::value* value = object.if_contains(candidate_key);
if (value == nullptr || !value->is_array()) {
continue;
}
const auto& array = value->as_array();
if (!array.empty() && array.front().is_string()) {
return std::string(array.front().as_string());
}
}
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
for (const char* candidate_key : {key, fallback_key}) {
const boost::json::value* value = object.if_contains(candidate_key);
if (value == nullptr || !value->is_array()) {
continue;
}
const auto& array = value->as_array();
if (!array.empty() && array.front().is_string()) {
return std::string(array.front().as_string());
}
}
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
} // namespace
@@ -115,154 +115,156 @@ CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths)
: filepaths_(std::move(filepaths)) {}
const LocationsList& CuratedJsonDataService::LoadLocations() {
if (!cache_.locations.empty()) {
return cache_.locations;
}
if (!cache_.locations.empty()) {
return cache_.locations;
}
const boost::json::value root =
ParseJsonFile(filepaths_.locations_path, "locations");
const boost::json::value root =
ParseJsonFile(filepaths_.locations_path, "locations");
if (!root.is_array()) {
throw std::runtime_error(
"Invalid locations JSON: root element must be an array");
}
LocationsList locations;
const auto& items = root.as_array();
locations.reserve(items.size());
for (const auto& item : items) {
if (!item.is_object()) {
if (!root.is_array()) {
throw std::runtime_error(
"Invalid locations JSON: each entry must be an object");
}
"Invalid locations JSON: root element must be an array");
}
const auto& object = item.as_object();
locations.push_back(City{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.postal_code = ReadPostalCodeSpec(object),
});
}
cache_.locations = std::move(locations);
return cache_.locations;
LocationsList locations;
const auto& items = root.as_array();
locations.reserve(items.size());
for (const auto& item : items) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid locations JSON: each entry must be an object");
}
const auto& object = item.as_object();
locations.push_back(City{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.postal_code = ReadPostalCodeSpec(object),
});
}
cache_.locations = std::move(locations);
return cache_.locations;
}
const PersonasList& CuratedJsonDataService::LoadPersonas() {
if (!cache_.personas.empty()) {
return cache_.personas;
}
if (!cache_.personas.empty()) {
return cache_.personas;
}
const boost::json::value root =
ParseJsonFile(filepaths_.personas_path, "personas");
const boost::json::value root =
ParseJsonFile(filepaths_.personas_path, "personas");
if (!root.is_array()) {
throw std::runtime_error(
"Invalid personas JSON: root element must be an array");
}
PersonasList personas;
const auto& items = root.as_array();
personas.reserve(items.size());
for (const auto& item : items) {
if (!item.is_object()) {
if (!root.is_array()) {
throw std::runtime_error(
"Invalid personas JSON: each entry must be an object");
}
"Invalid personas JSON: root element must be an array");
}
const auto& object = item.as_object();
personas.push_back(UserPersona{
.name = ReadRequiredString(object, "name"),
.description = ReadRequiredString(object, "description"),
.style_affinities = ReadRequiredStringArray(object, "style_affinities"),
});
}
PersonasList personas;
const auto& items = root.as_array();
personas.reserve(items.size());
cache_.personas = std::move(personas);
return cache_.personas;
for (const auto& item : items) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid personas JSON: each entry must be an object");
}
const auto& object = item.as_object();
personas.push_back(UserPersona{
.name = ReadRequiredString(object, "name"),
.description = ReadRequiredString(object, "description"),
.style_affinities =
ReadRequiredStringArray(object, "style_affinities"),
});
}
cache_.personas = std::move(personas);
return cache_.personas;
}
const ForenamesByCountryMap& CuratedJsonDataService::LoadForenamesByCountry() {
if (!cache_.forenames_by_country.empty()) {
return cache_.forenames_by_country;
}
if (!cache_.forenames_by_country.empty()) {
return cache_.forenames_by_country;
}
const boost::json::value root =
ParseJsonFile(filepaths_.forenames_path, "forenames-by-country");
const boost::json::value root =
ParseJsonFile(filepaths_.forenames_path, "forenames-by-country");
if (!root.is_object()) {
throw std::runtime_error(
"Invalid forenames-by-country JSON: root element must be an object "
"keyed by ISO 3166-1 country code");
}
if (!root.is_object()) {
throw std::runtime_error(
"Invalid forenames-by-country JSON: root element must be an object "
"keyed by ISO 3166-1 country code");
}
ForenamesByCountryMap forenames_by_country;
for (const auto& [country_code, regions] : root.as_object()) {
if (!regions.is_array()) {
continue;
}
ForenameList entries;
for (const auto& region : regions.as_array()) {
if (!region.is_object()) {
continue;
ForenamesByCountryMap forenames_by_country;
for (const auto& [country_code, regions] : root.as_object()) {
if (!regions.is_array()) {
continue;
}
const boost::json::value* names = region.as_object().if_contains("names");
if (names == nullptr || !names->is_array()) {
continue;
ForenameList entries;
for (const auto& region : regions.as_array()) {
if (!region.is_object()) {
continue;
}
const boost::json::value* names =
region.as_object().if_contains("names");
if (names == nullptr || !names->is_array()) {
continue;
}
for (const auto& name_value : names->as_array()) {
if (!name_value.is_object()) {
continue;
}
const auto& name_object = name_value.as_object();
entries.emplace_back(ForenameEntry{
.name = ReadFirstOfStringArray(name_object, "romanized",
"localized"),
.gender = ReadRequiredString(name_object, "gender"),
});
}
}
for (const auto& name_value : names->as_array()) {
if (!name_value.is_object()) {
continue;
}
const auto& name_object = name_value.as_object();
entries.emplace_back(ForenameEntry{
.name =
ReadFirstOfStringArray(name_object, "romanized", "localized"),
.gender = ReadRequiredString(name_object, "gender"),
});
}
}
forenames_by_country.emplace(country_code, std::move(entries));
}
forenames_by_country.emplace(country_code, std::move(entries));
}
cache_.forenames_by_country = std::move(forenames_by_country);
return cache_.forenames_by_country;
cache_.forenames_by_country = std::move(forenames_by_country);
return cache_.forenames_by_country;
}
const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() {
if (!cache_.surnames_by_country.empty()) {
return cache_.surnames_by_country;
}
if (!cache_.surnames_by_country.empty()) {
return cache_.surnames_by_country;
}
const boost::json::value root =
ParseJsonFile(filepaths_.surnames_path, "surnames-by-country");
const boost::json::value root =
ParseJsonFile(filepaths_.surnames_path, "surnames-by-country");
if (!root.is_object()) {
throw std::runtime_error(
"Invalid surnames-by-country JSON: root element must be an object "
"keyed by ISO 3166-1 country code");
}
if (!root.is_object()) {
throw std::runtime_error(
"Invalid surnames-by-country JSON: root element must be an object "
"keyed by ISO 3166-1 country code");
}
SurnamesByCountryMap surnames_by_country;
for (const auto& [country_code, name_entries] : root.as_object()) {
if (!name_entries.is_array()) continue;
SurnamesByCountryMap surnames_by_country;
for (const auto& [country_code, name_entries] : root.as_object()) {
if (!name_entries.is_array()) continue;
SurnameList surnames;
for (const auto& name_value : name_entries.as_array()) {
if (!name_value.is_object()) {
continue;
SurnameList surnames;
for (const auto& name_value : name_entries.as_array()) {
if (!name_value.is_object()) {
continue;
}
surnames.emplace_back(ReadFirstOfStringArray(
name_value.as_object(), "romanized", "localized"));
}
surnames.emplace_back(ReadFirstOfStringArray(name_value.as_object(),
"romanized", "localized"));
}
surnames_by_country.emplace(country_code, std::move(surnames));
}
surnames_by_country.emplace(country_code, std::move(surnames));
}
cache_.surnames_by_country = std::move(surnames_by_country);
return cache_.surnames_by_country;
cache_.surnames_by_country = std::move(surnames_by_country);
return cache_.surnames_by_country;
}

6
tooling/pipeline/src/services/curated_data/mock_curated_data_service.cc
View File

@@ -92,15 +92,15 @@ MockCuratedDataService::MockCuratedDataService()
} {}
const LocationsList& MockCuratedDataService::LoadLocations() {
return locations_;
return locations_;
}
const PersonasList& MockCuratedDataService::LoadPersonas() { return personas_; }
const ForenamesByCountryMap& MockCuratedDataService::LoadForenamesByCountry() {
return forenames_by_country_;
return forenames_by_country_;
}
const SurnamesByCountryMap& MockCuratedDataService::LoadSurnamesByCountry() {
return surnames_by_country_;
return surnames_by_country_;
}

276
tooling/pipeline/src/services/enrichment/wikipedia/fetch_extract.cc
View File

@@ -15,147 +15,147 @@
using namespace boost;
std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
const std::string cache_key(query);
const std::string cache_key(query);
// 1. Cache Lookup
if (const auto cache_it = this->extract_cache_.find(cache_key);
cache_it != this->extract_cache_.end()) {
if (logger_) {
// 1. Cache Lookup
if (const auto cache_it = this->extract_cache_.find(cache_key);
cache_it != this->extract_cache_.end()) {
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format("Wikipedia: Cache hit for {}!",
cache_key)});
}
return cache_it->second;
}
const std::string encoded = this->client_->EncodeURL(cache_key);
const std::string url = std::format(
"https://en.wikipedia.org/w/"
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format="
"json",
encoded);
const std::string body = this->client_->Get(url);
{
using namespace std::literals::chrono_literals;
std::this_thread::sleep_for(1s);
}
// 2. Parse JSON
system::error_code ec;
json::value doc = json::parse(body, ec);
if (ec) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: JSON parse error for '{}': {}",
std::string(query), ec.message())});
}
return {};
}
// 3. Safe Extraction
const json::object* obj = doc.if_object();
if (obj == nullptr) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Expected root object for '{}'",
std::string(query))});
}
return {};
}
const json::value* query_ptr = obj->if_contains("query");
const json::value* pages_ptr =
((query_ptr != nullptr) && query_ptr->is_object())
? query_ptr->get_object().if_contains("pages")
: nullptr;
if ((pages_ptr == nullptr) || !pages_ptr->is_object()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Missing query.pages for '{}'",
std::string(query))});
}
return {};
}
const json::object& pages = pages_ptr->get_object();
if (pages.empty()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No pages returned for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// Wikipedia returns the page under a dynamic ID key; we just want the first
// one
const json::value& page_val = pages.begin()->value();
if (!page_val.is_object()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Unexpected page format for '{}'",
std::string(query))});
}
return {};
}
const json::object& page = page_val.get_object();
// Handle 404/Missing status
if (page.contains("missing")) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Page '{}' does not exist",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
const json::value* extract_ptr = page.if_contains("extract");
if ((extract_ptr == nullptr) || !extract_ptr->is_string()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No extract string found for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// 4. Success
std::string extract(extract_ptr->as_string());
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
{.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("Wikipedia: Cache hit for {}!", cache_key)});
}
return cache_it->second;
}
.message = std::format("WikipediaService: Fetched {} chars for '{}'",
extract.size(), std::string(query))});
}
const std::string encoded = this->client_->EncodeURL(cache_key);
const std::string url = std::format(
"https://en.wikipedia.org/w/"
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format="
"json",
encoded);
this->extract_cache_.insert_or_assign(cache_key, extract);
const std::string body = this->client_->Get(url);
{
using namespace std::literals::chrono_literals;
std::this_thread::sleep_for(1s);
}
// 2. Parse JSON
system::error_code ec;
json::value doc = json::parse(body, ec);
if (ec) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: JSON parse error for '{}': {}",
std::string(query), ec.message())});
}
return {};
}
// 3. Safe Extraction
const json::object* obj = doc.if_object();
if (obj == nullptr) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Expected root object for '{}'",
std::string(query))});
}
return {};
}
const json::value* query_ptr = obj->if_contains("query");
const json::value* pages_ptr =
((query_ptr != nullptr) && query_ptr->is_object())
? query_ptr->get_object().if_contains("pages")
: nullptr;
if ((pages_ptr == nullptr) || !pages_ptr->is_object()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Missing query.pages for '{}'",
std::string(query))});
}
return {};
}
const json::object& pages = pages_ptr->get_object();
if (pages.empty()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No pages returned for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// Wikipedia returns the page under a dynamic ID key; we just want the first
// one
const json::value& page_val = pages.begin()->value();
if (!page_val.is_object()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Unexpected page format for '{}'",
std::string(query))});
}
return {};
}
const json::object& page = page_val.get_object();
// Handle 404/Missing status
if (page.contains("missing")) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Page '{}' does not exist",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
const json::value* extract_ptr = page.if_contains("extract");
if ((extract_ptr == nullptr) || !extract_ptr->is_string()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No extract string found for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// 4. Success
std::string extract(extract_ptr->as_string());
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Fetched {} chars for '{}'",
extract.size(), std::string(query))});
}
this->extract_cache_.insert_or_assign(cache_key, extract);
return extract;
return extract;
}

91
tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc
View File

@@ -10,55 +10,54 @@
#include "services/enrichment/wikipedia_service.h"
std::string WikipediaEnrichmentService::GetLocationContext(
const City& loc) {
using namespace std::literals::chrono_literals;
if (!this->client_) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = "Wikipedia client is nullptr."});
}
return {};
}
std::string WikipediaEnrichmentService::GetLocationContext(const City& loc) {
using namespace std::literals::chrono_literals;
if (!this->client_) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = "Wikipedia client is nullptr."});
}
return {};
}
std::string result;
std::string result;
constexpr std::string_view brewing_query = "brewing";
const std::string location_query =
std::format("{}, {}", loc.city, loc.iso3166_2);
const std::string beer_query = std::format("beer in {}", loc.country);
constexpr std::string_view brewing_query = "brewing";
const std::string location_query =
std::format("{}, {}", loc.city, loc.iso3166_2);
const std::string beer_query = std::format("beer in {}", loc.country);
auto append_extract = [&result](const std::string& extract) -> void {
if (extract.empty()) {
return;
}
if (!result.empty()) {
result += "\n\n";
}
result += extract;
};
auto append_extract = [&result](const std::string& extract) -> void {
if (extract.empty()) {
return;
}
if (!result.empty()) {
result += "\n\n";
}
result += extract;
};
try {
append_extract(FetchExtract(brewing_query));
append_extract(FetchExtract(beer_query));
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"Done fetching for {}. Sleeping for 10 seconds.",
location_query)});
}
std::this_thread::sleep_for(10s);
try {
append_extract(FetchExtract(brewing_query));
append_extract(FetchExtract(beer_query));
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"Done fetching for {}. Sleeping for 10 seconds.",
location_query)});
}
std::this_thread::sleep_for(10s);
} catch (const std::runtime_error& e) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService lookup failed for '{}': {}",
location_query, e.what())});
}
}
return result;
} catch (const std::runtime_error& e) {
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService lookup failed for '{}': {}",
location_query, e.what())});
}
}
return result;
}

88
tooling/pipeline/src/services/logging/log_dispatcher.cc
View File

@@ -16,25 +16,25 @@
namespace {
[[nodiscard]] constexpr std::string_view PipelinePhaseToString(
PipelinePhase phase) {
switch (phase) {
case PipelinePhase::Startup:
return "Startup";
case PipelinePhase::Enrichment:
return "Enrichment";
case PipelinePhase::UserGeneration:
return "User Generation";
case PipelinePhase::BreweryAndBeerGeneration:
return "Brewery & Beer Gen";
case PipelinePhase::CheckinGeneration:
return "Checkin Gen";
case PipelinePhase::RatingGeneration:
return "Rating Gen";
case PipelinePhase::FollowGeneration:
return "Follow Gen";
case PipelinePhase::Teardown:
return "Teardown";
}
return "Unknown";
switch (phase) {
case PipelinePhase::Startup:
return "Startup";
case PipelinePhase::Enrichment:
return "Enrichment";
case PipelinePhase::UserGeneration:
return "User Generation";
case PipelinePhase::BreweryAndBeerGeneration:
return "Brewery & Beer Gen";
case PipelinePhase::CheckinGeneration:
return "Checkin Gen";
case PipelinePhase::RatingGeneration:
return "Rating Gen";
case PipelinePhase::FollowGeneration:
return "Follow Gen";
case PipelinePhase::Teardown:
return "Teardown";
}
return "Unknown";
}
} // namespace
@@ -42,35 +42,35 @@ LogDispatcher::LogDispatcher(BoundedChannel<LogEntry>& channel)
: channel_(channel) {}
void LogDispatcher::Run() {
auto logger = spdlog::default_logger();
auto logger = spdlog::default_logger();
while (true) {
auto entry = channel_.Receive();
if (!entry.has_value()) {
// Channel is closed and drained.
break;
}
while (true) {
auto entry = channel_.Receive();
if (!entry.has_value()) {
// Channel is closed and drained.
break;
}
const auto& log = entry.value();
const auto& log = entry.value();
logger->log(ToSpdlogLevel(log.level),
"{:<20} │ thread: {:016x} │ [{}:{}] │ {}",
PipelinePhaseToString(log.phase),
std::hash<std::thread::id>{}(log.thread_id),
log.origin.file_name(), log.origin.line(), log.message);
}
logger->log(ToSpdlogLevel(log.level),
"{:<20} │ thread: {:016x} │ [{}:{}] │ {}",
PipelinePhaseToString(log.phase),
std::hash<std::thread::id>{}(log.thread_id),
log.origin.file_name(), log.origin.line(), log.message);
}
}
spdlog::level::level_enum LogDispatcher::ToSpdlogLevel(LogLevel level) {
switch (level) {
case LogLevel::Debug:
return spdlog::level::debug;
case LogLevel::Info:
return spdlog::level::info;
case LogLevel::Warn:
return spdlog::level::warn;
case LogLevel::Error:
return spdlog::level::err;
}
return spdlog::level::info;
switch (level) {
case LogLevel::Debug:
return spdlog::level::debug;
case LogLevel::Info:
return spdlog::level::info;
case LogLevel::Warn:
return spdlog::level::warn;
case LogLevel::Error:
return spdlog::level::err;
}
return spdlog::level::info;
}

40
tooling/pipeline/src/services/postal_code/xeger_postal_code_service.cc
View File

@@ -18,27 +18,27 @@ XegerPostalCodeService::XegerPostalCodeService()
: rng_(std::random_device{}()) {}
std::string XegerPostalCodeService::GeneratePostalCode(const City& city) {
const PostalCodeSpec& spec = city.postal_code;
const PostalCodeSpec& spec = city.postal_code;
std::vector<std::string_view> city_patterns;
for (const std::string& pattern : spec.city_regexes) {
if (!pattern.empty()) {
city_patterns.push_back(pattern);
}
}
std::vector<std::string_view> city_patterns;
for (const std::string& pattern : spec.city_regexes) {
if (!pattern.empty()) {
city_patterns.push_back(pattern);
}
}
std::string_view pattern;
if (!city_patterns.empty()) {
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
pattern = city_patterns[dist(rng_)];
} else if (!spec.country_format_regex.empty()) {
pattern = spec.country_format_regex;
} else {
throw std::runtime_error(std::format(
"XegerPostalCodeService: no postal-code pattern available for "
"city '{}'",
city.city));
}
std::string_view pattern;
if (!city_patterns.empty()) {
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
pattern = city_patterns[dist(rng_)];
} else if (!spec.country_format_regex.empty()) {
pattern = spec.country_format_regex;
} else {
throw std::runtime_error(std::format(
"XegerPostalCodeService: no postal-code pattern available for "
"city '{}'",
city.city));
}
return GenerateStringFromRegex(pattern, rng_);
return GenerateStringFromRegex(pattern, rng_);
}

124
tooling/pipeline/src/services/prompt_directory.cc
View File

@@ -23,80 +23,80 @@ PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir)
PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir,
std::shared_ptr<ILogger> logger)
: prompt_dir_(prompt_dir), logger_(std::move(logger)) {
std::error_code ec;
std::error_code ec;
// Directory must exist.
if (!std::filesystem::exists(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory does not exist: " +
prompt_dir_.string());
}
// Directory must exist.
if (!std::filesystem::exists(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory does not exist: " +
prompt_dir_.string());
}
// Path must be a directory, not a file.
if (!std::filesystem::is_directory(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory path is not a directory: " +
prompt_dir_.string());
}
// Path must be a directory, not a file.
if (!std::filesystem::is_directory(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory path is not a directory: " +
prompt_dir_.string());
}
// Directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(std::format(
"PromptDirectory: prompt directory is not readable: {} ({})",
prompt_dir_.string(), ec.message()));
}
// Directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(std::format(
"PromptDirectory: prompt directory is not readable: {} ({})",
prompt_dir_.string(), ec.message()));
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
std::string("[PromptDirectory] Resolved prompt directory: ") +
prompt_dir_.string()});
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
std::string("[PromptDirectory] Resolved prompt directory: ") +
prompt_dir_.string()});
}
}
std::string PromptDirectory::Load(std::string_view key) {
const std::string key_str(key);
const std::string key_str(key);
// Return cached content if already loaded during this run.
const auto cache_it = cache_.find(key_str);
if (cache_it != cache_.end()) {
return cache_it->second;
}
// Return cached content if already loaded during this run.
const auto cache_it = cache_.find(key_str);
if (cache_it != cache_.end()) {
return cache_it->second;
}
// Resolve <prompt_dir>/<key>.md and require it to exist.
const std::filesystem::path file_path =
prompt_dir_ / std::filesystem::path(std::format("{}.md", key_str));
// Resolve <prompt_dir>/<key>.md and require it to exist.
const std::filesystem::path file_path =
prompt_dir_ / std::filesystem::path(std::format("{}.md", key_str));
std::ifstream file(file_path);
if (!file.is_open()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file not found for key '{}': {}",
key_str, file_path.string()));
}
std::ifstream file(file_path);
if (!file.is_open()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file not found for key '{}': {}",
key_str, file_path.string()));
}
std::string content((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
file.close();
std::string content((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
file.close();
if (content.empty()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
key_str, file_path.string()));
}
if (content.empty()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
key_str, file_path.string()));
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
key_str, file_path.string(), content.size())});
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
key_str, file_path.string(), content.size())});
}
// Cache the loaded content for future calls.
cache_.emplace(key_str, content);
return content;
// Cache the loaded content for future calls.
cache_.emplace(key_str, content);
return content;
}

40
tooling/pipeline/src/services/sqlite/finalize.cc
View File

@@ -9,26 +9,26 @@
#include "services/database/sqlite_export_service_helpers.h"
void SqliteExportService::Finalize() {
if (db_handle_ == nullptr) {
return;
}
if (db_handle_ == nullptr) {
return;
}
try {
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_city_stmt_.reset();
if (transaction_open_) {
sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
transaction_open_ = false;
}
try {
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_city_stmt_.reset();
if (transaction_open_) {
sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
transaction_open_ = false;
}
db_handle_.reset();
city_cache_.clear();
} catch (...) {
RollbackAndCloseNoThrow();
throw;
}
db_handle_.reset();
city_cache_.clear();
} catch (...) {
RollbackAndCloseNoThrow();
throw;
}
}

70
tooling/pipeline/src/services/sqlite/helpers/sqlite_connection_helpers.cc
View File

@@ -6,62 +6,62 @@
namespace sqlite_export_service_internal {
void SqliteDatabaseDeleter::operator()(sqlite3* handle) const noexcept {
if (handle != nullptr) {
sqlite3_close(handle);
}
if (handle != nullptr) {
sqlite3_close(handle);
}
}
void SqliteStatementDeleter::operator()(
sqlite3_stmt* statement) const noexcept {
if (statement != nullptr) {
sqlite3_finalize(statement);
}
if (statement != nullptr) {
sqlite3_finalize(statement);
}
}
void ThrowSqliteError(sqlite3* db_handle, std::string_view action) {
const std::string message =
db_handle != nullptr ? sqlite3_errmsg(db_handle) : "unknown SQLite error";
throw std::runtime_error(std::format("{}: {}", action, message));
const std::string message = db_handle != nullptr ? sqlite3_errmsg(db_handle)
: "unknown SQLite error";
throw std::runtime_error(std::format("{}: {}", action, message));
}
SqliteDatabaseHandle OpenDatabase(const std::filesystem::path& path) {
sqlite3* raw_handle = nullptr;
const int result = sqlite3_open(path.string().c_str(), &raw_handle);
sqlite3* raw_handle = nullptr;
const int result = sqlite3_open(path.string().c_str(), &raw_handle);
SqliteDatabaseHandle handle(raw_handle);
if (result != SQLITE_OK) {
const std::string message = raw_handle != nullptr
? sqlite3_errmsg(raw_handle)
: "unknown SQLite error";
throw std::runtime_error("Failed to open SQLite export database: " +
message);
}
SqliteDatabaseHandle handle(raw_handle);
if (result != SQLITE_OK) {
const std::string message = raw_handle != nullptr
? sqlite3_errmsg(raw_handle)
: "unknown SQLite error";
throw std::runtime_error("Failed to open SQLite export database: " +
message);
}
return handle;
return handle;
}
void ExecSql(const SqliteDatabaseHandle& db_handle, std::string_view sql,
const char* action) {
char* error_message = nullptr;
const std::string sql_text(sql);
const int result = sqlite3_exec(db_handle.get(), sql_text.c_str(), nullptr,
nullptr, &error_message);
if (result != SQLITE_OK) {
const std::string message = error_message != nullptr
? error_message
: sqlite3_errmsg(db_handle.get());
sqlite3_free(error_message);
throw std::runtime_error(std::format("{}: {}", action, message));
}
char* error_message = nullptr;
const std::string sql_text(sql);
const int result = sqlite3_exec(db_handle.get(), sql_text.c_str(), nullptr,
nullptr, &error_message);
if (result != SQLITE_OK) {
const std::string message = error_message != nullptr
? error_message
: sqlite3_errmsg(db_handle.get());
sqlite3_free(error_message);
throw std::runtime_error(std::format("{}: {}", action, message));
}
}
void RollbackTransactionNoThrow(
const SqliteDatabaseHandle& db_handle) noexcept {
if (!db_handle) {
return;
}
if (!db_handle) {
return;
}
sqlite3_exec(db_handle.get(), "ROLLBACK;", nullptr, nullptr, nullptr);
sqlite3_exec(db_handle.get(), "ROLLBACK;", nullptr, nullptr, nullptr);
}
} // namespace sqlite_export_service_internal

100
tooling/pipeline/src/services/sqlite/helpers/sqlite_statement_helpers.cc
View File

@@ -13,87 +13,87 @@ namespace sqlite_export_service_internal {
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql,
const char* action) {
sqlite3_stmt* raw_statement = nullptr;
const std::string sql_text(sql);
const int result = sqlite3_prepare_v2(db_handle.get(), sql_text.c_str(), -1,
&raw_statement, nullptr);
SqliteStatementHandle statement(raw_statement);
if (result != SQLITE_OK) {
ThrowSqliteError(db_handle.get(), action);
}
sqlite3_stmt* raw_statement = nullptr;
const std::string sql_text(sql);
const int result = sqlite3_prepare_v2(db_handle.get(), sql_text.c_str(), -1,
&raw_statement, nullptr);
SqliteStatementHandle statement(raw_statement);
if (result != SQLITE_OK) {
ThrowSqliteError(db_handle.get(), action);
}
return statement;
return statement;
}
void ResetStatement(SqliteStatementHandle& statement) {
if (statement != nullptr) {
sqlite3_reset(statement.get());
sqlite3_clear_bindings(statement.get());
}
if (statement != nullptr) {
sqlite3_reset(statement.get());
sqlite3_clear_bindings(statement.get());
}
}
void Bind(const SqliteStatementHandle& statement,
const BoundParam<std::string_view>& param) {
const auto byte_count = param.value.size();
if (byte_count > static_cast<std::size_t>(std::numeric_limits<int>::max())) {
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
const auto byte_count = param.value.size();
if (byte_count > static_cast<std::size_t>(std::numeric_limits<int>::max())) {
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
auto delete_char_array = [](void* data) noexcept {
// NOLINT(cppcoreguidelines-owning-memory)
delete[] static_cast<char*>(data);
};
auto delete_char_array = [](void* data) noexcept {
// NOLINT(cppcoreguidelines-owning-memory)
delete[] static_cast<char*>(data);
};
// NOLINT(cppcoreguidelines-avoid-c-arrays, modernize-avoid-c-arrays)
auto buffer = std::make_unique<char[]>(byte_count + 1);
std::memcpy(buffer.get(), param.value.data(), byte_count);
buffer[byte_count] = '\0';
// NOLINT(cppcoreguidelines-avoid-c-arrays, modernize-avoid-c-arrays)
auto buffer = std::make_unique<char[]>(byte_count + 1);
std::memcpy(buffer.get(), param.value.data(), byte_count);
buffer[byte_count] = '\0';
char* raw_buffer = buffer.release();
char* raw_buffer = buffer.release();
if (sqlite3_bind_text(statement.get(), param.index, raw_buffer,
static_cast<int>(byte_count),
delete_char_array) != SQLITE_OK) {
delete_char_array(raw_buffer);
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
if (sqlite3_bind_text(statement.get(), param.index, raw_buffer,
static_cast<int>(byte_count),
delete_char_array) != SQLITE_OK) {
delete_char_array(raw_buffer);
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
}
void Bind(const SqliteStatementHandle& statement,
const BoundParam<double>& param) {
if (sqlite3_bind_double(statement.get(), param.index, param.value) !=
SQLITE_OK) {
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
if (sqlite3_bind_double(statement.get(), param.index, param.value) !=
SQLITE_OK) {
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
}
void Bind(const SqliteStatementHandle& statement,
const BoundParam<sqlite3_int64>& param) {
if (sqlite3_bind_int64(statement.get(), param.index, param.value) !=
SQLITE_OK) {
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
if (sqlite3_bind_int64(statement.get(), param.index, param.value) !=
SQLITE_OK) {
ThrowSqliteError(sqlite3_db_handle(statement.get()), param.action);
}
}
void StepStatement(const SqliteDatabaseHandle& db_handle,
const SqliteStatementHandle& statement,
std::string_view action) {
if (sqlite3_step(statement.get()) != SQLITE_DONE) {
ThrowSqliteError(db_handle.get(), action);
}
if (sqlite3_step(statement.get()) != SQLITE_DONE) {
ThrowSqliteError(db_handle.get(), action);
}
}
sqlite3_int64 LastInsertRowId(const SqliteDatabaseHandle& db_handle) {
return sqlite3_last_insert_rowid(db_handle.get());
return sqlite3_last_insert_rowid(db_handle.get());
}
std::string SerializeVector(const std::vector<std::string>& str_vec) {
boost::json::array array;
array.reserve(str_vec.size());
for (const auto& s : str_vec) {
array.emplace_back(s);
}
return boost::json::serialize(array);
boost::json::array array;
array.reserve(str_vec.size());
for (const auto& s : str_vec) {
array.emplace_back(s);
}
return boost::json::serialize(array);
}
} // namespace sqlite_export_service_internal

152
tooling/pipeline/src/services/sqlite/initialize.cc
View File

@@ -12,99 +12,99 @@
#include "services/database/sqlite_export_service.h"
std::filesystem::path SqliteExportService::BuildDatabasePath() const {
std::filesystem::path base_filename("biergarten_seed_" + run_timestamp_utc_ +
".sqlite");
std::filesystem::path candidate = output_path_ / base_filename;
std::filesystem::path base_filename("biergarten_seed_" + run_timestamp_utc_ +
".sqlite");
std::filesystem::path candidate = output_path_ / base_filename;
for (int suffix = 1; std::filesystem::exists(candidate); ++suffix) {
candidate = output_path_ / std::filesystem::path(
std::format("biergarten_seed_{}-{}.sqlite",
run_timestamp_utc_, suffix));
}
for (int suffix = 1; std::filesystem::exists(candidate); ++suffix) {
candidate = output_path_ / std::filesystem::path(
std::format("biergarten_seed_{}-{}.sqlite",
run_timestamp_utc_, suffix));
}
return candidate;
return candidate;
}
void SqliteExportService::InitializeSchema() const {
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql,
"Failed to create SQLite cities table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
"Failed to create SQLite breweries table");
sqlite_export_service_internal::ExecSql(
db_handle_,
sqlite_export_service_internal::kCreateBreweryAddressesTableSql,
"Failed to create SQLite brewery_addresses table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
"Failed to create SQLite users table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUserAddressesTableSql,
"Failed to create SQLite user_addresses table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql,
"Failed to create SQLite cities table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
"Failed to create SQLite breweries table");
sqlite_export_service_internal::ExecSql(
db_handle_,
sqlite_export_service_internal::kCreateBreweryAddressesTableSql,
"Failed to create SQLite brewery_addresses table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
"Failed to create SQLite users table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUserAddressesTableSql,
"Failed to create SQLite user_addresses table");
}
void SqliteExportService::PrepareStatements() {
insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertCitySql,
"Failed to prepare SQLite city insert statement");
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
"Failed to prepare SQLite brewery insert statement");
insert_brewery_address_stmt_ =
sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql,
"Failed to prepare SQLite brewery address insert statement");
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserSql,
"Failed to prepare SQLite user insert statement");
insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserAddressSql,
"Failed to prepare SQLite user address insert statement");
insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertCitySql,
"Failed to prepare SQLite city insert statement");
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
"Failed to prepare SQLite brewery insert statement");
insert_brewery_address_stmt_ =
sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql,
"Failed to prepare SQLite brewery address insert statement");
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserSql,
"Failed to prepare SQLite user insert statement");
insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserAddressSql,
"Failed to prepare SQLite user address insert statement");
}
void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
if (db_handle_ == nullptr) {
return;
}
if (db_handle_ == nullptr) {
return;
}
if (transaction_open_) {
sqlite_export_service_internal::RollbackTransactionNoThrow(db_handle_);
transaction_open_ = false;
}
if (transaction_open_) {
sqlite_export_service_internal::RollbackTransactionNoThrow(db_handle_);
transaction_open_ = false;
}
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_city_stmt_.reset();
db_handle_.reset();
city_cache_.clear();
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_city_stmt_.reset();
db_handle_.reset();
city_cache_.clear();
}
void SqliteExportService::Initialize() {
if (db_handle_ != nullptr) {
throw std::runtime_error("SQLite export service is already initialized");
}
if (db_handle_ != nullptr) {
throw std::runtime_error("SQLite export service is already initialized");
}
run_timestamp_utc_ = date_time_provider_->GetUtcTimestamp();
database_path_ = BuildDatabasePath();
std::filesystem::create_directories(database_path_.parent_path());
run_timestamp_utc_ = date_time_provider_->GetUtcTimestamp();
database_path_ = BuildDatabasePath();
std::filesystem::create_directories(database_path_.parent_path());
db_handle_ = sqlite_export_service_internal::OpenDatabase(database_path_);
db_handle_ = sqlite_export_service_internal::OpenDatabase(database_path_);
try {
sqlite_export_service_internal::ExecSql(
db_handle_, "PRAGMA foreign_keys = ON;",
"Failed to enable SQLite foreign keys");
InitializeSchema();
PrepareStatements();
sqlite_export_service_internal::ExecSql(
db_handle_, "BEGIN IMMEDIATE TRANSACTION;",
"Failed to begin SQLite transaction");
transaction_open_ = true;
} catch (...) {
RollbackAndCloseNoThrow();
throw;
}
try {
sqlite_export_service_internal::ExecSql(
db_handle_, "PRAGMA foreign_keys = ON;",
"Failed to enable SQLite foreign keys");
InitializeSchema();
PrepareStatements();
sqlite_export_service_internal::ExecSql(
db_handle_, "BEGIN IMMEDIATE TRANSACTION;",
"Failed to begin SQLite transaction");
transaction_open_ = true;
} catch (...) {
RollbackAndCloseNoThrow();
throw;
}
}

275
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -12,164 +12,165 @@
#include "services/database/sqlite_export_service_helpers.h"
std::string SqliteExportService::BuildCityKey(const City& city) {
std::ostringstream key_stream;
key_stream << city.city << '\n'
<< city.state_province << '\n'
<< city.iso3166_2 << '\n'
<< city.country << '\n'
<< city.iso3166_1 << '\n'
<< sqlite_export_service_internal::SerializeVector(
city.local_languages);
return key_stream.str();
std::ostringstream key_stream;
key_stream << city.city << '\n'
<< city.state_province << '\n'
<< city.iso3166_2 << '\n'
<< city.country << '\n'
<< city.iso3166_1 << '\n'
<< sqlite_export_service_internal::SerializeVector(
city.local_languages);
return key_stream.str();
}
sqlite3_int64 SqliteExportService::ResolveCityId(const City& city) {
const std::string city_key = BuildCityKey(city);
const auto cached_city = city_cache_.find(city_key);
if (cached_city != city_cache_.end()) {
return cached_city->second;
}
const std::string city_key = BuildCityKey(city);
const auto cached_city = city_cache_.find(city_key);
if (cached_city != city_cache_.end()) {
return cached_city->second;
}
const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(city.local_languages);
const std::string city_regex_json =
sqlite_export_service_internal::SerializeVector(
city.postal_code.city_regexes);
const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(city.local_languages);
const std::string city_regex_json =
sqlite_export_service_internal::SerializeVector(
city.postal_code.city_regexes);
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityCityBindIndex,
.value = city.city,
.action = "Failed to bind SQLite city name"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityStateProvinceBindIndex,
.value = city.state_province,
.action = "Failed to bind SQLite city state/province"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityIso31662BindIndex,
.value = city.iso3166_2,
.action = "Failed to bind SQLite city ISO 3166-2 code"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityCountryBindIndex,
.value = city.country,
.action = "Failed to bind SQLite city country"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
.value = city.iso3166_1,
.action = "Failed to bind SQLite city ISO 3166-1 code"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityLanguagesBindIndex,
.value = local_languages_json,
.action = "Failed to bind SQLite city languages"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCountryFormatRegexBindIndex,
.value = city.postal_code.country_format_regex,
.action = "Failed to bind SQLite city postal-code format regex"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCityRegexJsonBindIndex,
.value = city_regex_json,
.action = "Failed to bind SQLite city postal-code city regexes"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityCityBindIndex,
.value = city.city,
.action = "Failed to bind SQLite city name"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityStateProvinceBindIndex,
.value = city.state_province,
.action = "Failed to bind SQLite city state/province"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityIso31662BindIndex,
.value = city.iso3166_2,
.action = "Failed to bind SQLite city ISO 3166-2 code"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityCountryBindIndex,
.value = city.country,
.action = "Failed to bind SQLite city country"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityIso31661BindIndex,
.value = city.iso3166_1,
.action = "Failed to bind SQLite city ISO 3166-1 code"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityLanguagesBindIndex,
.value = local_languages_json,
.action = "Failed to bind SQLite city languages"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCountryFormatRegexBindIndex,
.value = city.postal_code.country_format_regex,
.action = "Failed to bind SQLite city postal-code format regex"});
sqlite_export_service_internal::Bind(
insert_city_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kCityPostalCodeCityRegexJsonBindIndex,
.value = city_regex_json,
.action = "Failed to bind SQLite city postal-code city regexes"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_city_stmt_, "Failed to insert SQLite city row");
sqlite_export_service_internal::StepStatement(
db_handle_, insert_city_stmt_, "Failed to insert SQLite city row");
const sqlite3_int64 city_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
city_cache_.emplace(city_key, city_id);
sqlite_export_service_internal::ResetStatement(insert_city_stmt_);
const sqlite3_int64 city_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
city_cache_.emplace(city_key, city_id);
sqlite_export_service_internal::ResetStatement(insert_city_stmt_);
return city_id;
return city_id;
}
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
if (db_handle_ == nullptr || !transaction_open_) {
throw std::runtime_error("SQLite export service is not initialized");
}
if (db_handle_ == nullptr || !transaction_open_) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 city_id = ResolveCityId(brewery.address.city);
const sqlite3_int64 city_id = ResolveCityId(brewery.address.city);
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kBreweryEnglishNameBindIndex,
.value = brewery.brewery.name_en,
.action = "Failed to bind SQLite brewery English name"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kBreweryEnglishNameBindIndex,
.value = brewery.brewery.name_en,
.action = "Failed to bind SQLite brewery English name"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryEnglishDescriptionBindIndex,
.value = brewery.brewery.description_en,
.action = "Failed to bind SQLite brewery English description"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryEnglishDescriptionBindIndex,
.value = brewery.brewery.description_en,
.action = "Failed to bind SQLite brewery English description"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kBreweryLocalNameBindIndex,
.value = brewery.brewery.name_local,
.action = "Failed to bind SQLite brewery local name"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kBreweryLocalNameBindIndex,
.value = brewery.brewery.name_local,
.action = "Failed to bind SQLite brewery local name"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kBreweryLocalDescriptionBindIndex,
.value = brewery.brewery.description_local,
.action = "Failed to bind SQLite brewery local description"});
sqlite_export_service_internal::Bind(
insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryLocalDescriptionBindIndex,
.value = brewery.brewery.description_local,
.action = "Failed to bind SQLite brewery local description"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
const sqlite3_int64 brewery_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
const sqlite3_int64 brewery_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressBreweryIdBindIndex,
.value = brewery_id,
.action = "Failed to bind SQLite brewery address brewery id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::
kBreweryAddressBreweryIdBindIndex,
.value = brewery_id,
.action = "Failed to bind SQLite brewery address brewery id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_address_stmt_,
"Failed to insert SQLite brewery address row");
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_address_stmt_,
"Failed to insert SQLite brewery address row");
sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_);
sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_);
return static_cast<uint64_t>(brewery_id);
return static_cast<uint64_t>(brewery_id);
}

163
tooling/pipeline/src/services/sqlite/process_user_record.cc
View File

@@ -9,94 +9,95 @@
#include "services/database/sqlite_export_service_helpers.h"
uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
if (db_handle_ == nullptr || !transaction_open_) {
throw std::runtime_error("SQLite export service is not initialized");
}
if (db_handle_ == nullptr || !transaction_open_) {
throw std::runtime_error("SQLite export service is not initialized");
}
const sqlite3_int64 city_id = ResolveCityId(user.address.city);
const sqlite3_int64 city_id = ResolveCityId(user.address.city);
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserFirstNameBindIndex,
.value = user.user.first_name,
.action = "Failed to bind SQLite user first name"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserLastNameBindIndex,
.value = user.user.last_name,
.action = "Failed to bind SQLite user last name"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserGenderBindIndex,
.value = user.user.gender,
.action = "Failed to bind SQLite user gender"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserUsernameBindIndex,
.value = user.user.username,
.action = "Failed to bind SQLite user username"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserBioBindIndex,
.value = user.user.bio,
.action = "Failed to bind SQLite user bio"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index = sqlite_export_service_internal::kUserActivityWeightBindIndex,
.value = static_cast<double>(user.user.activity_weight),
.action = "Failed to bind SQLite user activity weight"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserEmailBindIndex,
.value = user.email,
.action = "Failed to bind SQLite user email"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserDateOfBirthBindIndex,
.value = user.date_of_birth,
.action = "Failed to bind SQLite user date of birth"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserFirstNameBindIndex,
.value = user.user.first_name,
.action = "Failed to bind SQLite user first name"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserLastNameBindIndex,
.value = user.user.last_name,
.action = "Failed to bind SQLite user last name"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserGenderBindIndex,
.value = user.user.gender,
.action = "Failed to bind SQLite user gender"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserUsernameBindIndex,
.value = user.user.username,
.action = "Failed to bind SQLite user username"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserBioBindIndex,
.value = user.user.bio,
.action = "Failed to bind SQLite user bio"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index =
sqlite_export_service_internal::kUserActivityWeightBindIndex,
.value = static_cast<double>(user.user.activity_weight),
.action = "Failed to bind SQLite user activity weight"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserEmailBindIndex,
.value = user.email,
.action = "Failed to bind SQLite user email"});
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kUserDateOfBirthBindIndex,
.value = user.date_of_birth,
.action = "Failed to bind SQLite user date of birth"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
const sqlite3_int64 user_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
const sqlite3_int64 user_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressUserIdBindIndex,
.value = user_id,
.action = "Failed to bind SQLite user address user id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
.value = user.address.postal_code,
.action = "Failed to bind SQLite user address postal code"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressUserIdBindIndex,
.value = user_id,
.action = "Failed to bind SQLite user address user id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index =
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
.value = user.address.postal_code,
.action = "Failed to bind SQLite user address postal code"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_,
"Failed to insert SQLite user address row");
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_,
"Failed to insert SQLite user address row");
sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_);
sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_);
return static_cast<uint64_t>(user_id);
return static_cast<uint64_t>(user_id);
}

6
tooling/pipeline/src/services/sqlite/sqlite_export_service.cc
View File

@@ -12,7 +12,7 @@ SqliteExportService::SqliteExportService(const ApplicationOptions& options)
output_path_(options.pipeline.output_path) {}
SqliteExportService::~SqliteExportService() {
if (db_handle_ != nullptr) {
RollbackAndCloseNoThrow();
}
if (db_handle_ != nullptr) {
RollbackAndCloseNoThrow();
}
}

8
tooling/pipeline/src/services/xeger/xeger.cc
View File

@@ -16,8 +16,8 @@
std::string GenerateStringFromRegex(std::string_view pattern, std::mt19937& rng,
const int unbounded_repeat_cap) {
xeger_internal::Parser parser(pattern);
const xeger_internal::NodePtr ast = parser.Parse();
xeger_internal::XegerGenerator generator(rng, unbounded_repeat_cap);
return generator.Run(*ast);
xeger_internal::Parser parser(pattern);
const xeger_internal::NodePtr ast = parser.Parse();
xeger_internal::XegerGenerator generator(rng, unbounded_repeat_cap);
return generator.Run(*ast);
}

124
tooling/pipeline/src/services/xeger/xeger_generator.cc
View File

@@ -3,108 +3,108 @@
* @brief Visitor that walks a regex AST and emits a random matching string.
*/
#include "services/xeger/xeger_internal.h"
#include <cassert>
#include <limits>
#include <random>
#include <stdexcept>
#include <string>
#include "services/xeger/xeger_internal.h"
namespace xeger_internal {
XegerGenerator::XegerGenerator(std::mt19937& rng, const int unbounded_cap)
: rng_(rng), unbounded_cap_(unbounded_cap) {}
std::string XegerGenerator::Run(const Node& root) {
out_.clear();
root.Accept(*this);
return out_;
out_.clear();
root.Accept(*this);
return out_;
}
void XegerGenerator::Visit(const Literal& node) { out_ += node.ch; }
void XegerGenerator::Visit(const AnyChar&) {
static constexpr int kPrintableLow = ' '; // empty space - character 32
static constexpr int kPrintableHigh = '~'; // tilde - character 126
static constexpr int kPrintableLow = ' '; // empty space - character 32
static constexpr int kPrintableHigh = '~'; // tilde - character 126
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
// assert that the rand number is in the char range
bool inCharRange = dist.min() >= std::numeric_limits<char>::min() &&
dist.max() <= std::numeric_limits<char>::max();
assert(inCharRange && "Random number is out of char range");
// assert that the rand number is in the char range
bool inCharRange = dist.min() >= std::numeric_limits<char>::min() &&
dist.max() <= std::numeric_limits<char>::max();
assert(inCharRange && "Random number is out of char range");
out_ += static_cast<char>(dist(rng_));
out_ += static_cast<char>(dist(rng_));
}
void XegerGenerator::Visit(const CharClass& node) {
out_ += PickFromClass(node);
out_ += PickFromClass(node);
}
void XegerGenerator::Visit(const Concat& node) {
for (const NodePtr& part : node.parts) {
part->Accept(*this);
}
for (const NodePtr& part : node.parts) {
part->Accept(*this);
}
}
void XegerGenerator::Visit(const Alternation& node) {
std::uniform_int_distribution<size_t> dist(0, node.options.size() - 1);
node.options[dist(rng_)]->Accept(*this);
std::uniform_int_distribution<size_t> dist(0, node.options.size() - 1);
node.options[dist(rng_)]->Accept(*this);
}
void XegerGenerator::Visit(const Repeat& node) {
const int lo = node.min;
const int hi = (node.max < 0) ? node.min + unbounded_cap_ : node.max;
std::uniform_int_distribution<int> dist(lo, hi);
const int count = dist(rng_);
for (int i = 0; i < count; ++i) {
node.child->Accept(*this);
}
const int lo = node.min;
const int hi = (node.max < 0) ? node.min + unbounded_cap_ : node.max;
std::uniform_int_distribution<int> dist(lo, hi);
const int count = dist(rng_);
for (int i = 0; i < count; ++i) {
node.child->Accept(*this);
}
}
bool XegerGenerator::InRanges(const CharClass& node, const char c) {
for (const auto& [lo, hi] : node.ranges) {
if (c >= lo && c <= hi) {
return true;
}
}
return false;
for (const auto& [lo, hi] : node.ranges) {
if (c >= lo && c <= hi) {
return true;
}
}
return false;
}
char XegerGenerator::PickFromClass(const CharClass& node) {
if (!node.negated) {
int total = 0;
for (const auto& [lo, hi] : node.ranges) {
total += hi - lo + 1;
}
std::uniform_int_distribution<int> dist(0, total - 1);
int k = dist(rng_);
for (const auto& [lo, hi] : node.ranges) {
const int width = hi - lo + 1;
if (k < width) {
return static_cast<char>(lo + k);
if (!node.negated) {
int total = 0;
for (const auto& [lo, hi] : node.ranges) {
total += hi - lo + 1;
}
k -= width;
}
return node.ranges.back().second; // unreachable
}
std::uniform_int_distribution<int> dist(0, total - 1);
int k = dist(rng_);
for (const auto& [lo, hi] : node.ranges) {
const int width = hi - lo + 1;
if (k < width) {
return static_cast<char>(lo + k);
}
k -= width;
}
return node.ranges.back().second; // unreachable
}
// Negated class: draw random printable characters until one falls
// outside the forbidden set.
static constexpr int kPrintableLow = 32;
static constexpr int kPrintableHigh = 126;
static constexpr int kMaxAttempts = 1000;
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
const char c = static_cast<char>(dist(rng_));
if (!InRanges(node, c)) {
return c;
}
}
throw std::runtime_error(
"GenerateStringFromRegex: negated class rules out every printable "
"character");
// Negated class: draw random printable characters until one falls
// outside the forbidden set.
static constexpr int kPrintableLow = 32;
static constexpr int kPrintableHigh = 126;
static constexpr int kMaxAttempts = 1000;
std::uniform_int_distribution<int> dist(kPrintableLow, kPrintableHigh);
for (int attempt = 0; attempt < kMaxAttempts; ++attempt) {
const char c = static_cast<char>(dist(rng_));
if (!InRanges(node, c)) {
return c;
}
}
throw std::runtime_error(
"GenerateStringFromRegex: negated class rules out every printable "
"character");
}
} // namespace xeger_internal

479
tooling/pipeline/src/services/xeger/xeger_parser.cc
View File

@@ -3,8 +3,6 @@
* @brief Recursive-descent parser that turns a regex pattern into an AST.
*/
#include "services/xeger/xeger_internal.h"
#include <cctype>
#include <format>
#include <memory>
@@ -14,31 +12,33 @@
#include <utility>
#include <vector>
#include "services/xeger/xeger_internal.h"
namespace xeger_internal {
Parser::Parser(std::string_view pattern) : src_(StripAnchors(pattern)) {}
NodePtr Parser::Parse() {
NodePtr node = ParseAlternation();
if (!AtEnd()) {
throw std::runtime_error(
std::format("GenerateStringFromRegex: unexpected '{}' at position {}",
Peek(), pos_));
}
return node;
NodePtr node = ParseAlternation();
if (!AtEnd()) {
throw std::runtime_error(
std::format("GenerateStringFromRegex: unexpected '{}' at position {}",
Peek(), pos_));
}
return node;
}
std::string Parser::StripAnchors(std::string_view pattern) {
if (!pattern.empty() && pattern.front() == kAnchorStart) {
pattern.remove_prefix(1);
}
const bool has_trailing_dollar =
!pattern.empty() && pattern.back() == kAnchorEnd &&
(pattern.size() < 2 || pattern[pattern.size() - 2] != kEscape);
if (has_trailing_dollar) {
pattern.remove_suffix(1);
}
return std::string(pattern);
if (!pattern.empty() && pattern.front() == kAnchorStart) {
pattern.remove_prefix(1);
}
const bool has_trailing_dollar =
!pattern.empty() && pattern.back() == kAnchorEnd &&
(pattern.size() < 2 || pattern[pattern.size() - 2] != kEscape);
if (has_trailing_dollar) {
pattern.remove_suffix(1);
}
return std::string(pattern);
}
bool Parser::AtEnd() const { return pos_ >= src_.size(); }
@@ -46,262 +46,263 @@ char Parser::Peek() const { return src_[pos_]; }
char Parser::Advance() { return src_[pos_++]; }
bool Parser::Match(const char c) {
if (!AtEnd() && Peek() == c) {
++pos_;
return true;
}
return false;
if (!AtEnd() && Peek() == c) {
++pos_;
return true;
}
return false;
}
void Parser::Expect(const char c) {
if (!Match(c)) {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: expected '{}' at position {}", c, pos_));
}
if (!Match(c)) {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: expected '{}' at position {}", c, pos_));
}
}
NodePtr Parser::ParseAlternation() {
std::vector<NodePtr> options;
options.push_back(ParseConcat());
while (Match(kAlternation)) {
options.push_back(ParseConcat());
}
if (options.size() == 1) {
return std::move(options.front());
}
auto alternation = std::make_unique<Alternation>();
alternation->options = std::move(options);
return alternation;
std::vector<NodePtr> options;
options.push_back(ParseConcat());
while (Match(kAlternation)) {
options.push_back(ParseConcat());
}
if (options.size() == 1) {
return std::move(options.front());
}
auto alternation = std::make_unique<Alternation>();
alternation->options = std::move(options);
return alternation;
}
NodePtr Parser::ParseConcat() {
auto concat = std::make_unique<Concat>();
while (!AtEnd() && Peek() != kAlternation && Peek() != kGroupClose) {
concat->parts.push_back(ParseRepeat());
}
if (concat->parts.size() == 1) {
return std::move(concat->parts.front());
}
return concat;
auto concat = std::make_unique<Concat>();
while (!AtEnd() && Peek() != kAlternation && Peek() != kGroupClose) {
concat->parts.push_back(ParseRepeat());
}
if (concat->parts.size() == 1) {
return std::move(concat->parts.front());
}
return concat;
}
NodePtr Parser::ParseRepeat() {
NodePtr atom = ParseAtom();
if (AtEnd()) {
return atom;
}
int lo = 0;
int hi = -1;
switch (Peek()) {
case kStar:
Advance();
lo = 0;
hi = -1;
break;
case kPlus:
Advance();
lo = 1;
hi = -1;
break;
case kOptional:
Advance();
lo = 0;
hi = 1;
break;
case kBraceOpen:
return ParseBrace(std::move(atom));
default:
NodePtr atom = ParseAtom();
if (AtEnd()) {
return atom;
}
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lo;
repeat->max = hi;
return repeat;
}
int lo = 0;
int hi = -1;
switch (Peek()) {
case kStar:
Advance();
lo = 0;
hi = -1;
break;
case kPlus:
Advance();
lo = 1;
hi = -1;
break;
case kOptional:
Advance();
lo = 0;
hi = 1;
break;
case kBraceOpen:
return ParseBrace(std::move(atom));
default:
return atom;
}
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lo;
repeat->max = hi;
return repeat;
}
NodePtr Parser::ParseBrace(NodePtr atom) {
Expect(kBraceOpen);
std::string lower_digits;
std::string upper_digits;
bool has_comma = false;
while (!AtEnd() && std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
lower_digits += Advance();
}
if (Match(kBraceComma)) {
has_comma = true;
while (!AtEnd() &&
std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
upper_digits += Advance();
}
}
Expect(kBraceClose);
Expect(kBraceOpen);
std::string lower_digits;
std::string upper_digits;
bool has_comma = false;
while (!AtEnd() && std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
lower_digits += Advance();
}
if (Match(kBraceComma)) {
has_comma = true;
while (!AtEnd() &&
std::isdigit(static_cast<unsigned char>(Peek())) != 0) {
upper_digits += Advance();
}
}
Expect(kBraceClose);
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lower_digits.empty() ? 0 : std::stoi(lower_digits);
if (!has_comma) {
repeat->max = repeat->min;
} else if (upper_digits.empty()) {
repeat->max = -1;
} else {
repeat->max = std::stoi(upper_digits);
}
if (repeat->max != -1 && repeat->max < repeat->min) {
throw std::runtime_error(
"GenerateStringFromRegex: repetition max is smaller than min");
}
return repeat;
auto repeat = std::make_unique<Repeat>();
repeat->child = std::move(atom);
repeat->min = lower_digits.empty() ? 0 : std::stoi(lower_digits);
if (!has_comma) {
repeat->max = repeat->min;
} else if (upper_digits.empty()) {
repeat->max = -1;
} else {
repeat->max = std::stoi(upper_digits);
}
if (repeat->max != -1 && repeat->max < repeat->min) {
throw std::runtime_error(
"GenerateStringFromRegex: repetition max is smaller than min");
}
return repeat;
}
NodePtr Parser::ParseAtom() {
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: pattern ended unexpectedly");
}
const char c = Peek();
switch (c) {
case kGroupOpen: {
Advance();
// Non-capturing group "(?:...)" -- capture semantics don't apply
// here (nothing back-references a group), so it's treated exactly
// like a plain group once the "?:" marker is consumed.
if (!AtEnd() && Peek() == kNonCapturingMarker) {
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == kNonCapturingColon) {
pos_ += 2;
} else {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: unsupported group syntax at "
"position {} (only plain '(...)' and non-capturing "
"'(?:...)' groups are supported)",
pos_));
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: pattern ended unexpectedly");
}
const char c = Peek();
switch (c) {
case kGroupOpen: {
Advance();
// Non-capturing group "(?:...)" -- capture semantics don't apply
// here (nothing back-references a group), so it's treated exactly
// like a plain group once the "?:" marker is consumed.
if (!AtEnd() && Peek() == kNonCapturingMarker) {
if (pos_ + 1 < src_.size() &&
src_[pos_ + 1] == kNonCapturingColon) {
pos_ += 2;
} else {
throw std::runtime_error(std::format(
"GenerateStringFromRegex: unsupported group syntax at "
"position {} (only plain '(...)' and non-capturing "
"'(?:...)' groups are supported)",
pos_));
}
}
NodePtr inner = ParseAlternation();
Expect(kGroupClose);
return inner;
}
NodePtr inner = ParseAlternation();
Expect(kGroupClose);
return inner;
}
case kClassOpen:
return ParseCharClass();
case kAnyChar:
Advance();
return std::make_unique<AnyChar>();
case kEscape:
Advance();
return ParseEscape();
case kStar:
case kPlus:
case kOptional:
case kBraceOpen:
throw std::runtime_error(std::format(
"GenerateStringFromRegex: quantifier with no preceding item at "
"position {}",
pos_));
default:
Advance();
return std::make_unique<Literal>(c);
}
case kClassOpen:
return ParseCharClass();
case kAnyChar:
Advance();
return std::make_unique<AnyChar>();
case kEscape:
Advance();
return ParseEscape();
case kStar:
case kPlus:
case kOptional:
case kBraceOpen:
throw std::runtime_error(std::format(
"GenerateStringFromRegex: quantifier with no preceding item at "
"position {}",
pos_));
default:
Advance();
return std::make_unique<Literal>(c);
}
}
NodePtr Parser::MakeClass(std::vector<std::pair<char, char>> ranges,
const bool negated) {
auto char_class = std::make_unique<CharClass>();
char_class->ranges = std::move(ranges);
char_class->negated = negated;
return char_class;
auto char_class = std::make_unique<CharClass>();
char_class->ranges = std::move(ranges);
char_class->negated = negated;
return char_class;
}
NodePtr Parser::ParseEscape() {
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: a backslash at the very end has nothing "
"to escape");
}
const char c = Advance();
switch (c) {
case 'd':
return MakeClass({{'0', '9'}}, false);
case 'D':
return MakeClass({{'0', '9'}}, true);
case 'w':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
false);
case 'W':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
true);
case 's':
return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}},
false);
case 'S':
return MakeClass({{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}},
true);
case 'n':
return std::make_unique<Literal>('\n');
case 't':
return std::make_unique<Literal>('\t');
case 'r':
return std::make_unique<Literal>('\r');
default:
// \. \* \( \\ etc. -- the literal character itself.
return std::make_unique<Literal>(c);
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: a backslash at the very end has nothing "
"to escape");
}
const char c = Advance();
switch (c) {
case 'd':
return MakeClass({{'0', '9'}}, false);
case 'D':
return MakeClass({{'0', '9'}}, true);
case 'w':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
false);
case 'W':
return MakeClass({{'a', 'z'}, {'A', 'Z'}, {'0', '9'}, {'_', '_'}},
true);
case 's':
return MakeClass(
{{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}}, false);
case 'S':
return MakeClass(
{{' ', ' '}, {'\t', '\t'}, {'\n', '\n'}, {'\r', '\r'}}, true);
case 'n':
return std::make_unique<Literal>('\n');
case 't':
return std::make_unique<Literal>('\t');
case 'r':
return std::make_unique<Literal>('\r');
default:
// \. \* \( \\ etc. -- the literal character itself.
return std::make_unique<Literal>(c);
}
}
NodePtr Parser::ParseCharClass() {
Expect(kClassOpen);
auto char_class = std::make_unique<CharClass>();
if (Match(kClassNegate)) {
char_class->negated = true;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: character class '[' is never closed");
}
Expect(kClassOpen);
auto char_class = std::make_unique<CharClass>();
if (Match(kClassNegate)) {
char_class->negated = true;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: character class '[' is never closed");
}
while (!AtEnd() && Peek() != kClassClose) {
char lo = ClassChar();
const bool is_range = Peek() == kClassRange && pos_ + 1 < src_.size() &&
src_[pos_ + 1] != kClassClose;
if (is_range) {
Advance();
char hi = ClassChar();
if (hi < lo) {
std::swap(lo, hi);
while (!AtEnd() && Peek() != kClassClose) {
char lo = ClassChar();
const bool is_range = Peek() == kClassRange && pos_ + 1 < src_.size() &&
src_[pos_ + 1] != kClassClose;
if (is_range) {
Advance();
char hi = ClassChar();
if (hi < lo) {
std::swap(lo, hi);
}
char_class->ranges.emplace_back(lo, hi);
} else {
char_class->ranges.emplace_back(lo, lo);
}
char_class->ranges.emplace_back(lo, hi);
} else {
char_class->ranges.emplace_back(lo, lo);
}
}
Expect(kClassClose);
if (char_class->ranges.empty()) {
throw std::runtime_error(
"GenerateStringFromRegex: empty character class []");
}
return char_class;
}
Expect(kClassClose);
if (char_class->ranges.empty()) {
throw std::runtime_error(
"GenerateStringFromRegex: empty character class []");
}
return char_class;
}
char Parser::ClassChar() {
const char c = Advance();
if (c != kEscape) {
return c;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: backslash at end of character class");
}
const char escaped = Advance();
switch (escaped) {
case 'n':
return '\n';
case 't':
return '\t';
case 'r':
return '\r';
default:
return escaped;
}
const char c = Advance();
if (c != kEscape) {
return c;
}
if (AtEnd()) {
throw std::runtime_error(
"GenerateStringFromRegex: backslash at end of character class");
}
const char escaped = Advance();
switch (escaped) {
case 'n':
return '\n';
case 't':
return '\t';
case 'r':
return '\r';
default:
return escaped;
}
}
} // namespace xeger_internal

60
tooling/pipeline/src/web_client/http_web_client.cc
View File

@@ -26,47 +26,47 @@ const std::regex kUrlRegex(
R"(^(https?://[^/?#]+)(/[^?#]*(?:\?[^#]*)?(?:#.*)?)?)");
std::pair<std::string, std::string> SplitUrl(const std::string& url) {
std::smatch match;
if (!std::regex_match(url, match, kUrlRegex)) {
throw std::invalid_argument("[HttpWebClient] Malformed URL: " + url);
}
std::smatch match;
if (!std::regex_match(url, match, kUrlRegex)) {
throw std::invalid_argument("[HttpWebClient] Malformed URL: " + url);
}
return {match[1].str(), match[2].matched ? match[2].str() : "/"};
return {match[1].str(), match[2].matched ? match[2].str() : "/"};
}
} // namespace
std::string HttpWebClient::Get(const std::string& url) {
const auto [origin, path] = SplitUrl(url);
const auto [origin, path] = SplitUrl(url);
httplib::Client client(origin);
client.set_follow_location(true);
client.set_connection_timeout(kConnectionTimeoutSeconds);
client.set_read_timeout(kReadTimeoutSeconds);
client.set_default_headers({{"Accept", "application/json"},
{"User-Agent", "biergarten-pipeline/1.0"}});
httplib::Client client(origin);
client.set_follow_location(true);
client.set_connection_timeout(kConnectionTimeoutSeconds);
client.set_read_timeout(kReadTimeoutSeconds);
client.set_default_headers({{"Accept", "application/json"},
{"User-Agent", "biergarten-pipeline/1.0"}});
const httplib::Result result = client.Get(path);
const httplib::Result result = client.Get(path);
if (!result) {
throw std::runtime_error(
std::format("[HttpWebClient] Request failed for URL: {} — {}", url,
httplib::to_string(result.error())));
}
if (!result) {
throw std::runtime_error(
std::format("[HttpWebClient] Request failed for URL: {} — {}", url,
httplib::to_string(result.error())));
}
if (result->status < kSuccessMin || result->status >= kSuccessMax) {
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[HttpWebClient] Request failed for URL: {}", url)});
}
throw std::runtime_error(std::format("[HttpWebClient] HTTP {} for URL: {}",
result->status, url));
}
if (result->status < kSuccessMin || result->status >= kSuccessMax) {
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[HttpWebClient] Request failed for URL: {}", url)});
}
throw std::runtime_error(std::format(
"[HttpWebClient] HTTP {} for URL: {}", result->status, url));
}
return result->body;
return result->body;
}
std::string HttpWebClient::EncodeURL(const std::string& value) {
return httplib::encode_uri_component(value);
return httplib::encode_uri_component(value);
}