AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 09:37:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
Aaron Po
2026-06-21 13:13:02 -04:00
parent cbadd3bb59
commit 75e0b1aa94
27 changed files with 313 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tooling/pipeline/includes/biergarten_pipeline_orchestrator.h
View File

@@ -17,7 +17,6 @@
#include "data_model/generated_models.h"
#include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
/**
@@ -32,7 +31,8 @@ class BiergartenPipelineOrchestrator {
* @brief Constructs the orchestrator with injected pipeline dependencies.
*
* @param context_service Provides regional context for locations.
* @param generator Implementation (Llama or Mock) for brewery/user generation.
* @param generator Implementation (Llama or Mock) for brewery/user
* generation.
* @param exporter Database backend for persisting generated records.
* @param application_options CLI configuration and paths.
*/
@@ -52,10 +52,11 @@ class BiergartenPipelineOrchestrator {
* 3. Generate brewery data for sampled cities
*
* @note STRUCTURAL CONCURRENCY REQUIREMENT:
* When transitioned to a multithreaded design, this method MUST structurally
* enforce that all deployed worker threads are joined before returning (e.g.
* by using std::jthread or a structured concurrency primitive). This ensures
* workers do not attempt to log to a closed channel during application teardown.
* When transitioned to a multithreaded design, this method MUST
* structurally enforce that all deployed worker threads are joined before
* returning (e.g. by using std::jthread or a structured concurrency
* primitive). This ensures workers do not attempt to log to a closed
* channel during application teardown.
*
* @return true if successful, false if not
*/
@@ -95,9 +96,9 @@ class BiergartenPipelineOrchestrator {
* @brief Generate users grounded in sampled names and personas for
* enriched cities.
*
* Loads personas.json / forenames-by-country.json / surnames-by-country.json
* itself, mirroring how QueryCitiesWithCountries() owns its own
* locations.json load.
* Loads personas.json / forenames-by-country.json /
* surnames-by-country.json itself, mirroring how QueryCitiesWithCountries()
* owns its own locations.json load.
*
* @param cities Span of enriched city data.
*/

6
tooling/pipeline/includes/concurrency/bounded_channel.h
View File

@@ -9,10 +9,12 @@
/**
* @file bounded_channel.h
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous channel.
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous
* channel.
*
* Intent: Enables asynchronous inter-thread communication with backpressure.
* Models a synchronous channel where producers/consumers block on capacity limits.
* Models a synchronous channel where producers/consumers block on capacity
* limits.
*/
/**

6
tooling/pipeline/includes/data_generation/mock_generator.h
View File

@@ -57,8 +57,7 @@ class MockGenerator final : public DataGenerator {
* @return Deterministic hash value.
*/
static size_t DeterministicHash(const Location& location,
const UserPersona& persona,
const Name& name);
const UserPersona& persona, const Name& name);
// Hash stride constants for deterministic distribution across fixed-size
// arrays. These coprime strides spread hash values uniformly without
@@ -142,7 +141,8 @@ class MockGenerator final : public DataGenerator {
"Visits breweries for the stories, stays for the flagship pours.",
"Craft beer fan mapping tasting notes and favorite brew routes.",
"Always ready to trade recommendations for underrated local breweries.",
"Keeping a running list of must-try collab releases and tap takeovers."};
"Keeping a running list of must-try collab releases and tap "
"takeovers."};
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_MOCK_GENERATOR_H_

4
tooling/pipeline/includes/data_model/generated_models.h
View File

@@ -3,8 +3,8 @@
/**
* @file data_model/generated_models.h
* @brief Generated output models from the pipeline: brewery/user results, enriched data,
* and complete generation results.
* @brief Generated output models from the pipeline: brewery/user results,
* enriched data, and complete generation results.
*/
#include <string>

6
tooling/pipeline/includes/data_model/models.h
View File

@@ -148,8 +148,6 @@ struct GeneratorOptions {
/// @brief Use mocked generator instead of actual LLM inference.
bool use_mocked = false;
/// @brief Specific sampling parameters for this generator.
/// If nullopt, the application should use global defaults.
std::optional<SamplingOptions> sampling;
@@ -186,7 +184,7 @@ struct ApplicationOptions {
// Function Declarations
// ============================================================================
std::optional<ApplicationOptions> ParseArguments(const int argc, char** argv,
std::shared_ptr<ILogger> logger = nullptr);
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger = nullptr);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_MODELS_H_

3
tooling/pipeline/includes/data_model/names_by_country.h
View File

@@ -25,8 +25,7 @@
*/
class NamesByCountry {
public:
NamesByCountry(
std::unordered_map<std::string, std::vector<ForenameEntry>>
NamesByCountry(std::unordered_map<std::string, std::vector<ForenameEntry>>
forenames_by_country,
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country);

2
tooling/pipeline/includes/services/database/sqlite_export_service.h
View File

@@ -11,8 +11,8 @@
#include <string>
#include <unordered_map>
#include "data_model/models.h"
#include "../datetime/date_time_provider.h"
#include "data_model/models.h"
#include "export_service.h"
#include "sqlite_export_service_helpers.h"

5
tooling/pipeline/includes/services/logging/log_entry.h
View File

@@ -46,7 +46,8 @@ enum class PipelinePhase {
/**
* @struct LogDTO
* @brief User-provided subset of log fields. Used to capture call-site info transparently.
* @brief User-provided subset of log fields. Used to capture call-site info
* transparently.
*/
struct LogDTO {
LogLevel level;
@@ -74,7 +75,6 @@ struct LogEntry {
/// @brief Thread responsible for emitting the log.
std::thread::id thread_id{};
/// @brief Severity level of this entry.
LogLevel level;
@@ -83,7 +83,6 @@ struct LogEntry {
/// @brief Log message text.
std::string message;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_

3
tooling/pipeline/includes/services/logging/logger.h
View File

@@ -39,7 +39,8 @@ class ILogger {
*/
void Log(LogDTO payload,
std::source_location origin = std::source_location::current(),
std::chrono::system_clock::time_point timestamp = std::chrono::system_clock::now(),
std::chrono::system_clock::time_point timestamp =
std::chrono::system_clock::now(),
std::thread::id thread_id = std::this_thread::get_id()) {
LogEntry entry;
entry.timestamp = timestamp;

11
tooling/pipeline/includes/web_client/http_web_client.h
View File

@@ -6,14 +6,13 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
#define BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
#include "web_client/web_client.h"
#include "services/logging/logger.h"
#include <memory>
#include <string>
#include <utility>
#include "services/logging/logger.h"
#include "web_client/web_client.h"
/**
* @brief WebClient implementation backed by cpp-httplib.
*
@@ -34,7 +33,8 @@ public:
/**
* @brief Executes a blocking HTTP/HTTPS GET request against a full URL.
*
* @param url Fully-qualified URL, e.g. "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
* @param url Fully-qualified URL, e.g.
* "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
* @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
*/
std::string Get(const std::string& url) override;
@@ -52,5 +52,4 @@ public:
std::shared_ptr<ILogger> logger_;
};
#endif

15
tooling/pipeline/src/application_options/parse_arguments.cc
View File

@@ -51,7 +51,8 @@ std::optional<ApplicationOptions> ParseArguments(
prog_opts::value<std::string>()->default_value("pipeline.log"),
"Path for application logs");
opt("prompt-dir", prog_opts::value<std::string>()->default_value(""),
"Directory containing named prompt files (e.g. BREWERY_GENERATION.md)."
"Directory containing named prompt files (e.g. "
"BREWERY_GENERATION.md)."
" Required when not using --mocked.");
opt("location-count", prog_opts::value<uint32_t>()->default_value(10));
};
@@ -106,10 +107,12 @@ std::optional<ApplicationOptions> ParseArguments(
const std::string model_path = var_map["model"].as<std::string>();
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
// Enforce mutual exclusivity before any further configuration is applied.
// Enforce mutual exclusivity before any further configuration is
// applied.
if (use_mocked && !model_path.empty()) {
const std::string msg =
"Invalid arguments: --mocked and --model are mutually exclusive";
"Invalid arguments: --mocked and --model are mutually "
"exclusive";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
@@ -122,7 +125,8 @@ std::optional<ApplicationOptions> ParseArguments(
if (!use_mocked && model_path.empty()) {
const std::string msg =
"Invalid arguments: either --mocked or --model must be specified";
"Invalid arguments: either --mocked or --model must be "
"specified";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
@@ -137,7 +141,8 @@ std::optional<ApplicationOptions> ParseArguments(
// generator has no use for it and should not require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using --mocked";
"Invalid arguments: --prompt-dir is required when not using "
"--mocked";
if (logger) {
logger->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,

21
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_breweries.cc
View File

@@ -36,22 +36,26 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message =
std::format("[Pipeline] Generated brewery for '{}' ({}) but SQLite export failed: {}",
.message = std::format(
"[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}",
location.city, location.country, export_exception.what())});
}
} catch (const std::exception& e) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery generation failed: {}",
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery "
"generation failed: {}",
location.city, location.country, e.what())});
}
}
if (skipped_count > 0) {
logger_->Log({.level = LogLevel::Warn,
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to generation errors",
@@ -59,10 +63,11 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
}
if (export_failed_count > 0) {
logger_->Log({.level = LogLevel::Warn,
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format(
"[Pipeline] Failed to export {} generated brewery/breweries to SQLite",
.message = std::format("[Pipeline] Failed to export {} generated "
"brewery/breweries to SQLite",
export_failed_count)});
}
}

35
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -24,8 +24,8 @@ std::string Sanitize(std::string_view value) {
out.reserve(value.size());
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(
static_cast<char>(std::tolower(static_cast<unsigned char>(character))));
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
@@ -62,8 +62,7 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
sys_days{birth_year_anchor} - days{day_offset_dist(rng)};
const year_month_day birth_ymd{birth_date};
return std::format("{:04}-{:02}-{:02}",
static_cast<int>(birth_ymd.year()),
return std::format("{:04}-{:02}-{:02}", static_cast<int>(birth_ymd.year()),
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
}
@@ -71,7 +70,8 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
std::string GenerateRandomPassword(std::mt19937& rng) {
constexpr size_t kPasswordLength = 32;
constexpr std::string_view kCharset =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*";
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&"
"*";
std::uniform_int_distribution<size_t> char_dist(0, kCharset.size() - 1);
@@ -115,11 +115,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
if (!sampled_name.has_value()) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): no names available for "
"[Pipeline] Skipping city '{}' ({}): no names "
"available for "
"country '{}'",
city.location.city, city.location.country,
city.location.iso3166_1)});
@@ -147,21 +147,21 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Generated user for '{}' ({}) but SQLite export failed: {}",
"[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
city.location.city, city.location.country,
export_exception.what())});
}
} catch (const std::exception& e) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): user generation failed: {}",
"[Pipeline] Skipping city '{}' ({}): "
"user generation failed: {}",
city.location.city, city.location.country, e.what())});
}
}
@@ -176,11 +176,10 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
}
if (export_failed_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format(
"[Pipeline] Failed to export {} generated user/users to SQLite",
.message = std::format("[Pipeline] Failed to export {} "
"generated user/users to SQLite",
export_failed_count)});
}
}

1
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -38,7 +38,6 @@ void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array user_output;
for (const auto& generated_user : generated_users_) {
user_output.push_back(boost::json::object{
{"first_name", generated_user.user.first_name},

20
tooling/pipeline/src/biergarten_pipeline_orchestrator/run.cc
View File

@@ -28,20 +28,21 @@ bool BiergartenPipelineOrchestrator::Run() {
.region_context = std::move(region_context)});
} catch (const std::exception& exception) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): context lookup failed: {}",
"[Pipeline] Skipping city '{}' ({}): context "
"lookup failed: {}",
city.city, city.country, exception.what())});
}
}
if (skipped_count > 0) {
logger_->Log({.level = LogLevel::Warn,
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to context lookup errors",
.message = std::format("[Pipeline] Skipped {} city/cities due "
"to context lookup errors",
skipped_count)});
}
@@ -51,11 +52,10 @@ bool BiergartenPipelineOrchestrator::Run() {
this->LogResults();
return true;
} catch (const std::exception& e) {
logger_->Log(
{.level = LogLevel::Error,
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message =
std::format("Pipeline execution failed with error: {}", e.what())});
.message = std::format(
"Pipeline execution failed with error: {}", e.what())});
return false;
}
}

32
tooling/pipeline/src/data_generation/llama/generate_brewery.cc
View File

@@ -83,8 +83,8 @@ BreweryResult LlamaGenerator::GenerateBrewery(
/**
* RETRY LOOP with validation and error correction
* Attempts to generate valid brewery data up to 3 times, with feedback-based
* refinement
* Attempts to generate valid brewery data up to 3 times, with
* feedback-based refinement
*/
constexpr int max_attempts = 3;
std::string raw;
@@ -119,7 +119,8 @@ BreweryResult LlamaGenerator::GenerateBrewery(
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: successfully generated brewery data on attempt {}",
.message = std::format("LlamaGenerator: successfully generated "
"brewery data on attempt {}",
attempt + 1)});
}
@@ -133,32 +134,37 @@ BreweryResult LlamaGenerator::GenerateBrewery(
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message =
std::format("LlamaGenerator: malformed brewery JSON (attempt {}): {}",
.message = std::format(
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
// Update prompt with error details to guide LLM toward correct output.
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON with "
"exactly these keys, in this exact order: {{\"name_en\": \"<English "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"name_en\": "
"\"<English "
"brewery name>\", \"description_en\": \"<English single-paragraph "
"description>\", \"name_local\": \"<local-language brewery name>\", "
"description>\", \"name_local\": \"<local-language brewery "
"name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\nKeep the JSON strings concise enough "
"description>\"}}.\nDo not include markdown, comments, extra keys, "
"or "
"literal placeholder values.\n\nKeep the JSON strings concise "
"enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
// All retry attempts exhausted: log failure and throw exception
if (logger_) {
logger_->Log(
{.level = LogLevel::Error,
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery response after {} attempts: {}",
"LlamaGenerator: malformed brewery "
"response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed brewery response");

38
tooling/pipeline/src/data_generation/llama/generate_user.cc
View File

@@ -41,17 +41,17 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
std::string user_prompt = std::format(
"## NAME:\n{} {}\n\n## GENDER:\n{}\n\n## CITY:\n{}\n\n## COUNTRY:\n{}\n\n"
"## NAME:\n{} {}\n\n## GENDER:\n{}\n\n## CITY:\n{}\n\n## "
"COUNTRY:\n{}\n\n"
"## PERSONA:\n{}\n\n## PERSONA DESCRIPTION:\n{}\n\n## STYLE "
"AFFINITIES:\n{}",
name.first_name, name.last_name, name.gender, city.location.city,
city.location.country, persona.name, persona.description,
style_affinities);
const std::string retry_context =
std::format("Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
name.last_name, city.location.city, city.location.country,
persona.name);
const std::string retry_context = std::format(
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name, name.last_name,
city.location.city, city.location.country, persona.name);
constexpr int max_attempts = 3;
std::string raw;
@@ -59,8 +59,7 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
int max_tokens = kUserInitialMaxTokens;
for (int attempt = 0; attempt < max_attempts; ++attempt) {
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kUserJsonGrammar);
raw = this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
@@ -78,7 +77,8 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: successfully generated user data on attempt {}",
.message = std::format("LlamaGenerator: successfully "
"generated user data on attempt {}",
attempt + 1)});
}
@@ -90,31 +90,33 @@ UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message =
std::format("LlamaGenerator: malformed user JSON (attempt {}): {}",
.message = std::format(
"LlamaGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON with "
"exactly these keys, in this exact order: {{\"username\": \"<handle "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"username\": "
"\"<handle "
"derived from the given name>\", \"bio\": \"<first-person bio "
"grounded in the persona>\", \"activity_weight\": <number between 0 "
"grounded in the persona>\", \"activity_weight\": <number between "
"0 "
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Error,
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user response after {} attempts: {}",
"LlamaGenerator: malformed user response "
"after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed user response");

3
tooling/pipeline/src/data_model/names_by_country.cc
View File

@@ -29,8 +29,7 @@ std::optional<Name> NamesByCountry::SampleName(const std::string& iso3166_1,
const std::vector<ForenameEntry>& forenames = forenames_it->second;
const std::vector<std::string>& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0,
forenames.size() - 1);
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
const ForenameEntry& forename = forenames[forename_dist(rng)];

25
tooling/pipeline/src/json_handling/json_loader.cc
View File

@@ -6,12 +6,10 @@
#include "json_handling/json_loader.h"
#include <format>
#include "services/logging/logger.h"
#include <iostream>
#include <boost/json.hpp>
#include <format>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
@@ -19,6 +17,8 @@
#include <unordered_map>
#include <utility>
#include "services/logging/logger.h"
static std::string ReadRequiredString(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
@@ -67,8 +67,8 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
const char* what) {
std::ifstream input(filepath);
if (!input.is_open()) {
throw std::runtime_error(std::format("Failed to open {} file: {}", what,
filepath.string()));
throw std::runtime_error(
std::format("Failed to open {} file: {}", what, filepath.string()));
}
std::stringstream buffer;
@@ -87,8 +87,7 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
/// `fallback_key` if `key` is missing/empty (some source entries only have a
/// "localized" form and no "romanized" form).
std::string ReadFirstOfStringArray(const boost::json::object& object,
const char* key,
const char* fallback_key) {
const char* key, const char* fallback_key) {
for (const char* candidate_key : {key, fallback_key}) {
const boost::json::value* value = object.if_contains(candidate_key);
if (value == nullptr || !value->is_array()) {
@@ -163,8 +162,7 @@ std::vector<UserPersona> JsonLoader::LoadPersonas(
personas.push_back(UserPersona{
.name = ReadRequiredString(object, "name"),
.description = ReadRequiredString(object, "description"),
.style_affinities =
ReadRequiredStringArray(object, "style_affinities"),
.style_affinities = ReadRequiredStringArray(object, "style_affinities"),
});
}
@@ -207,8 +205,8 @@ NamesByCountry JsonLoader::LoadNamesByCountry(
}
const auto& name_object = name_value.as_object();
entries.push_back(ForenameEntry{
.name = ReadFirstOfStringArray(name_object, "romanized",
"localized"),
.name =
ReadFirstOfStringArray(name_object, "romanized", "localized"),
.gender = ReadRequiredString(name_object, "gender"),
});
}
@@ -216,8 +214,7 @@ NamesByCountry JsonLoader::LoadNamesByCountry(
forenames_by_country.emplace(country_code, std::move(entries));
}
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country;
std::unordered_map<std::string, std::vector<std::string>> surnames_by_country;
for (const auto& [country_code, name_entries] : surnames_root.as_object()) {
if (!name_entries.is_array()) {
continue;

3
tooling/pipeline/src/main.cc
View File

@@ -176,7 +176,8 @@ int main(const int argc, char** argv) {
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | temp={:.2f} "
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} "
"top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});

52
tooling/pipeline/src/services/enrichment/wikipedia/fetch_extract.cc
View File

@@ -20,7 +20,8 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (const auto cache_it = this->extract_cache_.find(cache_key);
cache_it != this->extract_cache_.end()) {
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format("Wikipedia: Cache hit for {}!", cache_key)});
}
@@ -30,7 +31,8 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
const std::string encoded = this->client_->EncodeURL(cache_key);
const std::string url = std::format(
"https://en.wikipedia.org/w/"
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format=json",
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format="
"json",
encoded);
const std::string body = this->client_->Get(url);
@@ -45,10 +47,10 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (ec) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: JSON parse error for '{}': {}",
.message = std::format(
"WikipediaService: JSON parse error for '{}': {}",
std::string(query), ec.message())});
}
return {};
@@ -58,11 +60,10 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
const json::object* obj = doc.if_object();
if (obj == nullptr) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: Expected root object for '{}'",
.message = std::format(
"WikipediaService: Expected root object for '{}'",
std::string(query))});
}
return {};
@@ -76,11 +77,10 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if ((pages_ptr == nullptr) || !pages_ptr->is_object()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: Missing query.pages for '{}'",
.message = std::format(
"WikipediaService: Missing query.pages for '{}'",
std::string(query))});
}
return {};
@@ -90,10 +90,10 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (pages.empty()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: No pages returned for '{}'",
.message = std::format(
"WikipediaService: No pages returned for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
@@ -106,11 +106,10 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if (!page_val.is_object()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: Unexpected page format for '{}'",
.message = std::format(
"WikipediaService: Unexpected page format for '{}'",
std::string(query))});
}
return {};
@@ -121,7 +120,8 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
// Handle 404/Missing status
if (page.contains("missing")) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Page '{}' does not exist",
std::string(query))});
@@ -134,11 +134,10 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
if ((extract_ptr == nullptr) || !extract_ptr->is_string()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message =
std::format("WikipediaService: No extract string found for '{}'",
.message = std::format(
"WikipediaService: No extract string found for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
@@ -148,7 +147,8 @@ std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
// 4. Success
std::string extract(extract_ptr->as_string());
if (logger_) {
logger_->Log({.level = LogLevel::Info,
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Fetched {} chars for '{}'",
extract.size(), std::string(query))});

3
tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc
View File

@@ -45,7 +45,8 @@ std::string WikipediaEnrichmentService::GetLocationContext(
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("Done fetching for {}. Sleeping for 10 seconds.",
.message = std::format(
"Done fetching for {}. Sleeping for 10 seconds.",
location_query)});
}
std::this_thread::sleep_for(10s);

13
tooling/pipeline/src/services/prompt_directory.cc
View File

@@ -44,8 +44,8 @@ PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir,
// Scenario 4: directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(
std::format("PromptDirectory: prompt directory is not readable: {} ({})",
throw std::runtime_error(std::format(
"PromptDirectory: prompt directory is not readable: {} ({})",
prompt_dir_.string(), ec.message()));
}
@@ -84,14 +84,17 @@ std::string PromptDirectory::Load(std::string_view key) {
file.close();
if (content.empty()) {
throw std::runtime_error(std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
throw std::runtime_error(
std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
key_str, file_path.string()));
}
if (logger_) {
logger_->Log({.level = LogLevel::Info,
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format("[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
.message = std::format(
"[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
key_str, file_path.string(), content.size())});
}

4
tooling/pipeline/src/services/sqlite/initialize.cc
View File

@@ -18,8 +18,8 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
std::filesystem::path candidate = output_path_ / base_filename;
for (int suffix = 1; std::filesystem::exists(candidate); ++suffix) {
candidate = output_path_ /
std::filesystem::path(std::format("biergarten_seed_{}-{}.sqlite",
candidate = output_path_ / std::filesystem::path(
std::format("biergarten_seed_{}-{}.sqlite",
run_timestamp_utc_, suffix));
}

6
tooling/pipeline/src/services/sqlite/process_record.cc
View File

@@ -38,8 +38,7 @@ sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
}
const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(
location.local_languages);
sqlite_export_service_internal::SerializeVector(location.local_languages);
sqlite_export_service_internal::Bind(
insert_location_stmt_,
@@ -92,7 +91,8 @@ sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
.action = "Failed to bind SQLite location longitude"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_location_stmt_, "Failed to insert SQLite location row");
db_handle_, insert_location_stmt_,
"Failed to insert SQLite location row");
const sqlite3_int64 location_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);

11
tooling/pipeline/src/web_client/http_web_client.cc
View File

@@ -48,18 +48,17 @@ std::string HttpWebClient::Get(const std::string& url) {
const httplib::Result result = client.Get(path);
if (!result) {
throw std::runtime_error(std::format(
"[HttpWebClient] Request failed for URL: {} — {}", url,
throw std::runtime_error(
std::format("[HttpWebClient] Request failed for URL: {} — {}", url,
httplib::to_string(result.error())));
}
if (result->status < kSuccessMin || result->status >= kSuccessMax) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Error,
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Enrichment,
.message =
std::format("[HttpWebClient] Request failed for URL: {}", url)});
.message = std::format(
"[HttpWebClient] Request failed for URL: {}", url)});
}
throw std::runtime_error(std::format("[HttpWebClient] HTTP {} for URL: {}",
result->status, url));