Pipeline: Add LLM and mocked user generation to the pipeline (#230)

This commit is contained in:
2026-07-01 07:47:24 -04:00
committed by GitHub
parent 880f73e004
commit 2b8a900d12
90 changed files with 57254 additions and 800 deletions

View File

@@ -0,0 +1,70 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
/**
* @file biergarten_pipeline.h
* @brief Master umbrella header — includes every public header in the
* Biergarten pipeline.
*/
// --- orchestrator ---
#include "biergarten_pipeline_orchestrator.h"
// --- concurrency ---
#include "concurrency/bounded_channel.h"
// --- data_generation ---
#include "data_generation/data_generator.h"
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
// --- data_model ---
#include "data_model/generated_models.h"
#include "data_model/models.h"
// --- json_handling ---
#include "json_handling/pretty_print.h"
#include "services/curated_data/curated_json_data_service.h"
// --- llama backend ---
#include "llama_backend_state.h"
// --- services: curated_data ---
#include "services/curated_data/curated_data_service.h"
#include "services/curated_data/mock_curated_data_service.h"
// --- services: database ---
#include "services/database/export_service.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
// --- services: datetime ---
#include "services/datetime/date_time_provider.h"
#include "services/datetime/timer.h"
// --- services: enrichment ---
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
// --- services: logging ---
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
// --- services: prompting ---
#include "services/prompting/prompt_directory.h"
// --- web_client ---
#include "web_client/http_web_client.h"
#include "web_client/web_client.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_

View File

@@ -1,12 +1,9 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
/**
* @file biergarten_data_generator.h
* @brief Orchestration for end-to-end brewery data generation pipeline.
*
* Intent: Coordinates location loading, enrichment, and generation phases
* to produce a complete dataset. Coordinates dependencies via composition root.
* @file biergarten_pipeline_orchestrator.h
* @brief Orchestration for end-to-end brewery and user data generation.
*/
#include <memory>
@@ -15,32 +12,36 @@
#include "data_generation/data_generator.h"
#include "data_model/generated_models.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
/**
* @brief Main data generator class for the Biergarten pipeline.
* @brief Main orchestrator for the Biergarten data generation pipeline.
*
* This class encapsulates the core logic for generating brewery data.
* It handles location loading, city enrichment, and brewery generation.
* Handles location loading, city enrichment, and brewery/user generation.
*/
class BiergartenPipelineOrchestrator {
public:
/**
* @brief Constructs the orchestrator with injected pipeline dependencies.
*
* @param context_service Provides regional context for locations.
* @param generator Implementation (Llama or Mock) for brewery/user generation.
* @param exporter Database backend for persisting generated records.
* @param application_options CLI configuration and paths.
*/
/**
* @brief Constructs the orchestrator with injected pipeline dependencies.
*
* @param logger Sink for pipeline diagnostics.
* @param context_service Provides regional context for locations.
* @param generator Implementation (Llama or Mock) for brewery/user
* generation.
* @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name
* data used to seed generation.
* @param application_options CLI configuration and paths.
*/
BiergartenPipelineOrchestrator(
std::shared_ptr<ILogger> logger,
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
const ApplicationOptions& application_options);
/**
@@ -49,38 +50,42 @@ class BiergartenPipelineOrchestrator {
* Performs the following steps:
* 1. Load curated locations from JSON
* 2. Resolve context for each city using the injected context service
* 3. Generate brewery data for sampled cities
* 3. Generate brewery and user data for sampled cities
*
* @note STRUCTURAL CONCURRENCY REQUIREMENT:
* When transitioned to a multithreaded design, this method MUST structurally
* enforce that all deployed worker threads are joined before returning (e.g.
* by using std::jthread or a structured concurrency primitive). This ensures
* workers do not attempt to log to a closed channel during application teardown.
* When transitioned to a multithreaded design, this method MUST
* structurally enforce that all deployed worker threads are joined before
* returning (e.g. by using std::jthread or a structured concurrency
* primitive). This ensures workers do not attempt to log to a closed
* channel during application teardown.
*
* @return true if successful, false if not
*/
bool Run();
private:
/// @brief Logger instance for emitting pipeline messages.
std::shared_ptr<ILogger> logger_;
/// @brief Owning context provider dependency.
std::unique_ptr<IEnrichmentService> context_service_;
/// @brief Generator dependency selected in the composition root.
/**
* @brief Generator implementation selected at the composition root (Llama
* or Mock).
*/
std::unique_ptr<DataGenerator> generator_;
/// @brief Storage backend for generated brewery records.
std::unique_ptr<IExportService> exporter_;
/**
* @brief Storage backend for generated brewery and user records.
*/
std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_;
/// @brief CLI configuration: paths, model settings, generation parameters.
ApplicationOptions application_options_;
ApplicationOptions application_options_;
/**
* @brief Load locations from JSON and sample cities.
*
* @return Vector of sampled locations capped at 50 entries.
* @return Vector of locations randomly sampled per
* PipelineOptions::location_count.
*/
std::vector<Location> QueryCitiesWithCountries();
@@ -92,11 +97,20 @@ class BiergartenPipelineOrchestrator {
void GenerateBreweries(std::span<const EnrichedCity> cities);
/**
* @brief Log the generated brewery results.
* @brief Generate users grounded in sampled names and personas for
* enriched cities.
*
* @param cities Span of enriched city data.
*/
void GenerateUsers(std::span<const EnrichedCity> cities);
/**
* @brief Log the generated brewery and user results.
*/
void LogResults() const;
/// @brief Stores generated brewery data.
std::vector<GeneratedBrewery> generated_breweries_;
std::vector<BreweryRecord> generated_breweries_;
std::vector<UserRecord> generated_users_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_

View File

@@ -9,10 +9,12 @@
/**
* @file bounded_channel.h
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous channel.
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous
* channel.
*
* Intent: Enables asynchronous inter-thread communication with backpressure.
* Models a synchronous channel where producers/consumers block on capacity limits.
* Models a synchronous channel where producers/consumers block on capacity
* limits.
*/
/**

View File

@@ -1,4 +1,4 @@
#include "bounded_channel.h"
#include "concurrency/bounded_channel.h"
template <typename T>
void BoundedChannel<T>::Send(T item) {

View File

@@ -28,12 +28,16 @@ class DataGenerator {
const std::string& region_context) = 0;
/**
* @brief Generates a user profile for a locale.
* @brief Generates a user profile grounded in a sampled name and persona.
*
* @param locale Locale hint used by generator.
* @param city Enriched city the user is associated with.
* @param persona Persona archetype grounding the generated bio.
* @param name Sampled first/last name (and gender)
* @return User generation result.
*/
virtual UserResult GenerateUser(const std::string& locale) = 0;
virtual UserResult GenerateUser(const EnrichedCity& city,
const UserPersona& persona,
const Name& name) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_DATA_GENERATOR_H_

View File

@@ -0,0 +1,52 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
/**
* @file data_generation/json_grammars.h
* @brief GBNF grammars constraining structured JSON output from
* LlamaGenerator inference calls.
*/
#include <string_view>
// GBNF grammar for structured user JSON output.
// thought-block permits the model to emit free-form reasoning before the
// JSON object (the prompts explicitly invite this); only the "{...}" tail is
// constrained to the expected shape.
inline constexpr std::string_view kUserJsonGrammar = R"json_user(
root ::= thought-block (
"{" ws
"\"username\"" ws ":" ws string ws "," ws
"\"bio\"" ws ":" ws string ws "," ws
"\"activity_weight\"" ws ":" ws number ws
"}" ws
)
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)?
)json_user";
// GBNF grammar for structured brewery JSON output (see thought-block note
// above).
inline constexpr std::string_view kBreweryJsonGrammar = R"json_brewery(
root ::= thought-block (
"{" ws
"\"name_en\"" ws ":" ws string ws "," ws
"\"description_en\"" ws ":" ws string ws "," ws
"\"name_local\"" ws ":" ws string ws "," ws
"\"description_local\"" ws ":" ws string ws
"}" ws
)
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
)json_brewery";
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_

View File

@@ -1,24 +1,23 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#include <filesystem>
/**
* @file data_generation/llama_generator.h
* @brief llama.cpp-backed implementation of DataGenerator.
*/
#include <cstdint>
#include <filesystem>
#include <memory>
#include <random>
#include <string>
#include <string_view>
#include "../services/prompting/prompt_directory.h"
#include "data_generation/data_generator.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
#include "data_model/models.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
struct llama_model;
struct llama_context;
@@ -44,14 +43,9 @@ class LlamaGenerator final : public DataGenerator {
~LlamaGenerator() override;
// disable copy constructor
LlamaGenerator(const LlamaGenerator&) = delete;
// disable copy assignment operator
LlamaGenerator& operator=(const LlamaGenerator&) = delete;
// disable move constructor
LlamaGenerator(LlamaGenerator&&) = delete;
// disable move assignment operator
LlamaGenerator& operator=(LlamaGenerator&&) = delete;
/**
@@ -65,12 +59,15 @@ class LlamaGenerator final : public DataGenerator {
const std::string& region_context) override;
/**
* @brief Generates a user profile for the provided locale.
* @brief Generates a user profile grounded in a sampled name and persona.
*
* @param locale Locale hint.
* @param city Enriched city the user is associated with.
* @param persona Persona archetype grounding the generated bio.
* @param name Sampled first/last name -- not LLM-invented.
* @return Generated user profile.
*/
UserResult GenerateUser(const std::string& locale) override;
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
const Name& name) override;
private:
static constexpr int32_t kDefaultMaxTokens = 10000;

View File

@@ -47,4 +47,18 @@ void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out);
/**
* @brief Validates and parses user JSON output.
*
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
* and `last_name` are not LLM-authored and are set separately from the
* sampled Name.
*
* @param raw Raw model output.
* @param user_out Parsed user payload.
* @return Validation error message if invalid, or std::nullopt on success.
*/
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_HELPERS_H_

View File

@@ -28,12 +28,16 @@ class MockGenerator final : public DataGenerator {
const std::string& region_context) override;
/**
* @brief Generates deterministic user data for a locale.
* @brief Generates deterministic user data grounded in a sampled name and
* persona.
*
* @param locale Locale hint.
* @param city Enriched city the user is associated with.
* @param persona Persona archetype.
* @param name Sampled first/last name, copied directly into the result.
* @return Generated user result.
*/
UserResult GenerateUser(const std::string& locale) override;
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
const Name& name) override;
private:
/**
@@ -44,12 +48,25 @@ class MockGenerator final : public DataGenerator {
*/
static size_t DeterministicHash(const Location& location);
/**
* @brief Combines city, persona, and name into a stable hash value.
*
* @param location City and country names.
* @param persona Persona archetype.
* @param name Sampled first/last name.
* @return Deterministic hash value.
*/
static size_t DeterministicHash(const Location& location,
const UserPersona& persona, const Name& name);
// Hash stride constants for deterministic distribution across fixed-size
// arrays. These coprime strides spread hash values uniformly without
// clustering, ensuring diverse output across different hash inputs.
static constexpr size_t kNounHashStride = 7;
static constexpr size_t kDescriptionHashStride = 13;
static constexpr size_t kBioHashStride = 11;
static constexpr size_t kActivityWeightHashStride = 17;
static constexpr size_t kActivityWeightHashRange = 1000;
static constexpr std::array<std::string_view, 18> kBreweryAdjectives = {
"Craft", "Heritage", "Local", "Artisan", "Pioneer", "Golden",
@@ -124,7 +141,8 @@ class MockGenerator final : public DataGenerator {
"Visits breweries for the stories, stays for the flagship pours.",
"Craft beer fan mapping tasting notes and favorite brew routes.",
"Always ready to trade recommendations for underrated local breweries.",
"Keeping a running list of must-try collab releases and tap takeovers."};
"Keeping a running list of must-try collab releases and tap "
"takeovers."};
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_MOCK_GENERATOR_H_

View File

@@ -3,8 +3,8 @@
/**
* @file data_model/generated_models.h
* @brief Generated output models from the pipeline: brewery/user results, enriched data,
* and complete generation results.
* @brief Generated output models from the pipeline: brewery/user results,
* enriched data, and complete generation results.
*/
#include <string>
@@ -19,16 +19,24 @@
* @brief Generated brewery payload.
*/
struct BreweryResult {
/// @brief Brewery display name in English.
/**
* @brief Brewery display name in English.
*/
std::string name_en;
/// @brief Brewery description text in English.
/**
* @brief Brewery description text in English.
*/
std::string description_en;
/// @brief Brewery display name in the local language.
/**
* @brief Brewery display name in the local language.
*/
std::string name_local;
/// @brief Brewery description text in the local language.
/**
* @brief Brewery description text in the local language.
*/
std::string description_local;
};
@@ -36,11 +44,39 @@ struct BreweryResult {
* @brief Generated user profile payload.
*/
struct UserResult {
/// @brief Username handle.
/**
* @brief First (given) name, copied from the sampled Name -- not
* LLM-invented.
*/
std::string first_name{};
/**
* @brief Last (family) name, copied from the sampled Name -- not
* LLM-invented.
*/
std::string last_name{};
/**
* @brief Gender associated with the sampled first name, copied from the
* sampled Name -- not LLM-invented.
*/
std::string gender{};
/**
* @brief Username handle.
*/
std::string username{};
/// @brief Short user biography.
/**
* @brief Short user biography.
*/
std::string bio{};
/**
* @brief Relative check-in/activity weight for this user, used to drive
* a J-curve activity profile in later pipeline phases.
*/
float activity_weight{};
};
// ============================================================================
@@ -58,9 +94,23 @@ struct EnrichedCity {
/**
* @brief Helper struct to store generated brewery data.
*/
struct GeneratedBrewery {
struct BreweryRecord {
Location location;
BreweryResult brewery;
};
/**
* @brief Helper struct to store generated user data.
*
* `email` and `date_of_birth` are programmatically generated by the
* orchestrator (never LLM-authored) so a downstream auth-account seeding
* consumer can register real accounts from the pipeline's SQLite export.
*/
struct UserRecord {
Location location;
UserResult user;
std::string email{};
std::string date_of_birth{};
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_GENERATED_MODELS_H_

View File

@@ -10,6 +10,7 @@
#include <boost/program_options.hpp>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <memory>
#include <optional>
#include <string>
@@ -28,40 +29,106 @@ namespace prog_opts = boost::program_options;
* @brief Canonical location record for city-level generation.
*/
struct Location {
/// @brief City name.
std::string city{};
/// @brief State or province name.
std::string state_province{};
/// @brief ISO 3166-2 subdivision code.
/**
* @brief ISO 3166-2 subdivision code.
*/
std::string iso3166_2{};
/// @brief Country name.
std::string country{};
/// @brief ISO 3166-1 country code.
/**
* @brief ISO 3166-1 country code.
*/
std::string iso3166_1{};
/// @brief Local language codes in priority order.
/**
* @brief Local language codes in priority order.
*/
std::vector<std::string> local_languages{};
/// @brief Latitude in decimal degrees.
/**
* @brief Latitude in decimal degrees.
*/
double latitude{};
/// @brief Longitude in decimal degrees.
/**
* @brief Longitude in decimal degrees.
*/
double longitude{};
};
/**
* @brief Non-owning brewery location input.
*/
struct BreweryLocation {
/// @brief City name.
std::string_view city_name;
// ============================================================================
// Name / Persona Models
// ============================================================================
/// @brief Country name.
std::string_view country_name;
/**
* @brief A sampled first/last name pair, with the source forename's gender.
*
* Produced by the SampleName() helper in generate_users.cc.
*/
struct Name {
std::string first_name{};
std::string last_name{};
/**
* @brief Gender associated with the sampled forename (e.g. "M", "F"), as
* reported by the source dataset.
*/
std::string gender{};
};
/**
* @brief A single forename entry from the names-by-country fixture data.
*/
struct ForenameEntry {
/**
* @brief Romanized forename.
*/
std::string name{};
/**
* @brief Gender associated with this forename, as reported by the source
* dataset (e.g. "M", "F").
*/
std::string gender{};
bool operator==(const ForenameEntry& other) const {
return name == other.name && gender == other.gender;
}
};
namespace std {
template <>
struct hash<ForenameEntry> {
size_t operator()(const ForenameEntry& entry) const noexcept {
const size_t name_hash = std::hash<std::string>{}(entry.name);
const size_t gender_hash = std::hash<std::string>{}(entry.gender);
return name_hash ^ (gender_hash << 1);
}
};
} // namespace std
/**
* @brief A persona archetype used to ground LLM-generated user bios.
*/
struct UserPersona {
/**
* @brief Persona display name (e.g. "Hophead Explorer").
*/
std::string name{};
/**
* @brief Short description of the persona's interests and voice.
*/
std::string description{};
/**
* @brief Beer styles this persona gravitates toward.
*/
std::vector<std::string> style_affinities{};
};
// ============================================================================
@@ -72,22 +139,34 @@ struct BreweryLocation {
* @brief LLM sampling parameters.
*/
struct SamplingOptions {
/// @brief LLM sampling temperature (0.0 to 1.0, higher = more random).
/**
* @brief LLM sampling temperature (higher = more random).
*/
float temperature = 1.0F;
/// @brief LLM nucleus sampling top-p parameter.
/**
* @brief LLM nucleus sampling top-p parameter.
*/
float top_p = 0.95F;
/// @brief LLM top-k sampling parameter.
/**
* @brief LLM top-k sampling parameter.
*/
uint32_t top_k = 64;
/// @brief Context window size (tokens).
/**
* @brief Context window size (tokens).
*/
uint32_t n_ctx = 8192;
/// @brief Random seed (-1 for random, otherwise non-negative).
/**
* @brief Random seed (-1 for random, otherwise non-negative).
*/
int seed = -1;
/// @brief Number of layers to offload to GPU.
/**
* @brief Number of layers to offload to GPU.
*/
int n_gpu_layers = 0;
};
@@ -95,16 +174,20 @@ struct SamplingOptions {
* @brief Configuration for the LLM generator component.
*/
struct GeneratorOptions {
/// @brief Path to the LLM model file (gguf format).
/**
* @brief Path to the LLM model file (gguf format).
*/
std::filesystem::path model_path;
/// @brief Use mocked generator instead of actual LLM inference.
/**
* @brief Use mocked generator instead of actual LLM inference.
*/
bool use_mocked = false;
/// @brief Specific sampling parameters for this generator.
/// If nullopt, the application should use global defaults.
/**
* @brief Specific sampling parameters for this generator.
* If nullopt, the application should use global defaults.
*/
std::optional<SamplingOptions> sampling;
};
@@ -112,18 +195,23 @@ struct GeneratorOptions {
* @brief Configuration for the pipeline execution and output.
*/
struct PipelineOptions {
/// @brief Directory for generated artifacts.
/**
* @brief Directory for generated artifacts.
*/
std::filesystem::path output_path;
/// @brief Directory that contains named prompt files (e.g.
/// BREWERY_GENERATION.md).
/**
* @brief Directory that contains named prompt files (e.g.
* BREWERY_GENERATION.md).
*/
std::filesystem::path prompt_dir;
/// @brief Path for application logs.
std::filesystem::path log_path;
/// @brief Number of locations to sample from the dataset
/// More locations -> more users/more breweries
/**
* @brief Number of locations to sample from the dataset
* More locations -> more users/more breweries
*/
uint32_t location_count;
};
@@ -139,7 +227,7 @@ struct ApplicationOptions {
// Function Declarations
// ============================================================================
std::optional<ApplicationOptions> ParseArguments(const int argc, char** argv,
std::shared_ptr<ILogger> logger = nullptr);
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger = nullptr);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_MODELS_H_

View File

@@ -1,25 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
/**
* @file json_handling/json_loader.h
* @brief Loader API for curated location data.
*/
#include <filesystem>
#include <memory>
#include <vector>
#include "data_model/models.h"
#include "services/logging/logger.h"
/// @brief Loads curated world locations from a JSON file into memory.
class JsonLoader {
public:
/// @brief Parses a JSON array file and returns all location records.
static std::vector<Location> LoadLocations(
const std::filesystem::path& filepath,
std::shared_ptr<ILogger> logger = nullptr);
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

View File

@@ -106,4 +106,4 @@ inline void PrettyPrint(std::ostream& outstream,
}
}
#endif
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_

View File

@@ -16,16 +16,10 @@
*/
class LlamaBackendState {
public:
/// @brief Initializes global llama backend state.
LlamaBackendState() { llama_backend_init(); }
/// @brief Cleans up global llama backend state.
~LlamaBackendState() { llama_backend_free(); }
/// @brief Non-copyable type.
LlamaBackendState(const LlamaBackendState&) = delete;
/// @brief Non-copyable type.
LlamaBackendState& operator=(const LlamaBackendState&) = delete;
};

View File

@@ -0,0 +1,50 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
/**
* @file services/curated_data/curated_data_service.h
* @brief Abstraction for loading curated location, persona, and name data.
*/
#include <filesystem>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "data_model/models.h"
using ForenameList = std::vector<ForenameEntry>;
using SurnameList = std::vector<std::string>;
using LocationsList = std::vector<Location>;
using PersonasList = std::vector<UserPersona>;
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;
/**
* @brief Interface for services that load curated data used to seed
* brewery/user generation.
*/
class ICuratedDataService {
public:
ICuratedDataService() = default;
virtual ~ICuratedDataService() = default;
ICuratedDataService(const ICuratedDataService&) = delete;
ICuratedDataService& operator=(const ICuratedDataService&) = delete;
ICuratedDataService(ICuratedDataService&&) = delete;
ICuratedDataService& operator=(ICuratedDataService&&) = delete;
/**
* @brief Loads all curated location records.
*/
virtual const LocationsList& LoadLocations() = 0;
/**
* @brief Loads all curated persona records.
*/
virtual const PersonasList& LoadPersonas() = 0;
virtual const ForenamesByCountryMap& LoadForenamesByCountry() = 0;
virtual const SurnamesByCountryMap& LoadSurnamesByCountry() = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_

View File

@@ -0,0 +1,63 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
/**
* @file json_handling/json_loader.h
* @brief JSON-backed implementation of ICuratedDataService.
*/
#include <filesystem>
#include "data_model/models.h"
#include "services/curated_data/curated_data_service.h"
/**
* @brief File locations for the curated JSON fixtures consumed by
* CuratedJsonDataService.
*/
struct CuratedDataFilePaths {
std::filesystem::path locations_path;
std::filesystem::path personas_path;
std::filesystem::path forenames_path;
std::filesystem::path surnames_path;
};
/**
* @brief Loads curated location, persona, and name data from JSON files.
*/
class CuratedJsonDataService final : public ICuratedDataService {
struct cache {
LocationsList locations;
PersonasList personas;
ForenamesByCountryMap forenames_by_country;
SurnamesByCountryMap surnames_by_country;
cache() = default;
~cache() = default;
};
CuratedDataFilePaths filepaths_;
cache cache_;
public:
explicit CuratedJsonDataService(CuratedDataFilePaths filepaths);
/**
* @brief Parses a JSON array file and returns all location records.
*/
const LocationsList& LoadLocations() override;
/**
* @brief Parses a JSON array file and returns all persona records.
*/
const PersonasList& LoadPersonas() override;
const ForenamesByCountryMap& LoadForenamesByCountry() override;
/**
* @brief Parses a JSON file and returns all the forenames per country.
*/
const SurnamesByCountryMap& LoadSurnamesByCountry() override;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

View File

@@ -0,0 +1,38 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
/**
* @file services/curated_data/mock_curated_data_service.h
* @brief In-memory ICuratedDataService backed by a small fixed dataset, used
* when file-backed curated data is disabled (mock mode).
*/
#include <filesystem>
#include "data_model/models.h"
#include "services/curated_data/curated_data_service.h"
/**
* @brief Curated data service returning a small fixed in-memory dataset in
* place of the JSON fixture files used by JsonLoader.
*/
class MockCuratedDataService final : public ICuratedDataService {
public:
MockCuratedDataService();
const LocationsList& LoadLocations() override;
const PersonasList& LoadPersonas() override;
const ForenamesByCountryMap& LoadForenamesByCountry() override;
const SurnamesByCountryMap& LoadSurnamesByCountry() override;
private:
LocationsList locations_;
PersonasList personas_;
ForenamesByCountryMap forenames_by_country_;
SurnamesByCountryMap surnames_by_country_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_

View File

@@ -16,8 +16,6 @@
class IExportService {
public:
IExportService() = default;
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~IExportService() = default;
IExportService(const IExportService&) = delete;
@@ -25,7 +23,9 @@ class IExportService {
IExportService(IExportService&&) = delete;
IExportService& operator=(IExportService&&) = delete;
/// @brief Prepares the export destination for a new run.
/**
* @brief Prepares the export destination for a new run.
*/
virtual void Initialize() = 0;
/**
@@ -33,9 +33,18 @@ class IExportService {
*
* @param brewery Generated brewery payload to store.
*/
virtual uint64_t ProcessRecord(const GeneratedBrewery& brewery) = 0;
virtual uint64_t ProcessRecord(const BreweryRecord& brewery) = 0;
/// @brief Finalizes the export destination.
/**
* @brief Persists one generated user record.
*
* @param user Generated user payload to store.
*/
virtual uint64_t ProcessRecord(const UserRecord& user) = 0;
/**
* @brief Finalizes the export destination.
*/
virtual void Finalize() = 0;
};

View File

@@ -12,7 +12,7 @@
#include <string>
#include <string_view>
#include "sqlite_handle_types.h"
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal {

View File

@@ -12,9 +12,9 @@
#include <unordered_map>
#include "data_model/models.h"
#include "../datetime/date_time_provider.h"
#include "export_service.h"
#include "sqlite_export_service_helpers.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/datetime/date_time_provider.h"
/**
* @brief Persists generated brewery records into a fresh SQLite database.
@@ -30,7 +30,8 @@ class SqliteExportService final : public IExportService {
SqliteExportService& operator=(SqliteExportService&&) = delete;
void Initialize() override;
uint64_t ProcessRecord(const GeneratedBrewery& brewery) override;
uint64_t ProcessRecord(const BreweryRecord& brewery) override;
uint64_t ProcessRecord(const UserRecord& user) override;
void Finalize() override;
private:
@@ -46,6 +47,15 @@ class SqliteExportService final : public IExportService {
[[nodiscard]] std::filesystem::path BuildDatabasePath() const;
[[nodiscard]] static std::string BuildLocationKey(const Location& location);
/**
* @brief Returns the row id for @p location, inserting it first if it has
* not already been seen during this run.
*
* Shared by both ProcessRecord() overloads so breweries and users
* referencing the same location resolve to the same row.
*/
[[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location);
std::unique_ptr<IDateTimeProvider> date_time_provider_;
std::filesystem::path output_path_;
std::string run_timestamp_utc_;
@@ -53,6 +63,7 @@ class SqliteExportService final : public IExportService {
SqliteDatabaseHandle db_handle_;
SqliteStatementHandle insert_location_stmt_;
SqliteStatementHandle insert_brewery_stmt_;
SqliteStatementHandle insert_user_stmt_;
bool transaction_open_ = false;
std::unordered_map<std::string, sqlite3_int64> location_cache_;
};

View File

@@ -3,8 +3,8 @@
/* Umbrella header for backward compatibility. */
#include "sqlite_connection_helpers.h"
#include "sqlite_handle_types.h"
#include "sqlite_statement_helpers.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_

View File

@@ -24,8 +24,10 @@ using SqliteDatabaseHandle = std::unique_ptr<sqlite3, SqliteDatabaseDeleter>;
using SqliteStatementHandle =
std::unique_ptr<sqlite3_stmt, SqliteStatementDeleter>;
// Represents a parameter that is bound to a prepared SQLite statement.
// N.B. indices are 1 based.
template <typename T>
struct BindParam {
struct BoundParam {
int index;
T value;
std::string_view action;

View File

@@ -13,7 +13,7 @@
#include <string_view>
#include <vector>
#include "sqlite_handle_types.h"
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal {
@@ -50,6 +50,26 @@ CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id);
)sql";
inline constexpr std::string_view kCreateUsersTableSql = R"sql(
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
location_id INTEGER NOT NULL,
first_name TEXT NOT NULL,
last_name TEXT NOT NULL,
gender TEXT NOT NULL,
username TEXT NOT NULL,
bio TEXT NOT NULL,
activity_weight REAL NOT NULL,
email TEXT NOT NULL UNIQUE,
date_of_birth TEXT NOT NULL,
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id);
)sql";
inline constexpr std::string_view kInsertLocationSql = R"sql(
INSERT INTO locations (
city,
@@ -73,20 +93,52 @@ INSERT INTO breweries (
) VALUES (?, ?, ?, ?, ?);
)sql";
inline constexpr int kLocationCityBindIndex = 1;
inline constexpr int kLocationStateProvinceBindIndex = 2;
inline constexpr int kLocationIso31662BindIndex = 3;
inline constexpr int kLocationCountryBindIndex = 4;
inline constexpr int kLocationIso31661BindIndex = 5;
inline constexpr int kLocationLanguagesBindIndex = 6;
inline constexpr int kLocationLatitudeBindIndex = 7;
inline constexpr int kLocationLongitudeBindIndex = 8;
inline constexpr std::string_view kInsertUserSql = R"sql(
INSERT INTO users (
location_id,
first_name,
last_name,
gender,
username,
bio,
activity_weight,
email,
date_of_birth
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr int kBreweryLocationIdBindIndex = 1;
inline constexpr int kBreweryEnglishNameBindIndex = 2;
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3;
inline constexpr int kBreweryLocalNameBindIndex = 4;
inline constexpr int kBreweryLocalDescriptionBindIndex = 5;
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above.
enum LocationBindIndex {
kLocationCityBindIndex = 1,
kLocationStateProvinceBindIndex,
kLocationIso31662BindIndex,
kLocationCountryBindIndex,
kLocationIso31661BindIndex,
kLocationLanguagesBindIndex,
kLocationLatitudeBindIndex,
kLocationLongitudeBindIndex,
};
enum BreweryBindIndex {
kBreweryLocationIdBindIndex = 1,
kBreweryEnglishNameBindIndex,
kBreweryEnglishDescriptionBindIndex,
kBreweryLocalNameBindIndex,
kBreweryLocalDescriptionBindIndex,
};
enum UserBindIndex {
kUserLocationIdBindIndex = 1,
kUserFirstNameBindIndex,
kUserLastNameBindIndex,
kUserGenderBindIndex,
kUserUsernameBindIndex,
kUserBioBindIndex,
kUserActivityWeightBindIndex,
kUserEmailBindIndex,
kUserDateOfBirthBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql,
@@ -95,13 +147,13 @@ SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
void ResetStatement(SqliteStatementHandle& statement);
void Bind(const SqliteStatementHandle& statement,
const BindParam<std::string_view>& param);
const BoundParam<std::string_view>& param);
void Bind(const SqliteStatementHandle& statement,
const BindParam<double>& param);
const BoundParam<double>& param);
void Bind(const SqliteStatementHandle& statement,
const BindParam<sqlite3_int64>& param);
const BoundParam<sqlite3_int64>& param);
void StepStatement(const SqliteDatabaseHandle& db_handle,
const SqliteStatementHandle& statement,

View File

@@ -18,7 +18,6 @@
*/
class IDateTimeProvider {
public:
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~IDateTimeProvider() = default;
IDateTimeProvider() = default;

View File

@@ -15,7 +15,6 @@
*/
class IEnrichmentService {
public:
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~IEnrichmentService() = default;
/**

View File

@@ -1,13 +1,18 @@
//
// Created by aaronpo on 13/05/2026.
//
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
/**
* @file services/enrichment/mock_enrichment.h
* @brief No-op IEnrichmentService used when network enrichment is disabled.
*/
#include <string>
#include "enrichment_service.h"
#include "services/enrichment/enrichment_service.h"
/**
* @brief Enrichment service that returns no context for any location.
*/
class MockEnrichmentService final : public IEnrichmentService {
public:
std::string GetLocationContext(const Location& /*loc*/) override {

View File

@@ -11,25 +11,37 @@
#include <string_view>
#include <unordered_map>
#include "enrichment_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
#include "web_client/web_client.h"
/// @brief Provides Wikipedia summary lookups backed by cached raw extracts.
/**
* @brief Provides Wikipedia summary lookups backed by cached raw extracts.
*/
class WikipediaEnrichmentService final : public IEnrichmentService {
public:
/// @brief Creates a new Wikipedia service with the provided web client.
/**
* @brief Creates a new Wikipedia service with the provided web client.
*/
explicit WikipediaEnrichmentService(std::unique_ptr<WebClient> client,
std::shared_ptr<ILogger> logger);
/// @brief Returns the Wikipedia-derived context for a location.
/**
* @brief Returns the Wikipedia-derived context for a location.
*/
[[nodiscard]] std::string GetLocationContext(const Location& loc) override;
private:
std::string FetchExtract(std::string_view query);
std::unique_ptr<WebClient> client_;
std::shared_ptr<ILogger> logger_;
/// @brief Canonical cache for raw Wikipedia query extracts.
/**
* @brief Cache for raw Wikipedia query extracts, keyed by query string.
*
* GetLocationContext() always queries "brewing" and reuses "beer in
* {country}" for every city in that country, so caching avoids refetching
* the same extract across locations in a run.
*/
std::unordered_map<std::string, std::string> extract_cache_;
};

View File

@@ -20,10 +20,22 @@
* @brief Severity levels supported by the logging infra.
*/
enum class LogLevel {
Debug, ///< Development/debugging information.
Info, ///< General informational messages.
Warn, ///< Warning conditions.
Error, ///< Error conditions.
/**
* @brief Development/debugging information.
*/
Debug,
/**
* @brief General informational messages.
*/
Info,
/**
* @brief Warning conditions.
*/
Warn,
/**
* @brief Error conditions.
*/
Error,
};
/**
@@ -34,18 +46,44 @@ enum class LogLevel {
* pipeline that emitted it.
*/
enum class PipelinePhase {
Startup, ///< Initialization and validation.
UserGeneration, ///< User profile generation.
BreweryAndBeerGeneration, ///< Brewery and beer data generation.
CheckinGeneration, ///< Checkin (visit) record generation.
RatingGeneration, ///< Rating and review generation.
FollowGeneration, ///< Follow relationship generation.
Teardown, ///< Finalization and cleanup.
/**
* @brief Initialization and validation.
*/
Startup,
/**
* @brief Location/context enrichment (e.g. Wikipedia lookups).
*/
Enrichment,
/**
* @brief User profile generation.
*/
UserGeneration,
/**
* @brief Brewery and beer data generation.
*/
BreweryAndBeerGeneration,
/**
* @brief Checkin (visit) record generation.
*/
CheckinGeneration,
/**
* @brief Rating and review generation.
*/
RatingGeneration,
/**
* @brief Follow relationship generation.
*/
FollowGeneration,
/**
* @brief Finalization and cleanup.
*/
Teardown,
};
/**
* @struct LogDTO
* @brief User-provided subset of log fields. Used to capture call-site info transparently.
* @brief User-provided subset of log fields. Used to capture call-site info
* transparently.
*/
struct LogDTO {
LogLevel level;
@@ -64,25 +102,35 @@ struct LogDTO {
* before the entry is dispatched.
*/
struct LogEntry {
/// @brief Timestamp when the entry was created.
/**
* @brief Timestamp when the entry was created.
*/
std::chrono::system_clock::time_point timestamp{};
/// @brief Source location where the log call was made.
/**
* @brief Source location where the log call was made.
*/
std::source_location origin{};
/// @brief Thread responsible for emitting the log.
/**
* @brief Thread responsible for emitting the log.
*/
std::thread::id thread_id{};
/// @brief Severity level of this entry.
/**
* @brief Severity level of this entry.
*/
LogLevel level;
/// @brief Pipeline phase associated with the entry.
/**
* @brief Pipeline phase associated with the entry.
*/
PipelinePhase phase;
/// @brief Log message text.
/**
* @brief Log message text.
*/
std::string message;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_

View File

@@ -6,8 +6,8 @@
* a bounded channel for later processing by the dispatcher.
*/
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#include <string_view>
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
BoundedChannel<LogEntry>& channel_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_

View File

@@ -39,7 +39,8 @@ class ILogger {
*/
void Log(LogDTO payload,
std::source_location origin = std::source_location::current(),
std::chrono::system_clock::time_point timestamp = std::chrono::system_clock::now(),
std::chrono::system_clock::time_point timestamp =
std::chrono::system_clock::now(),
std::thread::id thread_id = std::this_thread::get_id()) {
LogEntry entry;
entry.timestamp = timestamp;

View File

@@ -1,19 +1,18 @@
/**
* @file web_client/http_web_client.h
* @brief cpp-httplib implementation of the WebClient interface.
*/
* @file web_client/http_web_client.h
* @brief cpp-httplib implementation of the WebClient interface.
*/
#ifndef BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
#define BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
#include "web_client/web_client.h"
#include "services/logging/logger.h"
#include <memory>
#include <string>
#include <utility>
#include "services/logging/logger.h"
#include "web_client/web_client.h"
/**
* @brief WebClient implementation backed by cpp-httplib.
*
@@ -26,7 +25,7 @@
* bound to a single origin at construction time.
*/
class HttpWebClient final : public WebClient {
public:
public:
explicit HttpWebClient(std::shared_ptr<ILogger> logger)
: logger_(std::move(logger)) {}
~HttpWebClient() override = default;
@@ -34,7 +33,8 @@ public:
/**
* @brief Executes a blocking HTTP/HTTPS GET request against a full URL.
*
* @param url Fully-qualified URL, e.g. "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
* @param url Fully-qualified URL, e.g.
* "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
* @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
*/
std::string Get(const std::string& url) override;
@@ -52,5 +52,4 @@ public:
std::shared_ptr<ILogger> logger_;
};
#endif
#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_

View File

@@ -13,7 +13,6 @@
*/
class WebClient {
public:
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~WebClient() = default;
/**