mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-07-16 17:47:22 +00:00
Pipeline: Add LLM and mocked user generation to the pipeline (#230)
This commit is contained in:
70
tooling/pipeline/includes/biergarten_pipeline.h
Normal file
70
tooling/pipeline/includes/biergarten_pipeline.h
Normal file
@@ -0,0 +1,70 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||
|
||||
/**
|
||||
* @file biergarten_pipeline.h
|
||||
* @brief Master umbrella header — includes every public header in the
|
||||
* Biergarten pipeline.
|
||||
*/
|
||||
|
||||
// --- orchestrator ---
|
||||
#include "biergarten_pipeline_orchestrator.h"
|
||||
|
||||
// --- concurrency ---
|
||||
#include "concurrency/bounded_channel.h"
|
||||
|
||||
// --- data_generation ---
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_generation/json_grammars.h"
|
||||
#include "data_generation/llama_generator.h"
|
||||
#include "data_generation/llama_generator_helpers.h"
|
||||
#include "data_generation/mock_generator.h"
|
||||
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
||||
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
||||
|
||||
// --- data_model ---
|
||||
#include "data_model/generated_models.h"
|
||||
#include "data_model/models.h"
|
||||
|
||||
// --- json_handling ---
|
||||
#include "json_handling/pretty_print.h"
|
||||
#include "services/curated_data/curated_json_data_service.h"
|
||||
|
||||
// --- llama backend ---
|
||||
#include "llama_backend_state.h"
|
||||
|
||||
// --- services: curated_data ---
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
#include "services/curated_data/mock_curated_data_service.h"
|
||||
|
||||
// --- services: database ---
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/database/sqlite_connection_helpers.h"
|
||||
#include "services/database/sqlite_export_service.h"
|
||||
#include "services/database/sqlite_export_service_helpers.h"
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
#include "services/database/sqlite_statement_helpers.h"
|
||||
|
||||
// --- services: datetime ---
|
||||
#include "services/datetime/date_time_provider.h"
|
||||
#include "services/datetime/timer.h"
|
||||
|
||||
// --- services: enrichment ---
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
#include "services/enrichment/mock_enrichment.h"
|
||||
#include "services/enrichment/wikipedia_service.h"
|
||||
|
||||
// --- services: logging ---
|
||||
#include "services/logging/log_dispatcher.h"
|
||||
#include "services/logging/log_entry.h"
|
||||
#include "services/logging/log_producer.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
// --- services: prompting ---
|
||||
#include "services/prompting/prompt_directory.h"
|
||||
|
||||
// --- web_client ---
|
||||
#include "web_client/http_web_client.h"
|
||||
#include "web_client/web_client.h"
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
|
||||
@@ -1,12 +1,9 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||
|
||||
/**
|
||||
* @file biergarten_data_generator.h
|
||||
* @brief Orchestration for end-to-end brewery data generation pipeline.
|
||||
*
|
||||
* Intent: Coordinates location loading, enrichment, and generation phases
|
||||
* to produce a complete dataset. Coordinates dependencies via composition root.
|
||||
* @file biergarten_pipeline_orchestrator.h
|
||||
* @brief Orchestration for end-to-end brewery and user data generation.
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
@@ -15,32 +12,36 @@
|
||||
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_model/generated_models.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
/**
|
||||
* @brief Main data generator class for the Biergarten pipeline.
|
||||
* @brief Main orchestrator for the Biergarten data generation pipeline.
|
||||
*
|
||||
* This class encapsulates the core logic for generating brewery data.
|
||||
* It handles location loading, city enrichment, and brewery generation.
|
||||
* Handles location loading, city enrichment, and brewery/user generation.
|
||||
*/
|
||||
class BiergartenPipelineOrchestrator {
|
||||
public:
|
||||
/**
|
||||
* @brief Constructs the orchestrator with injected pipeline dependencies.
|
||||
*
|
||||
* @param context_service Provides regional context for locations.
|
||||
* @param generator Implementation (Llama or Mock) for brewery/user generation.
|
||||
* @param exporter Database backend for persisting generated records.
|
||||
* @param application_options CLI configuration and paths.
|
||||
*/
|
||||
/**
|
||||
* @brief Constructs the orchestrator with injected pipeline dependencies.
|
||||
*
|
||||
* @param logger Sink for pipeline diagnostics.
|
||||
* @param context_service Provides regional context for locations.
|
||||
* @param generator Implementation (Llama or Mock) for brewery/user
|
||||
* generation.
|
||||
* @param exporter Database backend for persisting generated records.
|
||||
* @param curated_data_service Loads curated location, persona, and name
|
||||
* data used to seed generation.
|
||||
* @param application_options CLI configuration and paths.
|
||||
*/
|
||||
BiergartenPipelineOrchestrator(
|
||||
std::shared_ptr<ILogger> logger,
|
||||
std::unique_ptr<IEnrichmentService> context_service,
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||
const ApplicationOptions& application_options);
|
||||
|
||||
/**
|
||||
@@ -49,38 +50,42 @@ class BiergartenPipelineOrchestrator {
|
||||
* Performs the following steps:
|
||||
* 1. Load curated locations from JSON
|
||||
* 2. Resolve context for each city using the injected context service
|
||||
* 3. Generate brewery data for sampled cities
|
||||
* 3. Generate brewery and user data for sampled cities
|
||||
*
|
||||
* @note STRUCTURAL CONCURRENCY REQUIREMENT:
|
||||
* When transitioned to a multithreaded design, this method MUST structurally
|
||||
* enforce that all deployed worker threads are joined before returning (e.g.
|
||||
* by using std::jthread or a structured concurrency primitive). This ensures
|
||||
* workers do not attempt to log to a closed channel during application teardown.
|
||||
* When transitioned to a multithreaded design, this method MUST
|
||||
* structurally enforce that all deployed worker threads are joined before
|
||||
* returning (e.g. by using std::jthread or a structured concurrency
|
||||
* primitive). This ensures workers do not attempt to log to a closed
|
||||
* channel during application teardown.
|
||||
*
|
||||
* @return true if successful, false if not
|
||||
*/
|
||||
bool Run();
|
||||
|
||||
private:
|
||||
/// @brief Logger instance for emitting pipeline messages.
|
||||
std::shared_ptr<ILogger> logger_;
|
||||
|
||||
/// @brief Owning context provider dependency.
|
||||
std::unique_ptr<IEnrichmentService> context_service_;
|
||||
|
||||
/// @brief Generator dependency selected in the composition root.
|
||||
/**
|
||||
* @brief Generator implementation selected at the composition root (Llama
|
||||
* or Mock).
|
||||
*/
|
||||
std::unique_ptr<DataGenerator> generator_;
|
||||
|
||||
/// @brief Storage backend for generated brewery records.
|
||||
std::unique_ptr<IExportService> exporter_;
|
||||
/**
|
||||
* @brief Storage backend for generated brewery and user records.
|
||||
*/
|
||||
std::unique_ptr<IExportService> exporter_;
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service_;
|
||||
|
||||
/// @brief CLI configuration: paths, model settings, generation parameters.
|
||||
ApplicationOptions application_options_;
|
||||
ApplicationOptions application_options_;
|
||||
|
||||
/**
|
||||
* @brief Load locations from JSON and sample cities.
|
||||
*
|
||||
* @return Vector of sampled locations capped at 50 entries.
|
||||
* @return Vector of locations randomly sampled per
|
||||
* PipelineOptions::location_count.
|
||||
*/
|
||||
std::vector<Location> QueryCitiesWithCountries();
|
||||
|
||||
@@ -92,11 +97,20 @@ class BiergartenPipelineOrchestrator {
|
||||
void GenerateBreweries(std::span<const EnrichedCity> cities);
|
||||
|
||||
/**
|
||||
* @brief Log the generated brewery results.
|
||||
* @brief Generate users grounded in sampled names and personas for
|
||||
* enriched cities.
|
||||
*
|
||||
* @param cities Span of enriched city data.
|
||||
*/
|
||||
void GenerateUsers(std::span<const EnrichedCity> cities);
|
||||
|
||||
/**
|
||||
* @brief Log the generated brewery and user results.
|
||||
*/
|
||||
void LogResults() const;
|
||||
|
||||
/// @brief Stores generated brewery data.
|
||||
std::vector<GeneratedBrewery> generated_breweries_;
|
||||
std::vector<BreweryRecord> generated_breweries_;
|
||||
std::vector<UserRecord> generated_users_;
|
||||
};
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
|
||||
|
||||
@@ -9,10 +9,12 @@
|
||||
|
||||
/**
|
||||
* @file bounded_channel.h
|
||||
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous channel.
|
||||
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous
|
||||
* channel.
|
||||
*
|
||||
* Intent: Enables asynchronous inter-thread communication with backpressure.
|
||||
* Models a synchronous channel where producers/consumers block on capacity limits.
|
||||
* Models a synchronous channel where producers/consumers block on capacity
|
||||
* limits.
|
||||
*/
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#include "bounded_channel.h"
|
||||
#include "concurrency/bounded_channel.h"
|
||||
|
||||
template <typename T>
|
||||
void BoundedChannel<T>::Send(T item) {
|
||||
|
||||
@@ -28,12 +28,16 @@ class DataGenerator {
|
||||
const std::string& region_context) = 0;
|
||||
|
||||
/**
|
||||
* @brief Generates a user profile for a locale.
|
||||
* @brief Generates a user profile grounded in a sampled name and persona.
|
||||
*
|
||||
* @param locale Locale hint used by generator.
|
||||
* @param city Enriched city the user is associated with.
|
||||
* @param persona Persona archetype grounding the generated bio.
|
||||
* @param name Sampled first/last name (and gender)
|
||||
* @return User generation result.
|
||||
*/
|
||||
virtual UserResult GenerateUser(const std::string& locale) = 0;
|
||||
virtual UserResult GenerateUser(const EnrichedCity& city,
|
||||
const UserPersona& persona,
|
||||
const Name& name) = 0;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_DATA_GENERATOR_H_
|
||||
|
||||
52
tooling/pipeline/includes/data_generation/json_grammars.h
Normal file
52
tooling/pipeline/includes/data_generation/json_grammars.h
Normal file
@@ -0,0 +1,52 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
|
||||
|
||||
/**
|
||||
* @file data_generation/json_grammars.h
|
||||
* @brief GBNF grammars constraining structured JSON output from
|
||||
* LlamaGenerator inference calls.
|
||||
*/
|
||||
|
||||
#include <string_view>
|
||||
|
||||
// GBNF grammar for structured user JSON output.
|
||||
// thought-block permits the model to emit free-form reasoning before the
|
||||
// JSON object (the prompts explicitly invite this); only the "{...}" tail is
|
||||
// constrained to the expected shape.
|
||||
inline constexpr std::string_view kUserJsonGrammar = R"json_user(
|
||||
root ::= thought-block (
|
||||
"{" ws
|
||||
"\"username\"" ws ":" ws string ws "," ws
|
||||
"\"bio\"" ws ":" ws string ws "," ws
|
||||
"\"activity_weight\"" ws ":" ws number ws
|
||||
"}" ws
|
||||
)
|
||||
thought-block ::= [^{]*
|
||||
ws ::= [ \t\n\r]*
|
||||
string ::= "\"" char+ "\""
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
|
||||
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
|
||||
hex ::= [0-9a-fA-F]
|
||||
number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)?
|
||||
)json_user";
|
||||
|
||||
// GBNF grammar for structured brewery JSON output (see thought-block note
|
||||
// above).
|
||||
inline constexpr std::string_view kBreweryJsonGrammar = R"json_brewery(
|
||||
root ::= thought-block (
|
||||
"{" ws
|
||||
"\"name_en\"" ws ":" ws string ws "," ws
|
||||
"\"description_en\"" ws ":" ws string ws "," ws
|
||||
"\"name_local\"" ws ":" ws string ws "," ws
|
||||
"\"description_local\"" ws ":" ws string ws
|
||||
"}" ws
|
||||
)
|
||||
thought-block ::= [^{]*
|
||||
ws ::= [ \t\n\r]*
|
||||
string ::= "\"" char+ "\""
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
|
||||
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
|
||||
hex ::= [0-9a-fA-F]
|
||||
)json_brewery";
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
|
||||
@@ -1,24 +1,23 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
/**
|
||||
* @file data_generation/llama_generator.h
|
||||
* @brief llama.cpp-backed implementation of DataGenerator.
|
||||
*/
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "../services/prompting/prompt_directory.h"
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_generation/prompt_formatting/prompt_formatter.h"
|
||||
#include "data_model/models.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "services/prompting/prompt_directory.h"
|
||||
|
||||
struct llama_model;
|
||||
struct llama_context;
|
||||
@@ -44,14 +43,9 @@ class LlamaGenerator final : public DataGenerator {
|
||||
|
||||
~LlamaGenerator() override;
|
||||
|
||||
// disable copy constructor
|
||||
LlamaGenerator(const LlamaGenerator&) = delete;
|
||||
|
||||
// disable copy assignment operator
|
||||
LlamaGenerator& operator=(const LlamaGenerator&) = delete;
|
||||
// disable move constructor
|
||||
LlamaGenerator(LlamaGenerator&&) = delete;
|
||||
// disable move assignment operator
|
||||
LlamaGenerator& operator=(LlamaGenerator&&) = delete;
|
||||
|
||||
/**
|
||||
@@ -65,12 +59,15 @@ class LlamaGenerator final : public DataGenerator {
|
||||
const std::string& region_context) override;
|
||||
|
||||
/**
|
||||
* @brief Generates a user profile for the provided locale.
|
||||
* @brief Generates a user profile grounded in a sampled name and persona.
|
||||
*
|
||||
* @param locale Locale hint.
|
||||
* @param city Enriched city the user is associated with.
|
||||
* @param persona Persona archetype grounding the generated bio.
|
||||
* @param name Sampled first/last name -- not LLM-invented.
|
||||
* @return Generated user profile.
|
||||
*/
|
||||
UserResult GenerateUser(const std::string& locale) override;
|
||||
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
|
||||
const Name& name) override;
|
||||
|
||||
private:
|
||||
static constexpr int32_t kDefaultMaxTokens = 10000;
|
||||
|
||||
@@ -47,4 +47,18 @@ void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
|
||||
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
|
||||
BreweryResult& brewery_out);
|
||||
|
||||
/**
|
||||
* @brief Validates and parses user JSON output.
|
||||
*
|
||||
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
|
||||
* and `last_name` are not LLM-authored and are set separately from the
|
||||
* sampled Name.
|
||||
*
|
||||
* @param raw Raw model output.
|
||||
* @param user_out Parsed user payload.
|
||||
* @return Validation error message if invalid, or std::nullopt on success.
|
||||
*/
|
||||
std::optional<std::string> ValidateUserJson(const std::string& raw,
|
||||
UserResult& user_out);
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_HELPERS_H_
|
||||
|
||||
@@ -28,12 +28,16 @@ class MockGenerator final : public DataGenerator {
|
||||
const std::string& region_context) override;
|
||||
|
||||
/**
|
||||
* @brief Generates deterministic user data for a locale.
|
||||
* @brief Generates deterministic user data grounded in a sampled name and
|
||||
* persona.
|
||||
*
|
||||
* @param locale Locale hint.
|
||||
* @param city Enriched city the user is associated with.
|
||||
* @param persona Persona archetype.
|
||||
* @param name Sampled first/last name, copied directly into the result.
|
||||
* @return Generated user result.
|
||||
*/
|
||||
UserResult GenerateUser(const std::string& locale) override;
|
||||
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
|
||||
const Name& name) override;
|
||||
|
||||
private:
|
||||
/**
|
||||
@@ -44,12 +48,25 @@ class MockGenerator final : public DataGenerator {
|
||||
*/
|
||||
static size_t DeterministicHash(const Location& location);
|
||||
|
||||
/**
|
||||
* @brief Combines city, persona, and name into a stable hash value.
|
||||
*
|
||||
* @param location City and country names.
|
||||
* @param persona Persona archetype.
|
||||
* @param name Sampled first/last name.
|
||||
* @return Deterministic hash value.
|
||||
*/
|
||||
static size_t DeterministicHash(const Location& location,
|
||||
const UserPersona& persona, const Name& name);
|
||||
|
||||
// Hash stride constants for deterministic distribution across fixed-size
|
||||
// arrays. These coprime strides spread hash values uniformly without
|
||||
// clustering, ensuring diverse output across different hash inputs.
|
||||
static constexpr size_t kNounHashStride = 7;
|
||||
static constexpr size_t kDescriptionHashStride = 13;
|
||||
static constexpr size_t kBioHashStride = 11;
|
||||
static constexpr size_t kActivityWeightHashStride = 17;
|
||||
static constexpr size_t kActivityWeightHashRange = 1000;
|
||||
|
||||
static constexpr std::array<std::string_view, 18> kBreweryAdjectives = {
|
||||
"Craft", "Heritage", "Local", "Artisan", "Pioneer", "Golden",
|
||||
@@ -124,7 +141,8 @@ class MockGenerator final : public DataGenerator {
|
||||
"Visits breweries for the stories, stays for the flagship pours.",
|
||||
"Craft beer fan mapping tasting notes and favorite brew routes.",
|
||||
"Always ready to trade recommendations for underrated local breweries.",
|
||||
"Keeping a running list of must-try collab releases and tap takeovers."};
|
||||
"Keeping a running list of must-try collab releases and tap "
|
||||
"takeovers."};
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_MOCK_GENERATOR_H_
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
/**
|
||||
* @file data_model/generated_models.h
|
||||
* @brief Generated output models from the pipeline: brewery/user results, enriched data,
|
||||
* and complete generation results.
|
||||
* @brief Generated output models from the pipeline: brewery/user results,
|
||||
* enriched data, and complete generation results.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
@@ -19,16 +19,24 @@
|
||||
* @brief Generated brewery payload.
|
||||
*/
|
||||
struct BreweryResult {
|
||||
/// @brief Brewery display name in English.
|
||||
/**
|
||||
* @brief Brewery display name in English.
|
||||
*/
|
||||
std::string name_en;
|
||||
|
||||
/// @brief Brewery description text in English.
|
||||
/**
|
||||
* @brief Brewery description text in English.
|
||||
*/
|
||||
std::string description_en;
|
||||
|
||||
/// @brief Brewery display name in the local language.
|
||||
/**
|
||||
* @brief Brewery display name in the local language.
|
||||
*/
|
||||
std::string name_local;
|
||||
|
||||
/// @brief Brewery description text in the local language.
|
||||
/**
|
||||
* @brief Brewery description text in the local language.
|
||||
*/
|
||||
std::string description_local;
|
||||
};
|
||||
|
||||
@@ -36,11 +44,39 @@ struct BreweryResult {
|
||||
* @brief Generated user profile payload.
|
||||
*/
|
||||
struct UserResult {
|
||||
/// @brief Username handle.
|
||||
/**
|
||||
* @brief First (given) name, copied from the sampled Name -- not
|
||||
* LLM-invented.
|
||||
*/
|
||||
std::string first_name{};
|
||||
|
||||
/**
|
||||
* @brief Last (family) name, copied from the sampled Name -- not
|
||||
* LLM-invented.
|
||||
*/
|
||||
std::string last_name{};
|
||||
|
||||
/**
|
||||
* @brief Gender associated with the sampled first name, copied from the
|
||||
* sampled Name -- not LLM-invented.
|
||||
*/
|
||||
std::string gender{};
|
||||
|
||||
/**
|
||||
* @brief Username handle.
|
||||
*/
|
||||
std::string username{};
|
||||
|
||||
/// @brief Short user biography.
|
||||
/**
|
||||
* @brief Short user biography.
|
||||
*/
|
||||
std::string bio{};
|
||||
|
||||
/**
|
||||
* @brief Relative check-in/activity weight for this user, used to drive
|
||||
* a J-curve activity profile in later pipeline phases.
|
||||
*/
|
||||
float activity_weight{};
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
@@ -58,9 +94,23 @@ struct EnrichedCity {
|
||||
/**
|
||||
* @brief Helper struct to store generated brewery data.
|
||||
*/
|
||||
struct GeneratedBrewery {
|
||||
struct BreweryRecord {
|
||||
Location location;
|
||||
BreweryResult brewery;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Helper struct to store generated user data.
|
||||
*
|
||||
* `email` and `date_of_birth` are programmatically generated by the
|
||||
* orchestrator (never LLM-authored) so a downstream auth-account seeding
|
||||
* consumer can register real accounts from the pipeline's SQLite export.
|
||||
*/
|
||||
struct UserRecord {
|
||||
Location location;
|
||||
UserResult user;
|
||||
std::string email{};
|
||||
std::string date_of_birth{};
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_GENERATED_MODELS_H_
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <boost/program_options.hpp>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
@@ -28,40 +29,106 @@ namespace prog_opts = boost::program_options;
|
||||
* @brief Canonical location record for city-level generation.
|
||||
*/
|
||||
struct Location {
|
||||
/// @brief City name.
|
||||
std::string city{};
|
||||
|
||||
/// @brief State or province name.
|
||||
std::string state_province{};
|
||||
|
||||
/// @brief ISO 3166-2 subdivision code.
|
||||
/**
|
||||
* @brief ISO 3166-2 subdivision code.
|
||||
*/
|
||||
std::string iso3166_2{};
|
||||
|
||||
/// @brief Country name.
|
||||
std::string country{};
|
||||
|
||||
/// @brief ISO 3166-1 country code.
|
||||
/**
|
||||
* @brief ISO 3166-1 country code.
|
||||
*/
|
||||
std::string iso3166_1{};
|
||||
|
||||
/// @brief Local language codes in priority order.
|
||||
/**
|
||||
* @brief Local language codes in priority order.
|
||||
*/
|
||||
std::vector<std::string> local_languages{};
|
||||
|
||||
/// @brief Latitude in decimal degrees.
|
||||
/**
|
||||
* @brief Latitude in decimal degrees.
|
||||
*/
|
||||
double latitude{};
|
||||
|
||||
/// @brief Longitude in decimal degrees.
|
||||
/**
|
||||
* @brief Longitude in decimal degrees.
|
||||
*/
|
||||
double longitude{};
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Non-owning brewery location input.
|
||||
*/
|
||||
struct BreweryLocation {
|
||||
/// @brief City name.
|
||||
std::string_view city_name;
|
||||
// ============================================================================
|
||||
// Name / Persona Models
|
||||
// ============================================================================
|
||||
|
||||
/// @brief Country name.
|
||||
std::string_view country_name;
|
||||
/**
|
||||
* @brief A sampled first/last name pair, with the source forename's gender.
|
||||
*
|
||||
* Produced by the SampleName() helper in generate_users.cc.
|
||||
*/
|
||||
struct Name {
|
||||
std::string first_name{};
|
||||
std::string last_name{};
|
||||
|
||||
/**
|
||||
* @brief Gender associated with the sampled forename (e.g. "M", "F"), as
|
||||
* reported by the source dataset.
|
||||
*/
|
||||
std::string gender{};
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A single forename entry from the names-by-country fixture data.
|
||||
*/
|
||||
struct ForenameEntry {
|
||||
/**
|
||||
* @brief Romanized forename.
|
||||
*/
|
||||
std::string name{};
|
||||
|
||||
/**
|
||||
* @brief Gender associated with this forename, as reported by the source
|
||||
* dataset (e.g. "M", "F").
|
||||
*/
|
||||
std::string gender{};
|
||||
|
||||
bool operator==(const ForenameEntry& other) const {
|
||||
return name == other.name && gender == other.gender;
|
||||
}
|
||||
};
|
||||
|
||||
namespace std {
|
||||
template <>
|
||||
struct hash<ForenameEntry> {
|
||||
size_t operator()(const ForenameEntry& entry) const noexcept {
|
||||
const size_t name_hash = std::hash<std::string>{}(entry.name);
|
||||
const size_t gender_hash = std::hash<std::string>{}(entry.gender);
|
||||
return name_hash ^ (gender_hash << 1);
|
||||
}
|
||||
};
|
||||
} // namespace std
|
||||
|
||||
/**
|
||||
* @brief A persona archetype used to ground LLM-generated user bios.
|
||||
*/
|
||||
struct UserPersona {
|
||||
/**
|
||||
* @brief Persona display name (e.g. "Hophead Explorer").
|
||||
*/
|
||||
std::string name{};
|
||||
|
||||
/**
|
||||
* @brief Short description of the persona's interests and voice.
|
||||
*/
|
||||
std::string description{};
|
||||
|
||||
/**
|
||||
* @brief Beer styles this persona gravitates toward.
|
||||
*/
|
||||
std::vector<std::string> style_affinities{};
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
@@ -72,22 +139,34 @@ struct BreweryLocation {
|
||||
* @brief LLM sampling parameters.
|
||||
*/
|
||||
struct SamplingOptions {
|
||||
/// @brief LLM sampling temperature (0.0 to 1.0, higher = more random).
|
||||
/**
|
||||
* @brief LLM sampling temperature (higher = more random).
|
||||
*/
|
||||
float temperature = 1.0F;
|
||||
|
||||
/// @brief LLM nucleus sampling top-p parameter.
|
||||
/**
|
||||
* @brief LLM nucleus sampling top-p parameter.
|
||||
*/
|
||||
float top_p = 0.95F;
|
||||
|
||||
/// @brief LLM top-k sampling parameter.
|
||||
/**
|
||||
* @brief LLM top-k sampling parameter.
|
||||
*/
|
||||
uint32_t top_k = 64;
|
||||
|
||||
/// @brief Context window size (tokens).
|
||||
/**
|
||||
* @brief Context window size (tokens).
|
||||
*/
|
||||
uint32_t n_ctx = 8192;
|
||||
|
||||
/// @brief Random seed (-1 for random, otherwise non-negative).
|
||||
/**
|
||||
* @brief Random seed (-1 for random, otherwise non-negative).
|
||||
*/
|
||||
int seed = -1;
|
||||
|
||||
/// @brief Number of layers to offload to GPU.
|
||||
/**
|
||||
* @brief Number of layers to offload to GPU.
|
||||
*/
|
||||
int n_gpu_layers = 0;
|
||||
};
|
||||
|
||||
@@ -95,16 +174,20 @@ struct SamplingOptions {
|
||||
* @brief Configuration for the LLM generator component.
|
||||
*/
|
||||
struct GeneratorOptions {
|
||||
/// @brief Path to the LLM model file (gguf format).
|
||||
/**
|
||||
* @brief Path to the LLM model file (gguf format).
|
||||
*/
|
||||
std::filesystem::path model_path;
|
||||
|
||||
/// @brief Use mocked generator instead of actual LLM inference.
|
||||
/**
|
||||
* @brief Use mocked generator instead of actual LLM inference.
|
||||
*/
|
||||
bool use_mocked = false;
|
||||
|
||||
|
||||
|
||||
/// @brief Specific sampling parameters for this generator.
|
||||
/// If nullopt, the application should use global defaults.
|
||||
/**
|
||||
* @brief Specific sampling parameters for this generator.
|
||||
* If nullopt, the application should use global defaults.
|
||||
*/
|
||||
std::optional<SamplingOptions> sampling;
|
||||
};
|
||||
|
||||
@@ -112,18 +195,23 @@ struct GeneratorOptions {
|
||||
* @brief Configuration for the pipeline execution and output.
|
||||
*/
|
||||
struct PipelineOptions {
|
||||
/// @brief Directory for generated artifacts.
|
||||
/**
|
||||
* @brief Directory for generated artifacts.
|
||||
*/
|
||||
std::filesystem::path output_path;
|
||||
|
||||
/// @brief Directory that contains named prompt files (e.g.
|
||||
/// BREWERY_GENERATION.md).
|
||||
/**
|
||||
* @brief Directory that contains named prompt files (e.g.
|
||||
* BREWERY_GENERATION.md).
|
||||
*/
|
||||
std::filesystem::path prompt_dir;
|
||||
|
||||
/// @brief Path for application logs.
|
||||
std::filesystem::path log_path;
|
||||
|
||||
/// @brief Number of locations to sample from the dataset
|
||||
/// More locations -> more users/more breweries
|
||||
/**
|
||||
* @brief Number of locations to sample from the dataset
|
||||
* More locations -> more users/more breweries
|
||||
*/
|
||||
uint32_t location_count;
|
||||
};
|
||||
|
||||
@@ -139,7 +227,7 @@ struct ApplicationOptions {
|
||||
// Function Declarations
|
||||
// ============================================================================
|
||||
|
||||
std::optional<ApplicationOptions> ParseArguments(const int argc, char** argv,
|
||||
std::shared_ptr<ILogger> logger = nullptr);
|
||||
std::optional<ApplicationOptions> ParseArguments(
|
||||
const int argc, char** argv, std::shared_ptr<ILogger> logger = nullptr);
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_MODELS_H_
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
|
||||
|
||||
/**
|
||||
* @file json_handling/json_loader.h
|
||||
* @brief Loader API for curated location data.
|
||||
*/
|
||||
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "data_model/models.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
/// @brief Loads curated world locations from a JSON file into memory.
|
||||
class JsonLoader {
|
||||
public:
|
||||
/// @brief Parses a JSON array file and returns all location records.
|
||||
static std::vector<Location> LoadLocations(
|
||||
const std::filesystem::path& filepath,
|
||||
std::shared_ptr<ILogger> logger = nullptr);
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
|
||||
@@ -106,4 +106,4 @@ inline void PrettyPrint(std::ostream& outstream,
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_
|
||||
|
||||
@@ -16,16 +16,10 @@
|
||||
*/
|
||||
class LlamaBackendState {
|
||||
public:
|
||||
/// @brief Initializes global llama backend state.
|
||||
LlamaBackendState() { llama_backend_init(); }
|
||||
|
||||
/// @brief Cleans up global llama backend state.
|
||||
~LlamaBackendState() { llama_backend_free(); }
|
||||
|
||||
/// @brief Non-copyable type.
|
||||
LlamaBackendState(const LlamaBackendState&) = delete;
|
||||
|
||||
/// @brief Non-copyable type.
|
||||
LlamaBackendState& operator=(const LlamaBackendState&) = delete;
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
|
||||
|
||||
/**
|
||||
* @file services/curated_data/curated_data_service.h
|
||||
* @brief Abstraction for loading curated location, persona, and name data.
|
||||
*/
|
||||
|
||||
#include <filesystem>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include "data_model/models.h"
|
||||
|
||||
using ForenameList = std::vector<ForenameEntry>;
|
||||
using SurnameList = std::vector<std::string>;
|
||||
using LocationsList = std::vector<Location>;
|
||||
using PersonasList = std::vector<UserPersona>;
|
||||
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
|
||||
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;
|
||||
|
||||
/**
|
||||
* @brief Interface for services that load curated data used to seed
|
||||
* brewery/user generation.
|
||||
*/
|
||||
class ICuratedDataService {
|
||||
public:
|
||||
ICuratedDataService() = default;
|
||||
virtual ~ICuratedDataService() = default;
|
||||
|
||||
ICuratedDataService(const ICuratedDataService&) = delete;
|
||||
ICuratedDataService& operator=(const ICuratedDataService&) = delete;
|
||||
ICuratedDataService(ICuratedDataService&&) = delete;
|
||||
ICuratedDataService& operator=(ICuratedDataService&&) = delete;
|
||||
|
||||
/**
|
||||
* @brief Loads all curated location records.
|
||||
*/
|
||||
virtual const LocationsList& LoadLocations() = 0;
|
||||
|
||||
/**
|
||||
* @brief Loads all curated persona records.
|
||||
*/
|
||||
virtual const PersonasList& LoadPersonas() = 0;
|
||||
virtual const ForenamesByCountryMap& LoadForenamesByCountry() = 0;
|
||||
virtual const SurnamesByCountryMap& LoadSurnamesByCountry() = 0;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
|
||||
@@ -0,0 +1,63 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
|
||||
|
||||
/**
|
||||
* @file json_handling/json_loader.h
|
||||
* @brief JSON-backed implementation of ICuratedDataService.
|
||||
*/
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
#include "data_model/models.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
|
||||
/**
|
||||
* @brief File locations for the curated JSON fixtures consumed by
|
||||
* CuratedJsonDataService.
|
||||
*/
|
||||
struct CuratedDataFilePaths {
|
||||
std::filesystem::path locations_path;
|
||||
std::filesystem::path personas_path;
|
||||
std::filesystem::path forenames_path;
|
||||
std::filesystem::path surnames_path;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Loads curated location, persona, and name data from JSON files.
|
||||
*/
|
||||
class CuratedJsonDataService final : public ICuratedDataService {
|
||||
struct cache {
|
||||
LocationsList locations;
|
||||
PersonasList personas;
|
||||
ForenamesByCountryMap forenames_by_country;
|
||||
SurnamesByCountryMap surnames_by_country;
|
||||
|
||||
cache() = default;
|
||||
~cache() = default;
|
||||
};
|
||||
|
||||
CuratedDataFilePaths filepaths_;
|
||||
cache cache_;
|
||||
|
||||
public:
|
||||
explicit CuratedJsonDataService(CuratedDataFilePaths filepaths);
|
||||
|
||||
/**
|
||||
* @brief Parses a JSON array file and returns all location records.
|
||||
*/
|
||||
const LocationsList& LoadLocations() override;
|
||||
|
||||
/**
|
||||
* @brief Parses a JSON array file and returns all persona records.
|
||||
*/
|
||||
const PersonasList& LoadPersonas() override;
|
||||
|
||||
const ForenamesByCountryMap& LoadForenamesByCountry() override;
|
||||
|
||||
/**
|
||||
* @brief Parses a JSON file and returns all the forenames per country.
|
||||
*/
|
||||
const SurnamesByCountryMap& LoadSurnamesByCountry() override;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
|
||||
@@ -0,0 +1,38 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
|
||||
|
||||
/**
|
||||
* @file services/curated_data/mock_curated_data_service.h
|
||||
* @brief In-memory ICuratedDataService backed by a small fixed dataset, used
|
||||
* when file-backed curated data is disabled (mock mode).
|
||||
*/
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
#include "data_model/models.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
|
||||
/**
|
||||
* @brief Curated data service returning a small fixed in-memory dataset in
|
||||
* place of the JSON fixture files used by JsonLoader.
|
||||
*/
|
||||
class MockCuratedDataService final : public ICuratedDataService {
|
||||
public:
|
||||
MockCuratedDataService();
|
||||
|
||||
const LocationsList& LoadLocations() override;
|
||||
|
||||
const PersonasList& LoadPersonas() override;
|
||||
|
||||
const ForenamesByCountryMap& LoadForenamesByCountry() override;
|
||||
|
||||
const SurnamesByCountryMap& LoadSurnamesByCountry() override;
|
||||
|
||||
private:
|
||||
LocationsList locations_;
|
||||
PersonasList personas_;
|
||||
ForenamesByCountryMap forenames_by_country_;
|
||||
SurnamesByCountryMap surnames_by_country_;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
|
||||
@@ -16,8 +16,6 @@
|
||||
class IExportService {
|
||||
public:
|
||||
IExportService() = default;
|
||||
|
||||
/// @brief Virtual destructor for polymorphic cleanup.
|
||||
virtual ~IExportService() = default;
|
||||
|
||||
IExportService(const IExportService&) = delete;
|
||||
@@ -25,7 +23,9 @@ class IExportService {
|
||||
IExportService(IExportService&&) = delete;
|
||||
IExportService& operator=(IExportService&&) = delete;
|
||||
|
||||
/// @brief Prepares the export destination for a new run.
|
||||
/**
|
||||
* @brief Prepares the export destination for a new run.
|
||||
*/
|
||||
virtual void Initialize() = 0;
|
||||
|
||||
/**
|
||||
@@ -33,9 +33,18 @@ class IExportService {
|
||||
*
|
||||
* @param brewery Generated brewery payload to store.
|
||||
*/
|
||||
virtual uint64_t ProcessRecord(const GeneratedBrewery& brewery) = 0;
|
||||
virtual uint64_t ProcessRecord(const BreweryRecord& brewery) = 0;
|
||||
|
||||
/// @brief Finalizes the export destination.
|
||||
/**
|
||||
* @brief Persists one generated user record.
|
||||
*
|
||||
* @param user Generated user payload to store.
|
||||
*/
|
||||
virtual uint64_t ProcessRecord(const UserRecord& user) = 0;
|
||||
|
||||
/**
|
||||
* @brief Finalizes the export destination.
|
||||
*/
|
||||
virtual void Finalize() = 0;
|
||||
};
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "sqlite_handle_types.h"
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
|
||||
namespace sqlite_export_service_internal {
|
||||
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
#include <unordered_map>
|
||||
|
||||
#include "data_model/models.h"
|
||||
#include "../datetime/date_time_provider.h"
|
||||
#include "export_service.h"
|
||||
#include "sqlite_export_service_helpers.h"
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/database/sqlite_export_service_helpers.h"
|
||||
#include "services/datetime/date_time_provider.h"
|
||||
|
||||
/**
|
||||
* @brief Persists generated brewery records into a fresh SQLite database.
|
||||
@@ -30,7 +30,8 @@ class SqliteExportService final : public IExportService {
|
||||
SqliteExportService& operator=(SqliteExportService&&) = delete;
|
||||
|
||||
void Initialize() override;
|
||||
uint64_t ProcessRecord(const GeneratedBrewery& brewery) override;
|
||||
uint64_t ProcessRecord(const BreweryRecord& brewery) override;
|
||||
uint64_t ProcessRecord(const UserRecord& user) override;
|
||||
void Finalize() override;
|
||||
|
||||
private:
|
||||
@@ -46,6 +47,15 @@ class SqliteExportService final : public IExportService {
|
||||
[[nodiscard]] std::filesystem::path BuildDatabasePath() const;
|
||||
[[nodiscard]] static std::string BuildLocationKey(const Location& location);
|
||||
|
||||
/**
|
||||
* @brief Returns the row id for @p location, inserting it first if it has
|
||||
* not already been seen during this run.
|
||||
*
|
||||
* Shared by both ProcessRecord() overloads so breweries and users
|
||||
* referencing the same location resolve to the same row.
|
||||
*/
|
||||
[[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location);
|
||||
|
||||
std::unique_ptr<IDateTimeProvider> date_time_provider_;
|
||||
std::filesystem::path output_path_;
|
||||
std::string run_timestamp_utc_;
|
||||
@@ -53,6 +63,7 @@ class SqliteExportService final : public IExportService {
|
||||
SqliteDatabaseHandle db_handle_;
|
||||
SqliteStatementHandle insert_location_stmt_;
|
||||
SqliteStatementHandle insert_brewery_stmt_;
|
||||
SqliteStatementHandle insert_user_stmt_;
|
||||
bool transaction_open_ = false;
|
||||
std::unordered_map<std::string, sqlite3_int64> location_cache_;
|
||||
};
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
/* Umbrella header for backward compatibility. */
|
||||
|
||||
#include "sqlite_connection_helpers.h"
|
||||
#include "sqlite_handle_types.h"
|
||||
#include "sqlite_statement_helpers.h"
|
||||
#include "services/database/sqlite_connection_helpers.h"
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
#include "services/database/sqlite_statement_helpers.h"
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_
|
||||
|
||||
@@ -24,8 +24,10 @@ using SqliteDatabaseHandle = std::unique_ptr<sqlite3, SqliteDatabaseDeleter>;
|
||||
using SqliteStatementHandle =
|
||||
std::unique_ptr<sqlite3_stmt, SqliteStatementDeleter>;
|
||||
|
||||
// Represents a parameter that is bound to a prepared SQLite statement.
|
||||
// N.B. indices are 1 based.
|
||||
template <typename T>
|
||||
struct BindParam {
|
||||
struct BoundParam {
|
||||
int index;
|
||||
T value;
|
||||
std::string_view action;
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "sqlite_handle_types.h"
|
||||
#include "services/database/sqlite_handle_types.h"
|
||||
|
||||
namespace sqlite_export_service_internal {
|
||||
|
||||
@@ -50,6 +50,26 @@ CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id);
|
||||
|
||||
)sql";
|
||||
|
||||
inline constexpr std::string_view kCreateUsersTableSql = R"sql(
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
location_id INTEGER NOT NULL,
|
||||
first_name TEXT NOT NULL,
|
||||
last_name TEXT NOT NULL,
|
||||
gender TEXT NOT NULL,
|
||||
username TEXT NOT NULL,
|
||||
bio TEXT NOT NULL,
|
||||
activity_weight REAL NOT NULL,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
date_of_birth TEXT NOT NULL,
|
||||
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id);
|
||||
|
||||
)sql";
|
||||
|
||||
inline constexpr std::string_view kInsertLocationSql = R"sql(
|
||||
INSERT INTO locations (
|
||||
city,
|
||||
@@ -73,20 +93,52 @@ INSERT INTO breweries (
|
||||
) VALUES (?, ?, ?, ?, ?);
|
||||
)sql";
|
||||
|
||||
inline constexpr int kLocationCityBindIndex = 1;
|
||||
inline constexpr int kLocationStateProvinceBindIndex = 2;
|
||||
inline constexpr int kLocationIso31662BindIndex = 3;
|
||||
inline constexpr int kLocationCountryBindIndex = 4;
|
||||
inline constexpr int kLocationIso31661BindIndex = 5;
|
||||
inline constexpr int kLocationLanguagesBindIndex = 6;
|
||||
inline constexpr int kLocationLatitudeBindIndex = 7;
|
||||
inline constexpr int kLocationLongitudeBindIndex = 8;
|
||||
inline constexpr std::string_view kInsertUserSql = R"sql(
|
||||
INSERT INTO users (
|
||||
location_id,
|
||||
first_name,
|
||||
last_name,
|
||||
gender,
|
||||
username,
|
||||
bio,
|
||||
activity_weight,
|
||||
email,
|
||||
date_of_birth
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
)sql";
|
||||
|
||||
inline constexpr int kBreweryLocationIdBindIndex = 1;
|
||||
inline constexpr int kBreweryEnglishNameBindIndex = 2;
|
||||
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3;
|
||||
inline constexpr int kBreweryLocalNameBindIndex = 4;
|
||||
inline constexpr int kBreweryLocalDescriptionBindIndex = 5;
|
||||
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
||||
// placeholder order in the SQL above.
|
||||
enum LocationBindIndex {
|
||||
kLocationCityBindIndex = 1,
|
||||
kLocationStateProvinceBindIndex,
|
||||
kLocationIso31662BindIndex,
|
||||
kLocationCountryBindIndex,
|
||||
kLocationIso31661BindIndex,
|
||||
kLocationLanguagesBindIndex,
|
||||
kLocationLatitudeBindIndex,
|
||||
kLocationLongitudeBindIndex,
|
||||
};
|
||||
|
||||
enum BreweryBindIndex {
|
||||
kBreweryLocationIdBindIndex = 1,
|
||||
kBreweryEnglishNameBindIndex,
|
||||
kBreweryEnglishDescriptionBindIndex,
|
||||
kBreweryLocalNameBindIndex,
|
||||
kBreweryLocalDescriptionBindIndex,
|
||||
};
|
||||
|
||||
enum UserBindIndex {
|
||||
kUserLocationIdBindIndex = 1,
|
||||
kUserFirstNameBindIndex,
|
||||
kUserLastNameBindIndex,
|
||||
kUserGenderBindIndex,
|
||||
kUserUsernameBindIndex,
|
||||
kUserBioBindIndex,
|
||||
kUserActivityWeightBindIndex,
|
||||
kUserEmailBindIndex,
|
||||
kUserDateOfBirthBindIndex,
|
||||
};
|
||||
|
||||
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
||||
std::string_view sql,
|
||||
@@ -95,13 +147,13 @@ SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
||||
void ResetStatement(SqliteStatementHandle& statement);
|
||||
|
||||
void Bind(const SqliteStatementHandle& statement,
|
||||
const BindParam<std::string_view>& param);
|
||||
const BoundParam<std::string_view>& param);
|
||||
|
||||
void Bind(const SqliteStatementHandle& statement,
|
||||
const BindParam<double>& param);
|
||||
const BoundParam<double>& param);
|
||||
|
||||
void Bind(const SqliteStatementHandle& statement,
|
||||
const BindParam<sqlite3_int64>& param);
|
||||
const BoundParam<sqlite3_int64>& param);
|
||||
|
||||
void StepStatement(const SqliteDatabaseHandle& db_handle,
|
||||
const SqliteStatementHandle& statement,
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
*/
|
||||
class IDateTimeProvider {
|
||||
public:
|
||||
/// @brief Virtual destructor for polymorphic cleanup.
|
||||
virtual ~IDateTimeProvider() = default;
|
||||
|
||||
IDateTimeProvider() = default;
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
*/
|
||||
class IEnrichmentService {
|
||||
public:
|
||||
/// @brief Virtual destructor for polymorphic cleanup.
|
||||
virtual ~IEnrichmentService() = default;
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
//
|
||||
// Created by aaronpo on 13/05/2026.
|
||||
//
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
|
||||
|
||||
/**
|
||||
* @file services/enrichment/mock_enrichment.h
|
||||
* @brief No-op IEnrichmentService used when network enrichment is disabled.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "enrichment_service.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
|
||||
/**
|
||||
* @brief Enrichment service that returns no context for any location.
|
||||
*/
|
||||
class MockEnrichmentService final : public IEnrichmentService {
|
||||
public:
|
||||
std::string GetLocationContext(const Location& /*loc*/) override {
|
||||
|
||||
@@ -11,25 +11,37 @@
|
||||
#include <string_view>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "enrichment_service.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "web_client/web_client.h"
|
||||
|
||||
/// @brief Provides Wikipedia summary lookups backed by cached raw extracts.
|
||||
/**
|
||||
* @brief Provides Wikipedia summary lookups backed by cached raw extracts.
|
||||
*/
|
||||
class WikipediaEnrichmentService final : public IEnrichmentService {
|
||||
public:
|
||||
/// @brief Creates a new Wikipedia service with the provided web client.
|
||||
/**
|
||||
* @brief Creates a new Wikipedia service with the provided web client.
|
||||
*/
|
||||
explicit WikipediaEnrichmentService(std::unique_ptr<WebClient> client,
|
||||
std::shared_ptr<ILogger> logger);
|
||||
|
||||
/// @brief Returns the Wikipedia-derived context for a location.
|
||||
/**
|
||||
* @brief Returns the Wikipedia-derived context for a location.
|
||||
*/
|
||||
[[nodiscard]] std::string GetLocationContext(const Location& loc) override;
|
||||
|
||||
private:
|
||||
std::string FetchExtract(std::string_view query);
|
||||
std::unique_ptr<WebClient> client_;
|
||||
std::shared_ptr<ILogger> logger_;
|
||||
/// @brief Canonical cache for raw Wikipedia query extracts.
|
||||
/**
|
||||
* @brief Cache for raw Wikipedia query extracts, keyed by query string.
|
||||
*
|
||||
* GetLocationContext() always queries "brewing" and reuses "beer in
|
||||
* {country}" for every city in that country, so caching avoids refetching
|
||||
* the same extract across locations in a run.
|
||||
*/
|
||||
std::unordered_map<std::string, std::string> extract_cache_;
|
||||
};
|
||||
|
||||
|
||||
@@ -20,10 +20,22 @@
|
||||
* @brief Severity levels supported by the logging infra.
|
||||
*/
|
||||
enum class LogLevel {
|
||||
Debug, ///< Development/debugging information.
|
||||
Info, ///< General informational messages.
|
||||
Warn, ///< Warning conditions.
|
||||
Error, ///< Error conditions.
|
||||
/**
|
||||
* @brief Development/debugging information.
|
||||
*/
|
||||
Debug,
|
||||
/**
|
||||
* @brief General informational messages.
|
||||
*/
|
||||
Info,
|
||||
/**
|
||||
* @brief Warning conditions.
|
||||
*/
|
||||
Warn,
|
||||
/**
|
||||
* @brief Error conditions.
|
||||
*/
|
||||
Error,
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -34,18 +46,44 @@ enum class LogLevel {
|
||||
* pipeline that emitted it.
|
||||
*/
|
||||
enum class PipelinePhase {
|
||||
Startup, ///< Initialization and validation.
|
||||
UserGeneration, ///< User profile generation.
|
||||
BreweryAndBeerGeneration, ///< Brewery and beer data generation.
|
||||
CheckinGeneration, ///< Checkin (visit) record generation.
|
||||
RatingGeneration, ///< Rating and review generation.
|
||||
FollowGeneration, ///< Follow relationship generation.
|
||||
Teardown, ///< Finalization and cleanup.
|
||||
/**
|
||||
* @brief Initialization and validation.
|
||||
*/
|
||||
Startup,
|
||||
/**
|
||||
* @brief Location/context enrichment (e.g. Wikipedia lookups).
|
||||
*/
|
||||
Enrichment,
|
||||
/**
|
||||
* @brief User profile generation.
|
||||
*/
|
||||
UserGeneration,
|
||||
/**
|
||||
* @brief Brewery and beer data generation.
|
||||
*/
|
||||
BreweryAndBeerGeneration,
|
||||
/**
|
||||
* @brief Checkin (visit) record generation.
|
||||
*/
|
||||
CheckinGeneration,
|
||||
/**
|
||||
* @brief Rating and review generation.
|
||||
*/
|
||||
RatingGeneration,
|
||||
/**
|
||||
* @brief Follow relationship generation.
|
||||
*/
|
||||
FollowGeneration,
|
||||
/**
|
||||
* @brief Finalization and cleanup.
|
||||
*/
|
||||
Teardown,
|
||||
};
|
||||
|
||||
/**
|
||||
* @struct LogDTO
|
||||
* @brief User-provided subset of log fields. Used to capture call-site info transparently.
|
||||
* @brief User-provided subset of log fields. Used to capture call-site info
|
||||
* transparently.
|
||||
*/
|
||||
struct LogDTO {
|
||||
LogLevel level;
|
||||
@@ -64,25 +102,35 @@ struct LogDTO {
|
||||
* before the entry is dispatched.
|
||||
*/
|
||||
struct LogEntry {
|
||||
/// @brief Timestamp when the entry was created.
|
||||
/**
|
||||
* @brief Timestamp when the entry was created.
|
||||
*/
|
||||
std::chrono::system_clock::time_point timestamp{};
|
||||
|
||||
/// @brief Source location where the log call was made.
|
||||
/**
|
||||
* @brief Source location where the log call was made.
|
||||
*/
|
||||
std::source_location origin{};
|
||||
|
||||
/// @brief Thread responsible for emitting the log.
|
||||
/**
|
||||
* @brief Thread responsible for emitting the log.
|
||||
*/
|
||||
std::thread::id thread_id{};
|
||||
|
||||
|
||||
/// @brief Severity level of this entry.
|
||||
/**
|
||||
* @brief Severity level of this entry.
|
||||
*/
|
||||
LogLevel level;
|
||||
|
||||
/// @brief Pipeline phase associated with the entry.
|
||||
/**
|
||||
* @brief Pipeline phase associated with the entry.
|
||||
*/
|
||||
PipelinePhase phase;
|
||||
|
||||
/// @brief Log message text.
|
||||
/**
|
||||
* @brief Log message text.
|
||||
*/
|
||||
std::string message;
|
||||
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
* a bounded channel for later processing by the dispatcher.
|
||||
*/
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||
|
||||
#include <string_view>
|
||||
|
||||
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
|
||||
BoundedChannel<LogEntry>& channel_;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
|
||||
|
||||
@@ -39,7 +39,8 @@ class ILogger {
|
||||
*/
|
||||
void Log(LogDTO payload,
|
||||
std::source_location origin = std::source_location::current(),
|
||||
std::chrono::system_clock::time_point timestamp = std::chrono::system_clock::now(),
|
||||
std::chrono::system_clock::time_point timestamp =
|
||||
std::chrono::system_clock::now(),
|
||||
std::thread::id thread_id = std::this_thread::get_id()) {
|
||||
LogEntry entry;
|
||||
entry.timestamp = timestamp;
|
||||
|
||||
@@ -1,19 +1,18 @@
|
||||
/**
|
||||
* @file web_client/http_web_client.h
|
||||
* @brief cpp-httplib implementation of the WebClient interface.
|
||||
*/
|
||||
* @file web_client/http_web_client.h
|
||||
* @brief cpp-httplib implementation of the WebClient interface.
|
||||
*/
|
||||
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
|
||||
|
||||
|
||||
#include "web_client/web_client.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "services/logging/logger.h"
|
||||
#include "web_client/web_client.h"
|
||||
|
||||
/**
|
||||
* @brief WebClient implementation backed by cpp-httplib.
|
||||
*
|
||||
@@ -26,7 +25,7 @@
|
||||
* bound to a single origin at construction time.
|
||||
*/
|
||||
class HttpWebClient final : public WebClient {
|
||||
public:
|
||||
public:
|
||||
explicit HttpWebClient(std::shared_ptr<ILogger> logger)
|
||||
: logger_(std::move(logger)) {}
|
||||
~HttpWebClient() override = default;
|
||||
@@ -34,7 +33,8 @@ public:
|
||||
/**
|
||||
* @brief Executes a blocking HTTP/HTTPS GET request against a full URL.
|
||||
*
|
||||
* @param url Fully-qualified URL, e.g. "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
|
||||
* @param url Fully-qualified URL, e.g.
|
||||
* "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
|
||||
* @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
|
||||
*/
|
||||
std::string Get(const std::string& url) override;
|
||||
@@ -52,5 +52,4 @@ public:
|
||||
std::shared_ptr<ILogger> logger_;
|
||||
};
|
||||
|
||||
|
||||
#endif
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
*/
|
||||
class WebClient {
|
||||
public:
|
||||
/// @brief Virtual destructor for polymorphic cleanup.
|
||||
virtual ~WebClient() = default;
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user