Pipeline: Add LLM and mocked user generation to the pipeline (#230)

This commit is contained in:
2026-07-01 07:47:24 -04:00
committed by GitHub
parent 880f73e004
commit 2b8a900d12
90 changed files with 57254 additions and 800 deletions

View File

@@ -3,8 +3,8 @@
/**
* @file data_model/generated_models.h
* @brief Generated output models from the pipeline: brewery/user results, enriched data,
* and complete generation results.
* @brief Generated output models from the pipeline: brewery/user results,
* enriched data, and complete generation results.
*/
#include <string>
@@ -19,16 +19,24 @@
* @brief Generated brewery payload.
*/
struct BreweryResult {
/// @brief Brewery display name in English.
/**
* @brief Brewery display name in English.
*/
std::string name_en;
/// @brief Brewery description text in English.
/**
* @brief Brewery description text in English.
*/
std::string description_en;
/// @brief Brewery display name in the local language.
/**
* @brief Brewery display name in the local language.
*/
std::string name_local;
/// @brief Brewery description text in the local language.
/**
* @brief Brewery description text in the local language.
*/
std::string description_local;
};
@@ -36,11 +44,39 @@ struct BreweryResult {
* @brief Generated user profile payload.
*/
struct UserResult {
/// @brief Username handle.
/**
* @brief First (given) name, copied from the sampled Name -- not
* LLM-invented.
*/
std::string first_name{};
/**
* @brief Last (family) name, copied from the sampled Name -- not
* LLM-invented.
*/
std::string last_name{};
/**
* @brief Gender associated with the sampled first name, copied from the
* sampled Name -- not LLM-invented.
*/
std::string gender{};
/**
* @brief Username handle.
*/
std::string username{};
/// @brief Short user biography.
/**
* @brief Short user biography.
*/
std::string bio{};
/**
* @brief Relative check-in/activity weight for this user, used to drive
* a J-curve activity profile in later pipeline phases.
*/
float activity_weight{};
};
// ============================================================================
@@ -58,9 +94,23 @@ struct EnrichedCity {
/**
* @brief Helper struct to store generated brewery data.
*/
struct GeneratedBrewery {
struct BreweryRecord {
Location location;
BreweryResult brewery;
};
/**
* @brief Helper struct to store generated user data.
*
* `email` and `date_of_birth` are programmatically generated by the
* orchestrator (never LLM-authored) so a downstream auth-account seeding
* consumer can register real accounts from the pipeline's SQLite export.
*/
struct UserRecord {
Location location;
UserResult user;
std::string email{};
std::string date_of_birth{};
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_GENERATED_MODELS_H_

View File

@@ -10,6 +10,7 @@
#include <boost/program_options.hpp>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <memory>
#include <optional>
#include <string>
@@ -28,40 +29,106 @@ namespace prog_opts = boost::program_options;
* @brief Canonical location record for city-level generation.
*/
struct Location {
/// @brief City name.
std::string city{};
/// @brief State or province name.
std::string state_province{};
/// @brief ISO 3166-2 subdivision code.
/**
* @brief ISO 3166-2 subdivision code.
*/
std::string iso3166_2{};
/// @brief Country name.
std::string country{};
/// @brief ISO 3166-1 country code.
/**
* @brief ISO 3166-1 country code.
*/
std::string iso3166_1{};
/// @brief Local language codes in priority order.
/**
* @brief Local language codes in priority order.
*/
std::vector<std::string> local_languages{};
/// @brief Latitude in decimal degrees.
/**
* @brief Latitude in decimal degrees.
*/
double latitude{};
/// @brief Longitude in decimal degrees.
/**
* @brief Longitude in decimal degrees.
*/
double longitude{};
};
/**
* @brief Non-owning brewery location input.
*/
struct BreweryLocation {
/// @brief City name.
std::string_view city_name;
// ============================================================================
// Name / Persona Models
// ============================================================================
/// @brief Country name.
std::string_view country_name;
/**
* @brief A sampled first/last name pair, with the source forename's gender.
*
* Produced by the SampleName() helper in generate_users.cc.
*/
struct Name {
std::string first_name{};
std::string last_name{};
/**
* @brief Gender associated with the sampled forename (e.g. "M", "F"), as
* reported by the source dataset.
*/
std::string gender{};
};
/**
* @brief A single forename entry from the names-by-country fixture data.
*/
struct ForenameEntry {
/**
* @brief Romanized forename.
*/
std::string name{};
/**
* @brief Gender associated with this forename, as reported by the source
* dataset (e.g. "M", "F").
*/
std::string gender{};
bool operator==(const ForenameEntry& other) const {
return name == other.name && gender == other.gender;
}
};
namespace std {
template <>
struct hash<ForenameEntry> {
size_t operator()(const ForenameEntry& entry) const noexcept {
const size_t name_hash = std::hash<std::string>{}(entry.name);
const size_t gender_hash = std::hash<std::string>{}(entry.gender);
return name_hash ^ (gender_hash << 1);
}
};
} // namespace std
/**
* @brief A persona archetype used to ground LLM-generated user bios.
*/
struct UserPersona {
/**
* @brief Persona display name (e.g. "Hophead Explorer").
*/
std::string name{};
/**
* @brief Short description of the persona's interests and voice.
*/
std::string description{};
/**
* @brief Beer styles this persona gravitates toward.
*/
std::vector<std::string> style_affinities{};
};
// ============================================================================
@@ -72,22 +139,34 @@ struct BreweryLocation {
* @brief LLM sampling parameters.
*/
struct SamplingOptions {
/// @brief LLM sampling temperature (0.0 to 1.0, higher = more random).
/**
* @brief LLM sampling temperature (higher = more random).
*/
float temperature = 1.0F;
/// @brief LLM nucleus sampling top-p parameter.
/**
* @brief LLM nucleus sampling top-p parameter.
*/
float top_p = 0.95F;
/// @brief LLM top-k sampling parameter.
/**
* @brief LLM top-k sampling parameter.
*/
uint32_t top_k = 64;
/// @brief Context window size (tokens).
/**
* @brief Context window size (tokens).
*/
uint32_t n_ctx = 8192;
/// @brief Random seed (-1 for random, otherwise non-negative).
/**
* @brief Random seed (-1 for random, otherwise non-negative).
*/
int seed = -1;
/// @brief Number of layers to offload to GPU.
/**
* @brief Number of layers to offload to GPU.
*/
int n_gpu_layers = 0;
};
@@ -95,16 +174,20 @@ struct SamplingOptions {
* @brief Configuration for the LLM generator component.
*/
struct GeneratorOptions {
/// @brief Path to the LLM model file (gguf format).
/**
* @brief Path to the LLM model file (gguf format).
*/
std::filesystem::path model_path;
/// @brief Use mocked generator instead of actual LLM inference.
/**
* @brief Use mocked generator instead of actual LLM inference.
*/
bool use_mocked = false;
/// @brief Specific sampling parameters for this generator.
/// If nullopt, the application should use global defaults.
/**
* @brief Specific sampling parameters for this generator.
* If nullopt, the application should use global defaults.
*/
std::optional<SamplingOptions> sampling;
};
@@ -112,18 +195,23 @@ struct GeneratorOptions {
* @brief Configuration for the pipeline execution and output.
*/
struct PipelineOptions {
/// @brief Directory for generated artifacts.
/**
* @brief Directory for generated artifacts.
*/
std::filesystem::path output_path;
/// @brief Directory that contains named prompt files (e.g.
/// BREWERY_GENERATION.md).
/**
* @brief Directory that contains named prompt files (e.g.
* BREWERY_GENERATION.md).
*/
std::filesystem::path prompt_dir;
/// @brief Path for application logs.
std::filesystem::path log_path;
/// @brief Number of locations to sample from the dataset
/// More locations -> more users/more breweries
/**
* @brief Number of locations to sample from the dataset
* More locations -> more users/more breweries
*/
uint32_t location_count;
};
@@ -139,7 +227,7 @@ struct ApplicationOptions {
// Function Declarations
// ============================================================================
std::optional<ApplicationOptions> ParseArguments(const int argc, char** argv,
std::shared_ptr<ILogger> logger = nullptr);
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger = nullptr);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_MODELS_H_