Pipeline: Add LLM and mocked user generation to the pipeline (#230)

This commit is contained in:
2026-07-01 07:47:24 -04:00
committed by GitHub
parent 880f73e004
commit 2b8a900d12
90 changed files with 57254 additions and 800 deletions

View File

@@ -10,6 +10,7 @@
#include <boost/program_options.hpp>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <memory>
#include <optional>
#include <string>
@@ -28,40 +29,106 @@ namespace prog_opts = boost::program_options;
* @brief Canonical location record for city-level generation.
*/
struct Location {
/// @brief City name.
std::string city{};
/// @brief State or province name.
std::string state_province{};
/// @brief ISO 3166-2 subdivision code.
/**
* @brief ISO 3166-2 subdivision code.
*/
std::string iso3166_2{};
/// @brief Country name.
std::string country{};
/// @brief ISO 3166-1 country code.
/**
* @brief ISO 3166-1 country code.
*/
std::string iso3166_1{};
/// @brief Local language codes in priority order.
/**
* @brief Local language codes in priority order.
*/
std::vector<std::string> local_languages{};
/// @brief Latitude in decimal degrees.
/**
* @brief Latitude in decimal degrees.
*/
double latitude{};
/// @brief Longitude in decimal degrees.
/**
* @brief Longitude in decimal degrees.
*/
double longitude{};
};
/**
* @brief Non-owning brewery location input.
*/
struct BreweryLocation {
/// @brief City name.
std::string_view city_name;
// ============================================================================
// Name / Persona Models
// ============================================================================
/// @brief Country name.
std::string_view country_name;
/**
* @brief A sampled first/last name pair, with the source forename's gender.
*
* Produced by the SampleName() helper in generate_users.cc.
*/
struct Name {
std::string first_name{};
std::string last_name{};
/**
* @brief Gender associated with the sampled forename (e.g. "M", "F"), as
* reported by the source dataset.
*/
std::string gender{};
};
/**
* @brief A single forename entry from the names-by-country fixture data.
*/
struct ForenameEntry {
/**
* @brief Romanized forename.
*/
std::string name{};
/**
* @brief Gender associated with this forename, as reported by the source
* dataset (e.g. "M", "F").
*/
std::string gender{};
bool operator==(const ForenameEntry& other) const {
return name == other.name && gender == other.gender;
}
};
namespace std {
template <>
struct hash<ForenameEntry> {
size_t operator()(const ForenameEntry& entry) const noexcept {
const size_t name_hash = std::hash<std::string>{}(entry.name);
const size_t gender_hash = std::hash<std::string>{}(entry.gender);
return name_hash ^ (gender_hash << 1);
}
};
} // namespace std
/**
* @brief A persona archetype used to ground LLM-generated user bios.
*/
struct UserPersona {
/**
* @brief Persona display name (e.g. "Hophead Explorer").
*/
std::string name{};
/**
* @brief Short description of the persona's interests and voice.
*/
std::string description{};
/**
* @brief Beer styles this persona gravitates toward.
*/
std::vector<std::string> style_affinities{};
};
// ============================================================================
@@ -72,22 +139,34 @@ struct BreweryLocation {
* @brief LLM sampling parameters.
*/
struct SamplingOptions {
/// @brief LLM sampling temperature (0.0 to 1.0, higher = more random).
/**
* @brief LLM sampling temperature (higher = more random).
*/
float temperature = 1.0F;
/// @brief LLM nucleus sampling top-p parameter.
/**
* @brief LLM nucleus sampling top-p parameter.
*/
float top_p = 0.95F;
/// @brief LLM top-k sampling parameter.
/**
* @brief LLM top-k sampling parameter.
*/
uint32_t top_k = 64;
/// @brief Context window size (tokens).
/**
* @brief Context window size (tokens).
*/
uint32_t n_ctx = 8192;
/// @brief Random seed (-1 for random, otherwise non-negative).
/**
* @brief Random seed (-1 for random, otherwise non-negative).
*/
int seed = -1;
/// @brief Number of layers to offload to GPU.
/**
* @brief Number of layers to offload to GPU.
*/
int n_gpu_layers = 0;
};
@@ -95,16 +174,20 @@ struct SamplingOptions {
* @brief Configuration for the LLM generator component.
*/
struct GeneratorOptions {
/// @brief Path to the LLM model file (gguf format).
/**
* @brief Path to the LLM model file (gguf format).
*/
std::filesystem::path model_path;
/// @brief Use mocked generator instead of actual LLM inference.
/**
* @brief Use mocked generator instead of actual LLM inference.
*/
bool use_mocked = false;
/// @brief Specific sampling parameters for this generator.
/// If nullopt, the application should use global defaults.
/**
* @brief Specific sampling parameters for this generator.
* If nullopt, the application should use global defaults.
*/
std::optional<SamplingOptions> sampling;
};
@@ -112,18 +195,23 @@ struct GeneratorOptions {
* @brief Configuration for the pipeline execution and output.
*/
struct PipelineOptions {
/// @brief Directory for generated artifacts.
/**
* @brief Directory for generated artifacts.
*/
std::filesystem::path output_path;
/// @brief Directory that contains named prompt files (e.g.
/// BREWERY_GENERATION.md).
/**
* @brief Directory that contains named prompt files (e.g.
* BREWERY_GENERATION.md).
*/
std::filesystem::path prompt_dir;
/// @brief Path for application logs.
std::filesystem::path log_path;
/// @brief Number of locations to sample from the dataset
/// More locations -> more users/more breweries
/**
* @brief Number of locations to sample from the dataset
* More locations -> more users/more breweries
*/
uint32_t location_count;
};
@@ -139,7 +227,7 @@ struct ApplicationOptions {
// Function Declarations
// ============================================================================
std::optional<ApplicationOptions> ParseArguments(const int argc, char** argv,
std::shared_ptr<ILogger> logger = nullptr);
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger = nullptr);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_MODELS_H_