From 6d8904cf01372c804e7d7ec2cb5c48cd27dd9fc1 Mon Sep 17 00:00:00 2001 From: Aaron Po Date: Sun, 28 Jun 2026 13:07:14 -0400 Subject: [PATCH] Update user generation --- tooling/pipeline/CMakeLists.txt | 5 -- .../pipeline/includes/biergarten_pipeline.h | 69 +++++++++++++++++ .../biergarten_pipeline_orchestrator.h | 13 +--- .../includes/concurrency/bounded_channel.tcc | 2 +- .../data_generation/llama_generator.h | 5 +- tooling/pipeline/includes/data_model/models.h | 16 ++++ .../includes/data_model/names_by_country.h | 51 ------------ .../includes/json_handling/json_loader.h | 34 ++++---- .../includes/json_handling/pretty_print.h | 2 +- .../curated_data/curated_data_service.h | 18 ++--- .../database/sqlite_connection_helpers.h | 2 +- .../services/database/sqlite_export_service.h | 6 +- .../database/sqlite_export_service_helpers.h | 6 +- .../database/sqlite_statement_helpers.h | 52 +++++++------ .../services/enrichment/mock_enrichment.h | 2 +- .../services/enrichment/wikipedia_service.h | 2 +- .../includes/services/logging/log_producer.h | 6 +- .../includes/web_client/http_web_client.h | 2 +- .../biergarten_pipeline_orchestrator.cc | 13 ++-- .../generate_users.cc | 77 ++++++++++++++----- .../log_results.cc | 2 +- .../src/data_generation/llama/helpers.cc | 4 +- .../src/data_generation/llama/infer.cc | 3 +- .../data_generation/llama/llama_generator.cc | 3 +- .../src/data_generation/llama/load.cc | 5 +- .../src/data_model/names_by_country.cc | 40 ---------- .../pipeline/src/json_handling/json_loader.cc | 1 - tooling/pipeline/src/main.cc | 55 ++++--------- 28 files changed, 252 insertions(+), 244 deletions(-) create mode 100644 tooling/pipeline/includes/biergarten_pipeline.h delete mode 100644 tooling/pipeline/includes/data_model/names_by_country.h delete mode 100644 tooling/pipeline/src/data_model/names_by_country.cc diff --git a/tooling/pipeline/CMakeLists.txt b/tooling/pipeline/CMakeLists.txt index 1a8f66f..63ec6a7 100644 --- a/tooling/pipeline/CMakeLists.txt +++ b/tooling/pipeline/CMakeLists.txt @@ -151,11 +151,6 @@ target_sources(${PROJECT_NAME} PRIVATE src/json_handling/json_loader.cc ) -# --- data_model --- -target_sources(${PROJECT_NAME} PRIVATE - src/data_model/names_by_country.cc -) - # --- application_options --- target_sources(${PROJECT_NAME} PRIVATE src/application_options/parse_arguments.cc diff --git a/tooling/pipeline/includes/biergarten_pipeline.h b/tooling/pipeline/includes/biergarten_pipeline.h new file mode 100644 index 0000000..f0f2555 --- /dev/null +++ b/tooling/pipeline/includes/biergarten_pipeline.h @@ -0,0 +1,69 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_ + +/** + * @file biergarten_pipeline.h + * @brief Master umbrella header — includes every public header in the + * Biergarten pipeline. + */ + +// --- orchestrator --- +#include "biergarten_pipeline_orchestrator.h" + +// --- concurrency --- +#include "concurrency/bounded_channel.h" + +// --- data_generation --- +#include "data_generation/data_generator.h" +#include "data_generation/json_grammars.h" +#include "data_generation/llama_generator.h" +#include "data_generation/llama_generator_helpers.h" +#include "data_generation/mock_generator.h" +#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h" +#include "data_generation/prompt_formatting/prompt_formatter.h" + +// --- data_model --- +#include "data_model/generated_models.h" +#include "data_model/models.h" + +// --- json_handling --- +#include "json_handling/json_loader.h" +#include "json_handling/pretty_print.h" + +// --- llama backend --- +#include "llama_backend_state.h" + +// --- services: curated_data --- +#include "services/curated_data/curated_data_service.h" + +// --- services: database --- +#include "services/database/export_service.h" +#include "services/database/sqlite_connection_helpers.h" +#include "services/database/sqlite_export_service.h" +#include "services/database/sqlite_export_service_helpers.h" +#include "services/database/sqlite_handle_types.h" +#include "services/database/sqlite_statement_helpers.h" + +// --- services: datetime --- +#include "services/datetime/date_time_provider.h" +#include "services/datetime/timer.h" + +// --- services: enrichment --- +#include "services/enrichment/enrichment_service.h" +#include "services/enrichment/mock_enrichment.h" +#include "services/enrichment/wikipedia_service.h" + +// --- services: logging --- +#include "services/logging/log_dispatcher.h" +#include "services/logging/log_entry.h" +#include "services/logging/log_producer.h" +#include "services/logging/logger.h" + +// --- services: prompting --- +#include "services/prompting/prompt_directory.h" + +// --- web_client --- +#include "web_client/http_web_client.h" +#include "web_client/web_client.h" + +#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_ diff --git a/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h b/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h index 5833b45..c65219a 100644 --- a/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h +++ b/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h @@ -1,12 +1,9 @@ -#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ -#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_ /** * @file biergarten_pipeline_orchestrator.h * @brief Orchestration for end-to-end brewery and user data generation. - * - * Coordinates location loading, enrichment, and generation phases to produce - * a complete dataset, wiring dependencies selected at the composition root. */ #include @@ -103,10 +100,6 @@ class BiergartenPipelineOrchestrator { * @brief Generate users grounded in sampled names and personas for * enriched cities. * - * Loads personas.json / forenames-by-country.json / - * surnames-by-country.json itself, mirroring how QueryCitiesWithCountries() - * owns its own locations.json load. - * * @param cities Span of enriched city data. */ void GenerateUsers(std::span cities); @@ -120,4 +113,4 @@ class BiergartenPipelineOrchestrator { std::vector generated_users_; }; -#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ +#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_ diff --git a/tooling/pipeline/includes/concurrency/bounded_channel.tcc b/tooling/pipeline/includes/concurrency/bounded_channel.tcc index 951bfbd..5345eb7 100644 --- a/tooling/pipeline/includes/concurrency/bounded_channel.tcc +++ b/tooling/pipeline/includes/concurrency/bounded_channel.tcc @@ -1,4 +1,4 @@ -#include "bounded_channel.h" +#include "concurrency/bounded_channel.h" template void BoundedChannel::Send(T item) { diff --git a/tooling/pipeline/includes/data_generation/llama_generator.h b/tooling/pipeline/includes/data_generation/llama_generator.h index 8a43681..7d4dadb 100644 --- a/tooling/pipeline/includes/data_generation/llama_generator.h +++ b/tooling/pipeline/includes/data_generation/llama_generator.h @@ -1,24 +1,23 @@ #ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_ #define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_ -#include - /** * @file data_generation/llama_generator.h * @brief llama.cpp-backed implementation of DataGenerator. */ #include +#include #include #include #include #include -#include "../services/prompting/prompt_directory.h" #include "data_generation/data_generator.h" #include "data_generation/prompt_formatting/prompt_formatter.h" #include "data_model/models.h" #include "services/logging/logger.h" +#include "services/prompting/prompt_directory.h" struct llama_model; struct llama_context; diff --git a/tooling/pipeline/includes/data_model/models.h b/tooling/pipeline/includes/data_model/models.h index 6dcdb26..d413b82 100644 --- a/tooling/pipeline/includes/data_model/models.h +++ b/tooling/pipeline/includes/data_model/models.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -93,8 +94,23 @@ struct ForenameEntry { * dataset (e.g. "M", "F"). */ std::string gender{}; + + bool operator==(const ForenameEntry& other) const { + return name == other.name && gender == other.gender; + } }; +namespace std { +template <> +struct hash { + size_t operator()(const ForenameEntry& entry) const noexcept { + const size_t name_hash = std::hash{}(entry.name); + const size_t gender_hash = std::hash{}(entry.gender); + return name_hash ^ (gender_hash << 1); + } +}; +} // namespace std + /** * @brief A persona archetype used to ground LLM-generated user bios. */ diff --git a/tooling/pipeline/includes/data_model/names_by_country.h b/tooling/pipeline/includes/data_model/names_by_country.h deleted file mode 100644 index 8d01cde..0000000 --- a/tooling/pipeline/includes/data_model/names_by_country.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_ -#define BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_ - -/** - * @file data_model/names_by_country.h - * @brief Sampling over the names-by-country fixture data. - */ - -#include -#include -#include -#include -#include - -#include "data_model/models.h" - -/** - * @brief Samples locale-appropriate names from curated forename/surname - * fixture data, keyed by ISO 3166-1 country code. - * - * Forenames and surnames are kept in separate maps (mirroring the source - * dataset's own shape) so per-forename gender is preserved; pairing happens - * at sample time rather than being precomputed, so no information from the - * source dataset is discarded up front. - */ -class NamesByCountry { - public: - NamesByCountry(std::unordered_map> - forenames_by_country, - std::unordered_map> - surnames_by_country); - - /** - * @brief Samples a first/last name pair for the given country. - * - * @param iso3166_1 ISO 3166-1 country code to sample for. - * @param rng Random source used for sampling. - * @return A sampled Name, or std::nullopt if the country has no forename - * or no surname entries. - */ - [[nodiscard]] std::optional SampleName(const std::string& iso3166_1, - std::mt19937& rng) const; - - private: - std::unordered_map> - forenames_by_country_; - std::unordered_map> - surnames_by_country_; -}; - -#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_ diff --git a/tooling/pipeline/includes/json_handling/json_loader.h b/tooling/pipeline/includes/json_handling/json_loader.h index 45cb79f..fba4a82 100644 --- a/tooling/pipeline/includes/json_handling/json_loader.h +++ b/tooling/pipeline/includes/json_handling/json_loader.h @@ -7,41 +7,49 @@ */ #include +#include #include #include "data_model/models.h" -#include "data_model/names_by_country.h" #include "services/curated_data/curated_data_service.h" /** * @brief Loads curated location, persona, and name data from JSON files. */ class JsonLoader final : public ICuratedDataService { + struct cache { + std::vector locations; + std::vector personas; + std::unordered_map forenames_by_country; + std::unordered_map surnames_by_country; + + cache() = default; + ~cache() = default; + }; + + cache cache_; + public: JsonLoader() = default; /** * @brief Parses a JSON array file and returns all location records. */ - std::vector LoadLocations( - const std::filesystem::path& filepath) override; + std::vector LoadLocations(const std::filesystem::path&) override; /** * @brief Parses a JSON array file and returns all persona records. */ - std::vector LoadPersonas( - const std::filesystem::path& filepath) override; + std::vector LoadPersonas(const std::filesystem::path&) override; + + std::unordered_map LoadForenamesByCountry( + const std::filesystem::path&) override; /** - * @brief Parses the names-by-country fixture pair into a sampling-capable - * NamesByCountry. - * - * @param forenames_filepath Path to forenames-by-country.json. - * @param surnames_filepath Path to surnames-by-country.json. + * @brief Parses a JSON file and returns all the forenames per country. */ - NamesByCountry LoadNamesByCountry( - const std::filesystem::path& forenames_filepath, - const std::filesystem::path& surnames_filepath) override; + std::unordered_map LoadSurnamesByCountry( + const std::filesystem::path&) override; }; #endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_ diff --git a/tooling/pipeline/includes/json_handling/pretty_print.h b/tooling/pipeline/includes/json_handling/pretty_print.h index 51e7486..3d148c3 100644 --- a/tooling/pipeline/includes/json_handling/pretty_print.h +++ b/tooling/pipeline/includes/json_handling/pretty_print.h @@ -106,4 +106,4 @@ inline void PrettyPrint(std::ostream& outstream, } } -#endif +#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_ diff --git a/tooling/pipeline/includes/services/curated_data/curated_data_service.h b/tooling/pipeline/includes/services/curated_data/curated_data_service.h index e2ddbc5..2cf52d5 100644 --- a/tooling/pipeline/includes/services/curated_data/curated_data_service.h +++ b/tooling/pipeline/includes/services/curated_data/curated_data_service.h @@ -7,11 +7,13 @@ */ #include +#include #include #include "data_model/models.h" -#include "data_model/names_by_country.h" +using forename_list = std::unordered_set; +using surname_list = std::unordered_set; /** * @brief Interface for services that load curated data used to seed * brewery/user generation. @@ -38,16 +40,10 @@ class ICuratedDataService { virtual std::vector LoadPersonas( const std::filesystem::path& filepath) = 0; - /** - * @brief Loads the names-by-country fixture pair into a sampling-capable - * NamesByCountry. - * - * @param forenames_filepath Path to forenames-by-country.json. - * @param surnames_filepath Path to surnames-by-country.json. - */ - virtual NamesByCountry LoadNamesByCountry( - const std::filesystem::path& forenames_filepath, + virtual std::unordered_map LoadForenamesByCountry( + const std::filesystem::path& forenames_filepath) = 0; + virtual std::unordered_map LoadSurnamesByCountry( const std::filesystem::path& surnames_filepath) = 0; }; -#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_ +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_ \ No newline at end of file diff --git a/tooling/pipeline/includes/services/database/sqlite_connection_helpers.h b/tooling/pipeline/includes/services/database/sqlite_connection_helpers.h index 1bb4832..195361b 100644 --- a/tooling/pipeline/includes/services/database/sqlite_connection_helpers.h +++ b/tooling/pipeline/includes/services/database/sqlite_connection_helpers.h @@ -12,7 +12,7 @@ #include #include -#include "sqlite_handle_types.h" +#include "services/database/sqlite_handle_types.h" namespace sqlite_export_service_internal { diff --git a/tooling/pipeline/includes/services/database/sqlite_export_service.h b/tooling/pipeline/includes/services/database/sqlite_export_service.h index de16445..e5c4936 100644 --- a/tooling/pipeline/includes/services/database/sqlite_export_service.h +++ b/tooling/pipeline/includes/services/database/sqlite_export_service.h @@ -11,10 +11,10 @@ #include #include -#include "../datetime/date_time_provider.h" #include "data_model/models.h" -#include "export_service.h" -#include "sqlite_export_service_helpers.h" +#include "services/database/export_service.h" +#include "services/database/sqlite_export_service_helpers.h" +#include "services/datetime/date_time_provider.h" /** * @brief Persists generated brewery records into a fresh SQLite database. diff --git a/tooling/pipeline/includes/services/database/sqlite_export_service_helpers.h b/tooling/pipeline/includes/services/database/sqlite_export_service_helpers.h index 838d40e..3df35c0 100644 --- a/tooling/pipeline/includes/services/database/sqlite_export_service_helpers.h +++ b/tooling/pipeline/includes/services/database/sqlite_export_service_helpers.h @@ -3,8 +3,8 @@ /* Umbrella header for backward compatibility. */ -#include "sqlite_connection_helpers.h" -#include "sqlite_handle_types.h" -#include "sqlite_statement_helpers.h" +#include "services/database/sqlite_connection_helpers.h" +#include "services/database/sqlite_handle_types.h" +#include "services/database/sqlite_statement_helpers.h" #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_ diff --git a/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h b/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h index 1e7b874..c1b8373 100644 --- a/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h +++ b/tooling/pipeline/includes/services/database/sqlite_statement_helpers.h @@ -13,7 +13,7 @@ #include #include -#include "sqlite_handle_types.h" +#include "services/database/sqlite_handle_types.h" namespace sqlite_export_service_internal { @@ -109,30 +109,36 @@ INSERT INTO users ( // sqlite3_bind_*() parameter indices are 1-based, matching the "?" // placeholder order in the SQL above. -inline constexpr int kLocationCityBindIndex = 1; -inline constexpr int kLocationStateProvinceBindIndex = 2; -inline constexpr int kLocationIso31662BindIndex = 3; -inline constexpr int kLocationCountryBindIndex = 4; -inline constexpr int kLocationIso31661BindIndex = 5; -inline constexpr int kLocationLanguagesBindIndex = 6; -inline constexpr int kLocationLatitudeBindIndex = 7; -inline constexpr int kLocationLongitudeBindIndex = 8; +enum eLocationBindIndex { + kLocationCityBindIndex = 1, + kLocationStateProvinceBindIndex, + kLocationIso31662BindIndex, + kLocationCountryBindIndex, + kLocationIso31661BindIndex, + kLocationLanguagesBindIndex, + kLocationLatitudeBindIndex, + kLocationLongitudeBindIndex, +}; -inline constexpr int kBreweryLocationIdBindIndex = 1; -inline constexpr int kBreweryEnglishNameBindIndex = 2; -inline constexpr int kBreweryEnglishDescriptionBindIndex = 3; -inline constexpr int kBreweryLocalNameBindIndex = 4; -inline constexpr int kBreweryLocalDescriptionBindIndex = 5; +enum BreweryBindIndex { + kBreweryLocationIdBindIndex = 1, + kBreweryEnglishNameBindIndex, + kBreweryEnglishDescriptionBindIndex, + kBreweryLocalNameBindIndex, + kBreweryLocalDescriptionBindIndex, +}; -inline constexpr int kUserLocationIdBindIndex = 1; -inline constexpr int kUserFirstNameBindIndex = 2; -inline constexpr int kUserLastNameBindIndex = 3; -inline constexpr int kUserGenderBindIndex = 4; -inline constexpr int kUserUsernameBindIndex = 5; -inline constexpr int kUserBioBindIndex = 6; -inline constexpr int kUserActivityWeightBindIndex = 7; -inline constexpr int kUserEmailBindIndex = 8; -inline constexpr int kUserDateOfBirthBindIndex = 9; +enum UserBindIndex { + kUserLocationIdBindIndex = 1, + kUserFirstNameBindIndex, + kUserLastNameBindIndex, + kUserGenderBindIndex, + kUserUsernameBindIndex, + kUserBioBindIndex, + kUserActivityWeightBindIndex, + kUserEmailBindIndex, + kUserDateOfBirthBindIndex, +}; SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, std::string_view sql, diff --git a/tooling/pipeline/includes/services/enrichment/mock_enrichment.h b/tooling/pipeline/includes/services/enrichment/mock_enrichment.h index 6d527eb..9a45b3f 100644 --- a/tooling/pipeline/includes/services/enrichment/mock_enrichment.h +++ b/tooling/pipeline/includes/services/enrichment/mock_enrichment.h @@ -8,7 +8,7 @@ #include -#include "enrichment_service.h" +#include "services/enrichment/enrichment_service.h" /** * @brief Enrichment service that returns no context for any location. diff --git a/tooling/pipeline/includes/services/enrichment/wikipedia_service.h b/tooling/pipeline/includes/services/enrichment/wikipedia_service.h index 6dc5c4b..1063aea 100644 --- a/tooling/pipeline/includes/services/enrichment/wikipedia_service.h +++ b/tooling/pipeline/includes/services/enrichment/wikipedia_service.h @@ -11,7 +11,7 @@ #include #include -#include "enrichment_service.h" +#include "services/enrichment/enrichment_service.h" #include "services/logging/logger.h" #include "web_client/web_client.h" diff --git a/tooling/pipeline/includes/services/logging/log_producer.h b/tooling/pipeline/includes/services/logging/log_producer.h index d032f89..253c016 100644 --- a/tooling/pipeline/includes/services/logging/log_producer.h +++ b/tooling/pipeline/includes/services/logging/log_producer.h @@ -6,8 +6,8 @@ * a bounded channel for later processing by the dispatcher. */ -#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ -#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_ #include @@ -50,4 +50,4 @@ class LogProducer final : public ILogger { BoundedChannel& channel_; }; -#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_ diff --git a/tooling/pipeline/includes/web_client/http_web_client.h b/tooling/pipeline/includes/web_client/http_web_client.h index 883b56b..48306e5 100644 --- a/tooling/pipeline/includes/web_client/http_web_client.h +++ b/tooling/pipeline/includes/web_client/http_web_client.h @@ -52,4 +52,4 @@ class HttpWebClient final : public WebClient { std::shared_ptr logger_; }; -#endif +#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_ diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc index 7fa70e7..eae518a 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc @@ -14,10 +14,9 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator( std::unique_ptr exporter, std::unique_ptr curated_data_service, const ApplicationOptions& application_options) - : logger_(std::move(logger)), - context_service_(std::move(context_service)), - generator_(std::move(generator)), - exporter_(std::move(exporter)), - curated_data_service_(std::move(curated_data_service)), - application_options_(application_options) { -} \ No newline at end of file + : logger_(std::move(logger)), + context_service_(std::move(context_service)), + generator_(std::move(generator)), + exporter_(std::move(exporter)), + curated_data_service_(std::move(curated_data_service)), + application_options_(application_options) {} \ No newline at end of file diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc index 3592d86..7e67141 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc @@ -6,11 +6,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include "biergarten_pipeline_orchestrator.h" @@ -24,7 +26,7 @@ std::string Sanitize(std::string_view value) { for (const char character : value) { if (std::isalnum(static_cast(character)) != 0) { out.push_back(static_cast( - std::tolower(static_cast(character)))); + std::tolower(static_cast(character)))); } } return out; @@ -69,7 +71,37 @@ std::string GenerateDateOfBirth(std::mt19937& rng) { static_cast(birth_ymd.month()), static_cast(birth_ymd.day())); } -} // namespace + +std::optional SampleName( + const std::unordered_map& forenames_by_country, + const std::unordered_map& surnames_by_country, + const std::string& iso3166_1, std::mt19937& rng) { + const auto forenames_it = forenames_by_country.find(iso3166_1); + const auto surnames_it = surnames_by_country.find(iso3166_1); + + if (forenames_it == forenames_by_country.end() || + surnames_it == surnames_by_country.end() || + forenames_it->second.empty() || surnames_it->second.empty()) { + return std::nullopt; + } + + const forename_list& forenames = forenames_it->second; + const surname_list& surnames = surnames_it->second; + + std::uniform_int_distribution forename_dist(0, forenames.size() - 1); + std::uniform_int_distribution surname_dist(0, surnames.size() - 1); + + auto forename_it = forenames.begin(); + std::advance(forename_it, forename_dist(rng)); + + auto surname_it = surnames.begin(); + std::advance(surname_it, surname_dist(rng)); + + return Name{.first_name = forename_it->name, + .last_name = *surname_it, + .gender = forename_it->gender}; +} +} // namespace void BiergartenPipelineOrchestrator::GenerateUsers( std::span cities) { @@ -85,9 +117,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers( "No personas available in personas.json for user generation"); } - const NamesByCountry names_by_country = - curated_data_service_->LoadNamesByCountry( - "forenames-by-country.json", "surnames-by-country.json"); + const std::unordered_map forenames_by_country = + curated_data_service_->LoadForenamesByCountry( + "forenames-by-country.json"); + const std::unordered_map surnames_by_country = + curated_data_service_->LoadSurnamesByCountry("surnames-by-country.json"); std::mt19937 rng(std::random_device{}()); std::uniform_int_distribution persona_dist(0, personas.size() - 1); @@ -98,8 +132,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers( const auto generate_record = [this, &rng, &skipped_count]( - const EnrichedCity& city, const UserPersona& persona, - const Name& sampled_name) -> std::optional { + const EnrichedCity& city, const UserPersona& persona, + const Name& sampled_name) -> std::optional { try { std::unordered_set used_email_local_parts; @@ -125,25 +159,26 @@ void BiergartenPipelineOrchestrator::GenerateUsers( }; const auto export_record = [this, - &export_failed_count](const UserRecord& record) { + &export_failed_count](const UserRecord& record) { try { exporter_->ProcessRecord(record); } catch (const std::exception& export_exception) { ++export_failed_count; logger_->Log( - {.level = LogLevel::Warn, - .phase = PipelinePhase::UserGeneration, - .message = std::format("[Pipeline] Generated user for '{}' ({}) but " - "SQLite export failed: {}", - record.location.city, record.location.country, - export_exception.what())}); + {.level = LogLevel::Warn, + .phase = PipelinePhase::UserGeneration, + .message = std::format("[Pipeline] Generated user for '{}' ({}) but " + "SQLite export failed: {}", + record.location.city, record.location.country, + export_exception.what())}); } }; for (const auto& city : cities) { const std::optional sampled_name = - names_by_country.SampleName(city.location.iso3166_1, rng); + SampleName(forenames_by_country, surnames_by_country, + city.location.iso3166_1, rng); if (!sampled_name.has_value()) { ++skipped_count; @@ -171,11 +206,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers( if (skipped_count > 0) { logger_->Log( - {.level = LogLevel::Warn, - .phase = PipelinePhase::UserGeneration, - .message = std::format( - "[Pipeline] Skipped {} city/cities during user generation", - skipped_count)}); + {.level = LogLevel::Warn, + .phase = PipelinePhase::UserGeneration, + .message = std::format( + "[Pipeline] Skipped {} city/cities during user generation", + skipped_count)}); } if (export_failed_count > 0) { @@ -185,4 +220,4 @@ void BiergartenPipelineOrchestrator::GenerateUsers( "generated user/users to SQLite", export_failed_count)}); } -} \ No newline at end of file +} diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc index ca51601..9587ced 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc @@ -7,8 +7,8 @@ #include #include -#include "../../includes/json_handling/pretty_print.h" #include "biergarten_pipeline_orchestrator.h" +#include "json_handling/pretty_print.h" #include "services/logging/logger.h" void BiergartenPipelineOrchestrator::LogResults() const { diff --git a/tooling/pipeline/src/data_generation/llama/helpers.cc b/tooling/pipeline/src/data_generation/llama/helpers.cc index 17f6623..ffeabca 100644 --- a/tooling/pipeline/src/data_generation/llama/helpers.cc +++ b/tooling/pipeline/src/data_generation/llama/helpers.cc @@ -4,6 +4,8 @@ * parsing, token decoding, and JSON validation helpers for Llama modules. */ +#include + #include #include #include @@ -16,7 +18,7 @@ #include #include "data_generation/llama_generator_helpers.h" -#include "llama.h" + namespace { /** * String trimming: removes leading and trailing whitespace diff --git a/tooling/pipeline/src/data_generation/llama/infer.cc b/tooling/pipeline/src/data_generation/llama/infer.cc index fd9f2ee..4e12bde 100644 --- a/tooling/pipeline/src/data_generation/llama/infer.cc +++ b/tooling/pipeline/src/data_generation/llama/infer.cc @@ -5,6 +5,8 @@ * output tokens back to text for system+user chat prompts. */ +#include + #include #include #include @@ -16,7 +18,6 @@ #include "data_generation/llama_generator.h" #include "data_generation/llama_generator_helpers.h" -#include "llama.h" static constexpr size_t kPromptTokenSlack = 8; // Minimum tokens to keep when using top-p sampling. Ensures at least one diff --git a/tooling/pipeline/src/data_generation/llama/llama_generator.cc b/tooling/pipeline/src/data_generation/llama/llama_generator.cc index cc095dd..234514c 100644 --- a/tooling/pipeline/src/data_generation/llama/llama_generator.cc +++ b/tooling/pipeline/src/data_generation/llama/llama_generator.cc @@ -5,6 +5,8 @@ #include "data_generation/llama_generator.h" +#include + #include #include #include @@ -12,7 +14,6 @@ #include #include "data_model/models.h" -#include "llama.h" static constexpr uint32_t kMaxContextSize = 32768U; diff --git a/tooling/pipeline/src/data_generation/llama/load.cc b/tooling/pipeline/src/data_generation/llama/load.cc index 1866b2a..4b49ff0 100644 --- a/tooling/pipeline/src/data_generation/llama/load.cc +++ b/tooling/pipeline/src/data_generation/llama/load.cc @@ -4,6 +4,9 @@ * context, and resets prior resources during model initialization. */ +#include +#include + #include #include #include @@ -11,8 +14,6 @@ #include #include "data_generation/llama_generator.h" -#include "ggml-backend.h" -#include "llama.h" // Maximum batch size for decode operations. Capping the batch prevents // excessive memory allocation while maintaining inference performance. diff --git a/tooling/pipeline/src/data_model/names_by_country.cc b/tooling/pipeline/src/data_model/names_by_country.cc deleted file mode 100644 index 3a10bb6..0000000 --- a/tooling/pipeline/src/data_model/names_by_country.cc +++ /dev/null @@ -1,40 +0,0 @@ -/** - * @file data_model/names_by_country.cc - * @brief NamesByCountry::SampleName() implementation. - */ - -#include "data_model/names_by_country.h" - -#include - -NamesByCountry::NamesByCountry( - std::unordered_map> - forenames_by_country, - std::unordered_map> - surnames_by_country) - : forenames_by_country_(std::move(forenames_by_country)), - surnames_by_country_(std::move(surnames_by_country)) {} - -std::optional NamesByCountry::SampleName(const std::string& iso3166_1, - std::mt19937& rng) const { - const auto forenames_it = forenames_by_country_.find(iso3166_1); - const auto surnames_it = surnames_by_country_.find(iso3166_1); - - if (forenames_it == forenames_by_country_.end() || - surnames_it == surnames_by_country_.end() || - forenames_it->second.empty() || surnames_it->second.empty()) { - return std::nullopt; - } - - const std::vector& forenames = forenames_it->second; - const std::vector& surnames = surnames_it->second; - - std::uniform_int_distribution forename_dist(0, forenames.size() - 1); - std::uniform_int_distribution surname_dist(0, surnames.size() - 1); - - const ForenameEntry& forename = forenames[forename_dist(rng)]; - - return Name{.first_name = forename.name, - .last_name = surnames[surname_dist(rng)], - .gender = forename.gender}; -} diff --git a/tooling/pipeline/src/json_handling/json_loader.cc b/tooling/pipeline/src/json_handling/json_loader.cc index d279b57..6a6687c 100644 --- a/tooling/pipeline/src/json_handling/json_loader.cc +++ b/tooling/pipeline/src/json_handling/json_loader.cc @@ -137,7 +137,6 @@ std::vector JsonLoader::LoadLocations( .longitude = ReadRequiredNumber(object, "longitude"), }); } - return locations; } diff --git a/tooling/pipeline/src/main.cc b/tooling/pipeline/src/main.cc index eb06e36..57a520d 100644 --- a/tooling/pipeline/src/main.cc +++ b/tooling/pipeline/src/main.cc @@ -18,27 +18,7 @@ #include #include -#include "biergarten_pipeline_orchestrator.h" -#include "concurrency/bounded_channel.h" -#include "data_generation/llama_generator.h" -#include "data_generation/mock_generator.h" -#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h" -#include "data_model/models.h" -#include "json_handling/json_loader.h" -#include "llama_backend_state.h" -#include "services/curated_data/curated_data_service.h" -#include "services/database/export_service.h" -#include "services/database/sqlite_export_service.h" -#include "services/datetime/timer.h" -#include "services/enrichment/enrichment_service.h" -#include "services/enrichment/mock_enrichment.h" -#include "services/enrichment/wikipedia_service.h" -#include "services/logging/log_dispatcher.h" -#include "services/logging/log_entry.h" -#include "services/logging/log_producer.h" -#include "services/logging/logger.h" -#include "services/prompting/prompt_directory.h" -#include "web_client/http_web_client.h" +#include "biergarten_pipeline.h" namespace di = boost::di; @@ -49,7 +29,7 @@ int main(const int argc, char** argv) { spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v"); BoundedChannel log_channel(kLogMaxCount); - auto log_dispatcher = // + auto log_dispatcher = // std::make_unique(log_channel); std::shared_ptr log_producer = std::make_shared(log_channel); @@ -111,17 +91,17 @@ int main(const int argc, char** argv) { if (options.generator.use_mocked) { { log_producer->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = "Prompt formatter: none (mock mode)"}); + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = "Prompt formatter: none (mock mode)"}); } return std::unique_ptr(nullptr); } { log_producer->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = "Prompt formatter: Gemma4JinjaPromptFormatter"}); + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = "Prompt formatter: Gemma4JinjaPromptFormatter"}); } return std::unique_ptr( std::make_unique()); @@ -145,7 +125,7 @@ int main(const int argc, char** argv) { }), di::bind().to( [options, &log_producer]( - const auto& inj) -> std::unique_ptr { + const auto& inj) -> std::unique_ptr { if (options.generator.use_mocked) { log_producer->Log({.level = LogLevel::Info, .phase = PipelinePhase::Startup, @@ -164,8 +144,7 @@ int main(const int argc, char** argv) { }), di::bind().to( [&options, &model_path, &sampling, &prompt_directory, - &log_producer - ](const auto& inj) -> std::unique_ptr { + &log_producer](const auto& inj) -> std::unique_ptr { if (options.generator.use_mocked) { log_producer->Log({.level = LogLevel::Info, .phase = PipelinePhase::Startup, @@ -175,13 +154,13 @@ int main(const int argc, char** argv) { } log_producer->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = std::format( - "Generator: LlamaGenerator | model={} | " - "temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}", - model_path, sampling.temperature, sampling.top_p, - sampling.top_k, sampling.n_ctx, sampling.seed)}); + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = std::format( + "Generator: LlamaGenerator | model={} | " + "temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}", + model_path, sampling.temperature, sampling.top_p, + sampling.top_k, sampling.n_ctx, sampling.seed)}); return std::make_unique( options, model_path, log_producer,