AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 09:37:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update user generation

Browse Source
This commit is contained in:
Aaron Po
2026-06-28 13:07:14 -04:00
parent 98f97b8e2c
commit 6d8904cf01
28 changed files with 252 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tooling/pipeline/CMakeLists.txt
View File

@@ -151,11 +151,6 @@ target_sources(${PROJECT_NAME} PRIVATE
src/json_handling/json_loader.cc src/json_handling/json_loader.cc
) )
# --- data_model ---
target_sources(${PROJECT_NAME} PRIVATE
src/data_model/names_by_country.cc
)
# --- application_options --- # --- application_options ---
target_sources(${PROJECT_NAME} PRIVATE target_sources(${PROJECT_NAME} PRIVATE
src/application_options/parse_arguments.cc src/application_options/parse_arguments.cc

69
tooling/pipeline/includes/biergarten_pipeline.h Normal file
View File

@@ -0,0 +1,69 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
/**
* @file biergarten_pipeline.h
* @brief Master umbrella header — includes every public header in the
* Biergarten pipeline.
*/
// --- orchestrator ---
#include "biergarten_pipeline_orchestrator.h"
// --- concurrency ---
#include "concurrency/bounded_channel.h"
// --- data_generation ---
#include "data_generation/data_generator.h"
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
// --- data_model ---
#include "data_model/generated_models.h"
#include "data_model/models.h"
// --- json_handling ---
#include "json_handling/json_loader.h"
#include "json_handling/pretty_print.h"
// --- llama backend ---
#include "llama_backend_state.h"
// --- services: curated_data ---
#include "services/curated_data/curated_data_service.h"
// --- services: database ---
#include "services/database/export_service.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
// --- services: datetime ---
#include "services/datetime/date_time_provider.h"
#include "services/datetime/timer.h"
// --- services: enrichment ---
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
// --- services: logging ---
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
// --- services: prompting ---
#include "services/prompting/prompt_directory.h"
// --- web_client ---
#include "web_client/http_web_client.h"
#include "web_client/web_client.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_

13
tooling/pipeline/includes/biergarten_pipeline_orchestrator.h
View File

@@ -1,12 +1,9 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ #ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ #define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
/** /**
* @file biergarten_pipeline_orchestrator.h * @file biergarten_pipeline_orchestrator.h
* @brief Orchestration for end-to-end brewery and user data generation. * @brief Orchestration for end-to-end brewery and user data generation.
*
* Coordinates location loading, enrichment, and generation phases to produce
* a complete dataset, wiring dependencies selected at the composition root.
*/ */
#include <memory> #include <memory>
@@ -103,10 +100,6 @@ class BiergartenPipelineOrchestrator {
* @brief Generate users grounded in sampled names and personas for * @brief Generate users grounded in sampled names and personas for
* enriched cities. * enriched cities.
* *
* Loads personas.json / forenames-by-country.json /
* surnames-by-country.json itself, mirroring how QueryCitiesWithCountries()
* owns its own locations.json load.
*
* @param cities Span of enriched city data. * @param cities Span of enriched city data.
*/ */
void GenerateUsers(std::span<const EnrichedCity> cities); void GenerateUsers(std::span<const EnrichedCity> cities);
@@ -120,4 +113,4 @@ class BiergartenPipelineOrchestrator {
std::vector<UserRecord> generated_users_; std::vector<UserRecord> generated_users_;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_

2
tooling/pipeline/includes/concurrency/bounded_channel.tcc
View File

@@ -1,4 +1,4 @@
#include "bounded_channel.h" #include "concurrency/bounded_channel.h"
template <typename T> template <typename T>
void BoundedChannel<T>::Send(T item) { void BoundedChannel<T>::Send(T item) {

5
tooling/pipeline/includes/data_generation/llama_generator.h
View File

@@ -1,24 +1,23 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_ #ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_ #define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#include <filesystem>
/** /**
* @file data_generation/llama_generator.h * @file data_generation/llama_generator.h
* @brief llama.cpp-backed implementation of DataGenerator. * @brief llama.cpp-backed implementation of DataGenerator.
*/ */
#include <cstdint> #include <cstdint>
#include <filesystem>
#include <memory> #include <memory>
#include <random> #include <random>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include "../services/prompting/prompt_directory.h"
#include "data_generation/data_generator.h" #include "data_generation/data_generator.h"
#include "data_generation/prompt_formatting/prompt_formatter.h" #include "data_generation/prompt_formatting/prompt_formatter.h"
#include "data_model/models.h" #include "data_model/models.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
struct llama_model; struct llama_model;
struct llama_context; struct llama_context;

16
tooling/pipeline/includes/data_model/models.h
View File

@@ -10,6 +10,7 @@
#include <boost/program_options.hpp> #include <boost/program_options.hpp>
#include <cstdint> #include <cstdint>
#include <filesystem> #include <filesystem>
#include <functional>
#include <memory> #include <memory>
#include <optional> #include <optional>
#include <string> #include <string>
@@ -93,8 +94,23 @@ struct ForenameEntry {
* dataset (e.g. "M", "F"). * dataset (e.g. "M", "F").
*/ */
std::string gender{}; std::string gender{};
bool operator==(const ForenameEntry& other) const {
return name == other.name && gender == other.gender;
}
}; };
namespace std {
template <>
struct hash<ForenameEntry> {
size_t operator()(const ForenameEntry& entry) const noexcept {
const size_t name_hash = std::hash<std::string>{}(entry.name);
const size_t gender_hash = std::hash<std::string>{}(entry.gender);
return name_hash ^ (gender_hash << 1);
}
};
} // namespace std
/** /**
* @brief A persona archetype used to ground LLM-generated user bios. * @brief A persona archetype used to ground LLM-generated user bios.
*/ */

51
tooling/pipeline/includes/data_model/names_by_country.h
View File

@@ -1,51 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_
/**
* @file data_model/names_by_country.h
* @brief Sampling over the names-by-country fixture data.
*/
#include <optional>
#include <random>
#include <string>
#include <unordered_map>
#include <vector>
#include "data_model/models.h"
/**
* @brief Samples locale-appropriate names from curated forename/surname
* fixture data, keyed by ISO 3166-1 country code.
*
* Forenames and surnames are kept in separate maps (mirroring the source
* dataset's own shape) so per-forename gender is preserved; pairing happens
* at sample time rather than being precomputed, so no information from the
* source dataset is discarded up front.
*/
class NamesByCountry {
public:
NamesByCountry(std::unordered_map<std::string, std::vector<ForenameEntry>>
forenames_by_country,
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country);
/**
* @brief Samples a first/last name pair for the given country.
*
* @param iso3166_1 ISO 3166-1 country code to sample for.
* @param rng Random source used for sampling.
* @return A sampled Name, or std::nullopt if the country has no forename
* or no surname entries.
*/
[[nodiscard]] std::optional<Name> SampleName(const std::string& iso3166_1,
std::mt19937& rng) const;
private:
std::unordered_map<std::string, std::vector<ForenameEntry>>
forenames_by_country_;
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_

34
tooling/pipeline/includes/json_handling/json_loader.h
View File

@@ -7,41 +7,49 @@
*/ */
#include <filesystem> #include <filesystem>
#include <unordered_set>
#include <vector> #include <vector>
#include "data_model/models.h" #include "data_model/models.h"
#include "data_model/names_by_country.h"
#include "services/curated_data/curated_data_service.h" #include "services/curated_data/curated_data_service.h"
/** /**
* @brief Loads curated location, persona, and name data from JSON files. * @brief Loads curated location, persona, and name data from JSON files.
*/ */
class JsonLoader final : public ICuratedDataService { class JsonLoader final : public ICuratedDataService {
struct cache {
std::vector<Location> locations;
std::vector<UserPersona> personas;
std::unordered_map<std::string, forename_list> forenames_by_country;
std::unordered_map<std::string, surname_list> surnames_by_country;
cache() = default;
~cache() = default;
};
cache cache_;
public: public:
JsonLoader() = default; JsonLoader() = default;
/** /**
* @brief Parses a JSON array file and returns all location records. * @brief Parses a JSON array file and returns all location records.
*/ */
std::vector<Location> LoadLocations( std::vector<Location> LoadLocations(const std::filesystem::path&) override;
const std::filesystem::path& filepath) override;
/** /**
* @brief Parses a JSON array file and returns all persona records. * @brief Parses a JSON array file and returns all persona records.
*/ */
std::vector<UserPersona> LoadPersonas( std::vector<UserPersona> LoadPersonas(const std::filesystem::path&) override;
const std::filesystem::path& filepath) override;
std::unordered_map<std::string, forename_list> LoadForenamesByCountry(
const std::filesystem::path&) override;
/** /**
* @brief Parses the names-by-country fixture pair into a sampling-capable * @brief Parses a JSON file and returns all the forenames per country.
* NamesByCountry.
*
* @param forenames_filepath Path to forenames-by-country.json.
* @param surnames_filepath Path to surnames-by-country.json.
*/ */
NamesByCountry LoadNamesByCountry( std::unordered_map<std::string, surname_list> LoadSurnamesByCountry(
const std::filesystem::path& forenames_filepath, const std::filesystem::path&) override;
const std::filesystem::path& surnames_filepath) override;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

2
tooling/pipeline/includes/json_handling/pretty_print.h
View File

@@ -106,4 +106,4 @@ inline void PrettyPrint(std::ostream& outstream,
} }
} }
#endif #endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_

16
tooling/pipeline/includes/services/curated_data/curated_data_service.h
View File

@@ -7,11 +7,13 @@
*/ */
#include <filesystem> #include <filesystem>
#include <unordered_set>
#include <vector> #include <vector>
#include "data_model/models.h" #include "data_model/models.h"
#include "data_model/names_by_country.h"
using forename_list = std::unordered_set<ForenameEntry>;
using surname_list = std::unordered_set<std::string>;
/** /**
* @brief Interface for services that load curated data used to seed * @brief Interface for services that load curated data used to seed
* brewery/user generation. * brewery/user generation.
@@ -38,15 +40,9 @@ class ICuratedDataService {
virtual std::vector<UserPersona> LoadPersonas( virtual std::vector<UserPersona> LoadPersonas(
const std::filesystem::path& filepath) = 0; const std::filesystem::path& filepath) = 0;
/** virtual std::unordered_map<std::string, forename_list> LoadForenamesByCountry(
* @brief Loads the names-by-country fixture pair into a sampling-capable const std::filesystem::path& forenames_filepath) = 0;
* NamesByCountry. virtual std::unordered_map<std::string, surname_list> LoadSurnamesByCountry(
*
* @param forenames_filepath Path to forenames-by-country.json.
* @param surnames_filepath Path to surnames-by-country.json.
*/
virtual NamesByCountry LoadNamesByCountry(
const std::filesystem::path& forenames_filepath,
const std::filesystem::path& surnames_filepath) = 0; const std::filesystem::path& surnames_filepath) = 0;
}; };

2
tooling/pipeline/includes/services/database/sqlite_connection_helpers.h
View File

@@ -12,7 +12,7 @@
#include <string> #include <string>
#include <string_view> #include <string_view>
#include "sqlite_handle_types.h" #include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {

6
tooling/pipeline/includes/services/database/sqlite_export_service.h
View File

@@ -11,10 +11,10 @@
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include "../datetime/date_time_provider.h"
#include "data_model/models.h" #include "data_model/models.h"
#include "export_service.h" #include "services/database/export_service.h"
#include "sqlite_export_service_helpers.h" #include "services/database/sqlite_export_service_helpers.h"
#include "services/datetime/date_time_provider.h"
/** /**
* @brief Persists generated brewery records into a fresh SQLite database. * @brief Persists generated brewery records into a fresh SQLite database.

6
tooling/pipeline/includes/services/database/sqlite_export_service_helpers.h
View File

@@ -3,8 +3,8 @@
/* Umbrella header for backward compatibility. */ /* Umbrella header for backward compatibility. */
#include "sqlite_connection_helpers.h" #include "services/database/sqlite_connection_helpers.h"
#include "sqlite_handle_types.h" #include "services/database/sqlite_handle_types.h"
#include "sqlite_statement_helpers.h" #include "services/database/sqlite_statement_helpers.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_

52
tooling/pipeline/includes/services/database/sqlite_statement_helpers.h
View File

@@ -13,7 +13,7 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include "sqlite_handle_types.h" #include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {
@@ -109,30 +109,36 @@ INSERT INTO users (
// sqlite3_bind_*() parameter indices are 1-based, matching the "?" // sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above. // placeholder order in the SQL above.
inline constexpr int kLocationCityBindIndex = 1; enum eLocationBindIndex {
inline constexpr int kLocationStateProvinceBindIndex = 2; kLocationCityBindIndex = 1,
inline constexpr int kLocationIso31662BindIndex = 3; kLocationStateProvinceBindIndex,
inline constexpr int kLocationCountryBindIndex = 4; kLocationIso31662BindIndex,
inline constexpr int kLocationIso31661BindIndex = 5; kLocationCountryBindIndex,
inline constexpr int kLocationLanguagesBindIndex = 6; kLocationIso31661BindIndex,
inline constexpr int kLocationLatitudeBindIndex = 7; kLocationLanguagesBindIndex,
inline constexpr int kLocationLongitudeBindIndex = 8; kLocationLatitudeBindIndex,
kLocationLongitudeBindIndex,
};
inline constexpr int kBreweryLocationIdBindIndex = 1; enum BreweryBindIndex {
inline constexpr int kBreweryEnglishNameBindIndex = 2; kBreweryLocationIdBindIndex = 1,
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3; kBreweryEnglishNameBindIndex,
inline constexpr int kBreweryLocalNameBindIndex = 4; kBreweryEnglishDescriptionBindIndex,
inline constexpr int kBreweryLocalDescriptionBindIndex = 5; kBreweryLocalNameBindIndex,
kBreweryLocalDescriptionBindIndex,
};
inline constexpr int kUserLocationIdBindIndex = 1; enum UserBindIndex {
inline constexpr int kUserFirstNameBindIndex = 2; kUserLocationIdBindIndex = 1,
inline constexpr int kUserLastNameBindIndex = 3; kUserFirstNameBindIndex,
inline constexpr int kUserGenderBindIndex = 4; kUserLastNameBindIndex,
inline constexpr int kUserUsernameBindIndex = 5; kUserGenderBindIndex,
inline constexpr int kUserBioBindIndex = 6; kUserUsernameBindIndex,
inline constexpr int kUserActivityWeightBindIndex = 7; kUserBioBindIndex,
inline constexpr int kUserEmailBindIndex = 8; kUserActivityWeightBindIndex,
inline constexpr int kUserDateOfBirthBindIndex = 9; kUserEmailBindIndex,
kUserDateOfBirthBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql, std::string_view sql,

2
tooling/pipeline/includes/services/enrichment/mock_enrichment.h
View File

@@ -8,7 +8,7 @@
#include <string> #include <string>
#include "enrichment_service.h" #include "services/enrichment/enrichment_service.h"
/** /**
* @brief Enrichment service that returns no context for any location. * @brief Enrichment service that returns no context for any location.

2
tooling/pipeline/includes/services/enrichment/wikipedia_service.h
View File

@@ -11,7 +11,7 @@
#include <string_view> #include <string_view>
#include <unordered_map> #include <unordered_map>
#include "enrichment_service.h" #include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
#include "web_client/web_client.h" #include "web_client/web_client.h"

6
tooling/pipeline/includes/services/logging/log_producer.h
View File

@@ -6,8 +6,8 @@
* a bounded channel for later processing by the dispatcher. * a bounded channel for later processing by the dispatcher.
*/ */
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ #ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ #define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#include <string_view> #include <string_view>
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
BoundedChannel<LogEntry>& channel_; BoundedChannel<LogEntry>& channel_;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_

2
tooling/pipeline/includes/web_client/http_web_client.h
View File

@@ -52,4 +52,4 @@ class HttpWebClient final : public WebClient {
std::shared_ptr<ILogger> logger_; std::shared_ptr<ILogger> logger_;
}; };
#endif #endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_

13
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -14,10 +14,9 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
const ApplicationOptions& application_options) const ApplicationOptions& application_options)
: logger_(std::move(logger)), : logger_(std::move(logger)),
context_service_(std::move(context_service)), context_service_(std::move(context_service)),
generator_(std::move(generator)), generator_(std::move(generator)),
exporter_(std::move(exporter)), exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)), curated_data_service_(std::move(curated_data_service)),
application_options_(application_options) { application_options_(application_options) {}
}

75
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -6,11 +6,13 @@
#include <cctype> #include <cctype>
#include <chrono> #include <chrono>
#include <format> #include <format>
#include <iterator>
#include <optional> #include <optional>
#include <random> #include <random>
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include "biergarten_pipeline_orchestrator.h" #include "biergarten_pipeline_orchestrator.h"
@@ -24,7 +26,7 @@ std::string Sanitize(std::string_view value) {
for (const char character : value) { for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) { if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>( out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character)))); std::tolower(static_cast<unsigned char>(character))));
} }
} }
return out; return out;
@@ -69,7 +71,37 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
static_cast<unsigned>(birth_ymd.month()), static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day())); static_cast<unsigned>(birth_ymd.day()));
} }
} // namespace
std::optional<Name> SampleName(
const std::unordered_map<std::string, forename_list>& forenames_by_country,
const std::unordered_map<std::string, surname_list>& surnames_by_country,
const std::string& iso3166_1, std::mt19937& rng) {
const auto forenames_it = forenames_by_country.find(iso3166_1);
const auto surnames_it = surnames_by_country.find(iso3166_1);
if (forenames_it == forenames_by_country.end() ||
surnames_it == surnames_by_country.end() ||
forenames_it->second.empty() || surnames_it->second.empty()) {
return std::nullopt;
}
const forename_list& forenames = forenames_it->second;
const surname_list& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
auto forename_it = forenames.begin();
std::advance(forename_it, forename_dist(rng));
auto surname_it = surnames.begin();
std::advance(surname_it, surname_dist(rng));
return Name{.first_name = forename_it->name,
.last_name = *surname_it,
.gender = forename_it->gender};
}
} // namespace
void BiergartenPipelineOrchestrator::GenerateUsers( void BiergartenPipelineOrchestrator::GenerateUsers(
std::span<const EnrichedCity> cities) { std::span<const EnrichedCity> cities) {
@@ -85,9 +117,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
"No personas available in personas.json for user generation"); "No personas available in personas.json for user generation");
} }
const NamesByCountry names_by_country = const std::unordered_map<std::string, forename_list> forenames_by_country =
curated_data_service_->LoadNamesByCountry( curated_data_service_->LoadForenamesByCountry(
"forenames-by-country.json", "surnames-by-country.json"); "forenames-by-country.json");
const std::unordered_map<std::string, surname_list> surnames_by_country =
curated_data_service_->LoadSurnamesByCountry("surnames-by-country.json");
std::mt19937 rng(std::random_device{}()); std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1); std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
@@ -98,8 +132,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
const auto generate_record = const auto generate_record =
[this, &rng, &skipped_count]( [this, &rng, &skipped_count](
const EnrichedCity& city, const UserPersona& persona, const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> { const Name& sampled_name) -> std::optional<UserRecord> {
try { try {
std::unordered_set<std::string> used_email_local_parts; std::unordered_set<std::string> used_email_local_parts;
@@ -125,25 +159,26 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
}; };
const auto export_record = [this, const auto export_record = [this,
&export_failed_count](const UserRecord& record) { &export_failed_count](const UserRecord& record) {
try { try {
exporter_->ProcessRecord(record); exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) { } catch (const std::exception& export_exception) {
++export_failed_count; ++export_failed_count;
logger_->Log( logger_->Log(
{.level = LogLevel::Warn, {.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration, .phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but " .message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}", "SQLite export failed: {}",
record.location.city, record.location.country, record.location.city, record.location.country,
export_exception.what())}); export_exception.what())});
} }
}; };
for (const auto& city : cities) { for (const auto& city : cities) {
const std::optional<Name> sampled_name = const std::optional<Name> sampled_name =
names_by_country.SampleName(city.location.iso3166_1, rng); SampleName(forenames_by_country, surnames_by_country,
city.location.iso3166_1, rng);
if (!sampled_name.has_value()) { if (!sampled_name.has_value()) {
++skipped_count; ++skipped_count;
@@ -171,11 +206,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
if (skipped_count > 0) { if (skipped_count > 0) {
logger_->Log( logger_->Log(
{.level = LogLevel::Warn, {.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration, .phase = PipelinePhase::UserGeneration,
.message = std::format( .message = std::format(
"[Pipeline] Skipped {} city/cities during user generation", "[Pipeline] Skipped {} city/cities during user generation",
skipped_count)}); skipped_count)});
} }
if (export_failed_count > 0) { if (export_failed_count > 0) {

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/log_results.cc
View File

@@ -7,8 +7,8 @@
#include <chrono> #include <chrono>
#include <format> #include <format>
#include "../../includes/json_handling/pretty_print.h"
#include "biergarten_pipeline_orchestrator.h" #include "biergarten_pipeline_orchestrator.h"
#include "json_handling/pretty_print.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
void BiergartenPipelineOrchestrator::LogResults() const { void BiergartenPipelineOrchestrator::LogResults() const {

4
tooling/pipeline/src/data_generation/llama/helpers.cc
View File

@@ -4,6 +4,8 @@
* parsing, token decoding, and JSON validation helpers for Llama modules. * parsing, token decoding, and JSON validation helpers for Llama modules.
*/ */
#include <llama.h>
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <boost/json.hpp> #include <boost/json.hpp>
@@ -16,7 +18,7 @@
#include <vector> #include <vector>
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
#include "llama.h"
namespace { namespace {
/** /**
* String trimming: removes leading and trailing whitespace * String trimming: removes leading and trailing whitespace

3
tooling/pipeline/src/data_generation/llama/infer.cc
View File

@@ -5,6 +5,8 @@
* output tokens back to text for system+user chat prompts. * output tokens back to text for system+user chat prompts.
*/ */
#include <llama.h>
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
#include <format> #include <format>
@@ -16,7 +18,6 @@
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h" #include "data_generation/llama_generator_helpers.h"
#include "llama.h"
static constexpr size_t kPromptTokenSlack = 8; static constexpr size_t kPromptTokenSlack = 8;
// Minimum tokens to keep when using top-p sampling. Ensures at least one // Minimum tokens to keep when using top-p sampling. Ensures at least one

3
tooling/pipeline/src/data_generation/llama/llama_generator.cc
View File

@@ -5,6 +5,8 @@
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include <llama.h>
#include <filesystem> #include <filesystem>
#include <memory> #include <memory>
#include <random> #include <random>
@@ -12,7 +14,6 @@
#include <string> #include <string>
#include "data_model/models.h" #include "data_model/models.h"
#include "llama.h"
static constexpr uint32_t kMaxContextSize = 32768U; static constexpr uint32_t kMaxContextSize = 32768U;

5
tooling/pipeline/src/data_generation/llama/load.cc
View File

@@ -4,6 +4,9 @@
* context, and resets prior resources during model initialization. * context, and resets prior resources during model initialization.
*/ */
#include <ggml-backend.h>
#include <llama.h>
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
#include <stdexcept> #include <stdexcept>
@@ -11,8 +14,6 @@
#include <utility> #include <utility>
#include "data_generation/llama_generator.h" #include "data_generation/llama_generator.h"
#include "ggml-backend.h"
#include "llama.h"
// Maximum batch size for decode operations. Capping the batch prevents // Maximum batch size for decode operations. Capping the batch prevents
// excessive memory allocation while maintaining inference performance. // excessive memory allocation while maintaining inference performance.

40
tooling/pipeline/src/data_model/names_by_country.cc
View File

@@ -1,40 +0,0 @@
/**
* @file data_model/names_by_country.cc
* @brief NamesByCountry::SampleName() implementation.
*/
#include "data_model/names_by_country.h"
#include <utility>
NamesByCountry::NamesByCountry(
std::unordered_map<std::string, std::vector<ForenameEntry>>
forenames_by_country,
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country)
: forenames_by_country_(std::move(forenames_by_country)),
surnames_by_country_(std::move(surnames_by_country)) {}
std::optional<Name> NamesByCountry::SampleName(const std::string& iso3166_1,
std::mt19937& rng) const {
const auto forenames_it = forenames_by_country_.find(iso3166_1);
const auto surnames_it = surnames_by_country_.find(iso3166_1);
if (forenames_it == forenames_by_country_.end() ||
surnames_it == surnames_by_country_.end() ||
forenames_it->second.empty() || surnames_it->second.empty()) {
return std::nullopt;
}
const std::vector<ForenameEntry>& forenames = forenames_it->second;
const std::vector<std::string>& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
const ForenameEntry& forename = forenames[forename_dist(rng)];
return Name{.first_name = forename.name,
.last_name = surnames[surname_dist(rng)],
.gender = forename.gender};
}

1
tooling/pipeline/src/json_handling/json_loader.cc
View File

@@ -137,7 +137,6 @@ std::vector<Location> JsonLoader::LoadLocations(
.longitude = ReadRequiredNumber(object, "longitude"), .longitude = ReadRequiredNumber(object, "longitude"),
}); });
} }
return locations; return locations;
} }

55
tooling/pipeline/src/main.cc
View File

@@ -18,27 +18,7 @@
#include <string> #include <string>
#include <thread> #include <thread>
#include "biergarten_pipeline_orchestrator.h" #include "biergarten_pipeline.h"
#include "concurrency/bounded_channel.h"
#include "data_generation/llama_generator.h"
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_model/models.h"
#include "json_handling/json_loader.h"
#include "llama_backend_state.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service.h"
#include "services/datetime/timer.h"
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
#include "web_client/http_web_client.h"
namespace di = boost::di; namespace di = boost::di;
@@ -49,7 +29,7 @@ int main(const int argc, char** argv) {
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v"); spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount); BoundedChannel<LogEntry> log_channel(kLogMaxCount);
auto log_dispatcher = // auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel); std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer = std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel); std::make_shared<LogProducer>(log_channel);
@@ -111,17 +91,17 @@ int main(const int argc, char** argv) {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {
{ {
log_producer->Log( log_producer->Log(
{.level = LogLevel::Info, {.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"}); .message = "Prompt formatter: none (mock mode)"});
} }
return std::unique_ptr<IPromptFormatter>(nullptr); return std::unique_ptr<IPromptFormatter>(nullptr);
} }
{ {
log_producer->Log( log_producer->Log(
{.level = LogLevel::Info, {.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"}); .message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
} }
return std::unique_ptr<IPromptFormatter>( return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>()); std::make_unique<Gemma4JinjaPromptFormatter>());
@@ -145,7 +125,7 @@ int main(const int argc, char** argv) {
}), }),
di::bind<IEnrichmentService>().to( di::bind<IEnrichmentService>().to(
[options, &log_producer]( [options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> { const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info, log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
@@ -164,8 +144,7 @@ int main(const int argc, char** argv) {
}), }),
di::bind<DataGenerator>().to( di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory, [&options, &model_path, &sampling, &prompt_directory,
&log_producer &log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info, log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
@@ -175,13 +154,13 @@ int main(const int argc, char** argv) {
} }
log_producer->Log( log_producer->Log(
{.level = LogLevel::Info, {.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = std::format( .message = std::format(
"Generator: LlamaGenerator | model={} | " "Generator: LlamaGenerator | model={} | "
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}", "temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p, model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)}); sampling.top_k, sampling.n_ctx, sampling.seed)});
return std::make_unique<LlamaGenerator>( return std::make_unique<LlamaGenerator>(
options, model_path, log_producer, options, model_path, log_producer,