Update user generation

This commit is contained in:
Aaron Po
2026-06-28 13:07:14 -04:00
parent 98f97b8e2c
commit 6d8904cf01
28 changed files with 252 additions and 244 deletions

View File

@@ -151,11 +151,6 @@ target_sources(${PROJECT_NAME} PRIVATE
src/json_handling/json_loader.cc
)
# --- data_model ---
target_sources(${PROJECT_NAME} PRIVATE
src/data_model/names_by_country.cc
)
# --- application_options ---
target_sources(${PROJECT_NAME} PRIVATE
src/application_options/parse_arguments.cc

View File

@@ -0,0 +1,69 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
/**
* @file biergarten_pipeline.h
* @brief Master umbrella header — includes every public header in the
* Biergarten pipeline.
*/
// --- orchestrator ---
#include "biergarten_pipeline_orchestrator.h"
// --- concurrency ---
#include "concurrency/bounded_channel.h"
// --- data_generation ---
#include "data_generation/data_generator.h"
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
// --- data_model ---
#include "data_model/generated_models.h"
#include "data_model/models.h"
// --- json_handling ---
#include "json_handling/json_loader.h"
#include "json_handling/pretty_print.h"
// --- llama backend ---
#include "llama_backend_state.h"
// --- services: curated_data ---
#include "services/curated_data/curated_data_service.h"
// --- services: database ---
#include "services/database/export_service.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
// --- services: datetime ---
#include "services/datetime/date_time_provider.h"
#include "services/datetime/timer.h"
// --- services: enrichment ---
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
// --- services: logging ---
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
// --- services: prompting ---
#include "services/prompting/prompt_directory.h"
// --- web_client ---
#include "web_client/http_web_client.h"
#include "web_client/web_client.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_

View File

@@ -1,12 +1,9 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
/**
* @file biergarten_pipeline_orchestrator.h
* @brief Orchestration for end-to-end brewery and user data generation.
*
* Coordinates location loading, enrichment, and generation phases to produce
* a complete dataset, wiring dependencies selected at the composition root.
*/
#include <memory>
@@ -103,10 +100,6 @@ class BiergartenPipelineOrchestrator {
* @brief Generate users grounded in sampled names and personas for
* enriched cities.
*
* Loads personas.json / forenames-by-country.json /
* surnames-by-country.json itself, mirroring how QueryCitiesWithCountries()
* owns its own locations.json load.
*
* @param cities Span of enriched city data.
*/
void GenerateUsers(std::span<const EnrichedCity> cities);
@@ -120,4 +113,4 @@ class BiergartenPipelineOrchestrator {
std::vector<UserRecord> generated_users_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_

View File

@@ -1,4 +1,4 @@
#include "bounded_channel.h"
#include "concurrency/bounded_channel.h"
template <typename T>
void BoundedChannel<T>::Send(T item) {

View File

@@ -1,24 +1,23 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#include <filesystem>
/**
* @file data_generation/llama_generator.h
* @brief llama.cpp-backed implementation of DataGenerator.
*/
#include <cstdint>
#include <filesystem>
#include <memory>
#include <random>
#include <string>
#include <string_view>
#include "../services/prompting/prompt_directory.h"
#include "data_generation/data_generator.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
#include "data_model/models.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
struct llama_model;
struct llama_context;

View File

@@ -10,6 +10,7 @@
#include <boost/program_options.hpp>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <memory>
#include <optional>
#include <string>
@@ -93,8 +94,23 @@ struct ForenameEntry {
* dataset (e.g. "M", "F").
*/
std::string gender{};
bool operator==(const ForenameEntry& other) const {
return name == other.name && gender == other.gender;
}
};
namespace std {
template <>
struct hash<ForenameEntry> {
size_t operator()(const ForenameEntry& entry) const noexcept {
const size_t name_hash = std::hash<std::string>{}(entry.name);
const size_t gender_hash = std::hash<std::string>{}(entry.gender);
return name_hash ^ (gender_hash << 1);
}
};
} // namespace std
/**
* @brief A persona archetype used to ground LLM-generated user bios.
*/

View File

@@ -1,51 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_
/**
* @file data_model/names_by_country.h
* @brief Sampling over the names-by-country fixture data.
*/
#include <optional>
#include <random>
#include <string>
#include <unordered_map>
#include <vector>
#include "data_model/models.h"
/**
* @brief Samples locale-appropriate names from curated forename/surname
* fixture data, keyed by ISO 3166-1 country code.
*
* Forenames and surnames are kept in separate maps (mirroring the source
* dataset's own shape) so per-forename gender is preserved; pairing happens
* at sample time rather than being precomputed, so no information from the
* source dataset is discarded up front.
*/
class NamesByCountry {
public:
NamesByCountry(std::unordered_map<std::string, std::vector<ForenameEntry>>
forenames_by_country,
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country);
/**
* @brief Samples a first/last name pair for the given country.
*
* @param iso3166_1 ISO 3166-1 country code to sample for.
* @param rng Random source used for sampling.
* @return A sampled Name, or std::nullopt if the country has no forename
* or no surname entries.
*/
[[nodiscard]] std::optional<Name> SampleName(const std::string& iso3166_1,
std::mt19937& rng) const;
private:
std::unordered_map<std::string, std::vector<ForenameEntry>>
forenames_by_country_;
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_NAMES_BY_COUNTRY_H_

View File

@@ -7,41 +7,49 @@
*/
#include <filesystem>
#include <unordered_set>
#include <vector>
#include "data_model/models.h"
#include "data_model/names_by_country.h"
#include "services/curated_data/curated_data_service.h"
/**
* @brief Loads curated location, persona, and name data from JSON files.
*/
class JsonLoader final : public ICuratedDataService {
struct cache {
std::vector<Location> locations;
std::vector<UserPersona> personas;
std::unordered_map<std::string, forename_list> forenames_by_country;
std::unordered_map<std::string, surname_list> surnames_by_country;
cache() = default;
~cache() = default;
};
cache cache_;
public:
JsonLoader() = default;
/**
* @brief Parses a JSON array file and returns all location records.
*/
std::vector<Location> LoadLocations(
const std::filesystem::path& filepath) override;
std::vector<Location> LoadLocations(const std::filesystem::path&) override;
/**
* @brief Parses a JSON array file and returns all persona records.
*/
std::vector<UserPersona> LoadPersonas(
const std::filesystem::path& filepath) override;
std::vector<UserPersona> LoadPersonas(const std::filesystem::path&) override;
std::unordered_map<std::string, forename_list> LoadForenamesByCountry(
const std::filesystem::path&) override;
/**
* @brief Parses the names-by-country fixture pair into a sampling-capable
* NamesByCountry.
*
* @param forenames_filepath Path to forenames-by-country.json.
* @param surnames_filepath Path to surnames-by-country.json.
* @brief Parses a JSON file and returns all the forenames per country.
*/
NamesByCountry LoadNamesByCountry(
const std::filesystem::path& forenames_filepath,
const std::filesystem::path& surnames_filepath) override;
std::unordered_map<std::string, surname_list> LoadSurnamesByCountry(
const std::filesystem::path&) override;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

View File

@@ -106,4 +106,4 @@ inline void PrettyPrint(std::ostream& outstream,
}
}
#endif
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_

View File

@@ -7,11 +7,13 @@
*/
#include <filesystem>
#include <unordered_set>
#include <vector>
#include "data_model/models.h"
#include "data_model/names_by_country.h"
using forename_list = std::unordered_set<ForenameEntry>;
using surname_list = std::unordered_set<std::string>;
/**
* @brief Interface for services that load curated data used to seed
* brewery/user generation.
@@ -38,15 +40,9 @@ class ICuratedDataService {
virtual std::vector<UserPersona> LoadPersonas(
const std::filesystem::path& filepath) = 0;
/**
* @brief Loads the names-by-country fixture pair into a sampling-capable
* NamesByCountry.
*
* @param forenames_filepath Path to forenames-by-country.json.
* @param surnames_filepath Path to surnames-by-country.json.
*/
virtual NamesByCountry LoadNamesByCountry(
const std::filesystem::path& forenames_filepath,
virtual std::unordered_map<std::string, forename_list> LoadForenamesByCountry(
const std::filesystem::path& forenames_filepath) = 0;
virtual std::unordered_map<std::string, surname_list> LoadSurnamesByCountry(
const std::filesystem::path& surnames_filepath) = 0;
};

View File

@@ -12,7 +12,7 @@
#include <string>
#include <string_view>
#include "sqlite_handle_types.h"
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal {

View File

@@ -11,10 +11,10 @@
#include <string>
#include <unordered_map>
#include "../datetime/date_time_provider.h"
#include "data_model/models.h"
#include "export_service.h"
#include "sqlite_export_service_helpers.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/datetime/date_time_provider.h"
/**
* @brief Persists generated brewery records into a fresh SQLite database.

View File

@@ -3,8 +3,8 @@
/* Umbrella header for backward compatibility. */
#include "sqlite_connection_helpers.h"
#include "sqlite_handle_types.h"
#include "sqlite_statement_helpers.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_

View File

@@ -13,7 +13,7 @@
#include <string_view>
#include <vector>
#include "sqlite_handle_types.h"
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal {
@@ -109,30 +109,36 @@ INSERT INTO users (
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above.
inline constexpr int kLocationCityBindIndex = 1;
inline constexpr int kLocationStateProvinceBindIndex = 2;
inline constexpr int kLocationIso31662BindIndex = 3;
inline constexpr int kLocationCountryBindIndex = 4;
inline constexpr int kLocationIso31661BindIndex = 5;
inline constexpr int kLocationLanguagesBindIndex = 6;
inline constexpr int kLocationLatitudeBindIndex = 7;
inline constexpr int kLocationLongitudeBindIndex = 8;
enum eLocationBindIndex {
kLocationCityBindIndex = 1,
kLocationStateProvinceBindIndex,
kLocationIso31662BindIndex,
kLocationCountryBindIndex,
kLocationIso31661BindIndex,
kLocationLanguagesBindIndex,
kLocationLatitudeBindIndex,
kLocationLongitudeBindIndex,
};
inline constexpr int kBreweryLocationIdBindIndex = 1;
inline constexpr int kBreweryEnglishNameBindIndex = 2;
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3;
inline constexpr int kBreweryLocalNameBindIndex = 4;
inline constexpr int kBreweryLocalDescriptionBindIndex = 5;
enum BreweryBindIndex {
kBreweryLocationIdBindIndex = 1,
kBreweryEnglishNameBindIndex,
kBreweryEnglishDescriptionBindIndex,
kBreweryLocalNameBindIndex,
kBreweryLocalDescriptionBindIndex,
};
inline constexpr int kUserLocationIdBindIndex = 1;
inline constexpr int kUserFirstNameBindIndex = 2;
inline constexpr int kUserLastNameBindIndex = 3;
inline constexpr int kUserGenderBindIndex = 4;
inline constexpr int kUserUsernameBindIndex = 5;
inline constexpr int kUserBioBindIndex = 6;
inline constexpr int kUserActivityWeightBindIndex = 7;
inline constexpr int kUserEmailBindIndex = 8;
inline constexpr int kUserDateOfBirthBindIndex = 9;
enum UserBindIndex {
kUserLocationIdBindIndex = 1,
kUserFirstNameBindIndex,
kUserLastNameBindIndex,
kUserGenderBindIndex,
kUserUsernameBindIndex,
kUserBioBindIndex,
kUserActivityWeightBindIndex,
kUserEmailBindIndex,
kUserDateOfBirthBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql,

View File

@@ -8,7 +8,7 @@
#include <string>
#include "enrichment_service.h"
#include "services/enrichment/enrichment_service.h"
/**
* @brief Enrichment service that returns no context for any location.

View File

@@ -11,7 +11,7 @@
#include <string_view>
#include <unordered_map>
#include "enrichment_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
#include "web_client/web_client.h"

View File

@@ -6,8 +6,8 @@
* a bounded channel for later processing by the dispatcher.
*/
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#include <string_view>
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
BoundedChannel<LogEntry>& channel_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_

View File

@@ -52,4 +52,4 @@ class HttpWebClient final : public WebClient {
std::shared_ptr<ILogger> logger_;
};
#endif
#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_

View File

@@ -14,10 +14,9 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
const ApplicationOptions& application_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)),
application_options_(application_options) {
}
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)),
application_options_(application_options) {}

View File

@@ -6,11 +6,13 @@
#include <cctype>
#include <chrono>
#include <format>
#include <iterator>
#include <optional>
#include <random>
#include <stdexcept>
#include <string>
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include "biergarten_pipeline_orchestrator.h"
@@ -24,7 +26,7 @@ std::string Sanitize(std::string_view value) {
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
@@ -69,7 +71,37 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
}
} // namespace
std::optional<Name> SampleName(
const std::unordered_map<std::string, forename_list>& forenames_by_country,
const std::unordered_map<std::string, surname_list>& surnames_by_country,
const std::string& iso3166_1, std::mt19937& rng) {
const auto forenames_it = forenames_by_country.find(iso3166_1);
const auto surnames_it = surnames_by_country.find(iso3166_1);
if (forenames_it == forenames_by_country.end() ||
surnames_it == surnames_by_country.end() ||
forenames_it->second.empty() || surnames_it->second.empty()) {
return std::nullopt;
}
const forename_list& forenames = forenames_it->second;
const surname_list& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
auto forename_it = forenames.begin();
std::advance(forename_it, forename_dist(rng));
auto surname_it = surnames.begin();
std::advance(surname_it, surname_dist(rng));
return Name{.first_name = forename_it->name,
.last_name = *surname_it,
.gender = forename_it->gender};
}
} // namespace
void BiergartenPipelineOrchestrator::GenerateUsers(
std::span<const EnrichedCity> cities) {
@@ -85,9 +117,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
"No personas available in personas.json for user generation");
}
const NamesByCountry names_by_country =
curated_data_service_->LoadNamesByCountry(
"forenames-by-country.json", "surnames-by-country.json");
const std::unordered_map<std::string, forename_list> forenames_by_country =
curated_data_service_->LoadForenamesByCountry(
"forenames-by-country.json");
const std::unordered_map<std::string, surname_list> surnames_by_country =
curated_data_service_->LoadSurnamesByCountry("surnames-by-country.json");
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
@@ -98,8 +132,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
const auto generate_record =
[this, &rng, &skipped_count](
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
try {
std::unordered_set<std::string> used_email_local_parts;
@@ -125,25 +159,26 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
};
const auto export_record = [this,
&export_failed_count](const UserRecord& record) {
&export_failed_count](const UserRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
}
};
for (const auto& city : cities) {
const std::optional<Name> sampled_name =
names_by_country.SampleName(city.location.iso3166_1, rng);
SampleName(forenames_by_country, surnames_by_country,
city.location.iso3166_1, rng);
if (!sampled_name.has_value()) {
++skipped_count;
@@ -171,11 +206,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
}
if (export_failed_count > 0) {

View File

@@ -7,8 +7,8 @@
#include <chrono>
#include <format>
#include "../../includes/json_handling/pretty_print.h"
#include "biergarten_pipeline_orchestrator.h"
#include "json_handling/pretty_print.h"
#include "services/logging/logger.h"
void BiergartenPipelineOrchestrator::LogResults() const {

View File

@@ -4,6 +4,8 @@
* parsing, token decoding, and JSON validation helpers for Llama modules.
*/
#include <llama.h>
#include <algorithm>
#include <array>
#include <boost/json.hpp>
@@ -16,7 +18,7 @@
#include <vector>
#include "data_generation/llama_generator_helpers.h"
#include "llama.h"
namespace {
/**
* String trimming: removes leading and trailing whitespace

View File

@@ -5,6 +5,8 @@
* output tokens back to text for system+user chat prompts.
*/
#include <llama.h>
#include <algorithm>
#include <chrono>
#include <format>
@@ -16,7 +18,6 @@
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#include "llama.h"
static constexpr size_t kPromptTokenSlack = 8;
// Minimum tokens to keep when using top-p sampling. Ensures at least one

View File

@@ -5,6 +5,8 @@
#include "data_generation/llama_generator.h"
#include <llama.h>
#include <filesystem>
#include <memory>
#include <random>
@@ -12,7 +14,6 @@
#include <string>
#include "data_model/models.h"
#include "llama.h"
static constexpr uint32_t kMaxContextSize = 32768U;

View File

@@ -4,6 +4,9 @@
* context, and resets prior resources during model initialization.
*/
#include <ggml-backend.h>
#include <llama.h>
#include <algorithm>
#include <chrono>
#include <stdexcept>
@@ -11,8 +14,6 @@
#include <utility>
#include "data_generation/llama_generator.h"
#include "ggml-backend.h"
#include "llama.h"
// Maximum batch size for decode operations. Capping the batch prevents
// excessive memory allocation while maintaining inference performance.

View File

@@ -1,40 +0,0 @@
/**
* @file data_model/names_by_country.cc
* @brief NamesByCountry::SampleName() implementation.
*/
#include "data_model/names_by_country.h"
#include <utility>
NamesByCountry::NamesByCountry(
std::unordered_map<std::string, std::vector<ForenameEntry>>
forenames_by_country,
std::unordered_map<std::string, std::vector<std::string>>
surnames_by_country)
: forenames_by_country_(std::move(forenames_by_country)),
surnames_by_country_(std::move(surnames_by_country)) {}
std::optional<Name> NamesByCountry::SampleName(const std::string& iso3166_1,
std::mt19937& rng) const {
const auto forenames_it = forenames_by_country_.find(iso3166_1);
const auto surnames_it = surnames_by_country_.find(iso3166_1);
if (forenames_it == forenames_by_country_.end() ||
surnames_it == surnames_by_country_.end() ||
forenames_it->second.empty() || surnames_it->second.empty()) {
return std::nullopt;
}
const std::vector<ForenameEntry>& forenames = forenames_it->second;
const std::vector<std::string>& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
const ForenameEntry& forename = forenames[forename_dist(rng)];
return Name{.first_name = forename.name,
.last_name = surnames[surname_dist(rng)],
.gender = forename.gender};
}

View File

@@ -137,7 +137,6 @@ std::vector<Location> JsonLoader::LoadLocations(
.longitude = ReadRequiredNumber(object, "longitude"),
});
}
return locations;
}

View File

@@ -18,27 +18,7 @@
#include <string>
#include <thread>
#include "biergarten_pipeline_orchestrator.h"
#include "concurrency/bounded_channel.h"
#include "data_generation/llama_generator.h"
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_model/models.h"
#include "json_handling/json_loader.h"
#include "llama_backend_state.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service.h"
#include "services/datetime/timer.h"
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
#include "web_client/http_web_client.h"
#include "biergarten_pipeline.h"
namespace di = boost::di;
@@ -49,7 +29,7 @@ int main(const int argc, char** argv) {
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
auto log_dispatcher = //
auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel);
@@ -111,17 +91,17 @@ int main(const int argc, char** argv) {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>());
@@ -145,7 +125,7 @@ int main(const int argc, char** argv) {
}),
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
@@ -164,8 +144,7 @@ int main(const int argc, char** argv) {
}),
di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory,
&log_producer
](const auto& inj) -> std::unique_ptr<DataGenerator> {
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
@@ -175,13 +154,13 @@ int main(const int argc, char** argv) {
}
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
return std::make_unique<LlamaGenerator>(
options, model_path, log_producer,