AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 17:47:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

rename json loader

Browse Source
This commit is contained in:
Aaron Po
2026-06-27 15:16:34 -04:00
parent 9137f4bddb
commit d1ad1eb397
9 changed files with 158 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tooling/pipeline/includes/biergarten_pipeline_orchestrator.h
View File

@@ -15,6 +15,7 @@
#include "data_generation/data_generator.h"
#include "data_model/generated_models.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
@@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator {
* @param generator Implementation (Llama or Mock) for brewery/user
* generation.
* @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name
* data used to seed generation.
* @param application_options CLI configuration and paths.
*/
BiergartenPipelineOrchestrator(
@@ -41,6 +44,7 @@ class BiergartenPipelineOrchestrator {
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
const ApplicationOptions& application_options);
/**
@@ -76,6 +80,7 @@ class BiergartenPipelineOrchestrator {
* @brief Storage backend for generated brewery and user records.
*/
std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_;
ApplicationOptions application_options_;
@@ -114,4 +119,5 @@ class BiergartenPipelineOrchestrator {
std::vector<BreweryRecord> generated_breweries_;
std::vector<UserRecord> generated_users_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_

23
tooling/pipeline/includes/json_handling/json_loader.h
View File

@@ -3,33 +3,34 @@
/**
* @file json_handling/json_loader.h
* @brief Loader API for curated location data.
* @brief JSON-backed implementation of ICuratedDataService.
*/
#include <filesystem>
#include <memory>
#include <vector>
#include "data_model/models.h"
#include "data_model/names_by_country.h"
#include "services/logging/logger.h"
#include "services/curated_data/curated_data_service.h"
/**
* @brief Loads curated world locations from a JSON file into memory.
* @brief Loads curated location, persona, and name data from JSON files.
*/
class JsonLoader {
class JsonLoader final : public ICuratedDataService {
public:
JsonLoader() = default;
/**
* @brief Parses a JSON array file and returns all location records.
*/
static std::vector<Location> LoadLocations(
const std::filesystem::path& filepath);
std::vector<Location> LoadLocations(
const std::filesystem::path& filepath) override;
/**
* @brief Parses a JSON array file and returns all persona records.
*/
static std::vector<UserPersona> LoadPersonas(
const std::filesystem::path& filepath);
std::vector<UserPersona> LoadPersonas(
const std::filesystem::path& filepath) override;
/**
* @brief Parses the names-by-country fixture pair into a sampling-capable
@@ -38,9 +39,9 @@ class JsonLoader {
* @param forenames_filepath Path to forenames-by-country.json.
* @param surnames_filepath Path to surnames-by-country.json.
*/
static NamesByCountry LoadNamesByCountry(
NamesByCountry LoadNamesByCountry(
const std::filesystem::path& forenames_filepath,
const std::filesystem::path& surnames_filepath);
const std::filesystem::path& surnames_filepath) override;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

53
tooling/pipeline/includes/services/curated_data/curated_data_service.h Normal file
View File

@@ -0,0 +1,53 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
/**
* @file services/curated_data/curated_data_service.h
* @brief Abstraction for loading curated location, persona, and name data.
*/
#include <filesystem>
#include <vector>
#include "data_model/models.h"
#include "data_model/names_by_country.h"
/**
* @brief Interface for services that load curated data used to seed
* brewery/user generation.
*/
class ICuratedDataService {
public:
ICuratedDataService() = default;
virtual ~ICuratedDataService() = default;
ICuratedDataService(const ICuratedDataService&) = delete;
ICuratedDataService& operator=(const ICuratedDataService&) = delete;
ICuratedDataService(ICuratedDataService&&) = delete;
ICuratedDataService& operator=(ICuratedDataService&&) = delete;
/**
* @brief Loads all curated location records.
*/
virtual std::vector<Location> LoadLocations(
const std::filesystem::path& filepath) = 0;
/**
* @brief Loads all curated persona records.
*/
virtual std::vector<UserPersona> LoadPersonas(
const std::filesystem::path& filepath) = 0;
/**
* @brief Loads the names-by-country fixture pair into a sampling-capable
* NamesByCountry.
*
* @param forenames_filepath Path to forenames-by-country.json.
* @param surnames_filepath Path to surnames-by-country.json.
*/
virtual NamesByCountry LoadNamesByCountry(
const std::filesystem::path& forenames_filepath,
const std::filesystem::path& surnames_filepath) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_

8
tooling/pipeline/run.sh Normal file
View File

@@ -0,0 +1,8 @@
./biergarten-pipeline \
--model ../models/google_gemma-4-E4B-it-Q6_K.gguf \
--temperature 1.0 \
--top-p 0.95 \
--top-k 64 \
--n-ctx 8192 \
--location-count 15 \
--prompt-dir ../prompts

15
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -12,9 +12,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
const ApplicationOptions& app_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
application_options_(app_options) {}
std::unique_ptr<ICuratedDataService> curated_data_service,
const ApplicationOptions& application_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)),
application_options_(application_options) {
}

43
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -18,14 +18,13 @@
#include "services/logging/logger.h"
namespace {
std::string Sanitize(std::string_view value) {
std::string out;
out.reserve(value.size());
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
@@ -70,8 +69,7 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
}
} // namespace
} // namespace
void BiergartenPipelineOrchestrator::GenerateUsers(
std::span<const EnrichedCity> cities) {
@@ -80,15 +78,16 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
.message = "=== SAMPLE USER GENERATION ==="});
const std::vector<UserPersona> personas =
JsonLoader::LoadPersonas("personas.json");
curated_data_service_->LoadPersonas("personas.json");
if (personas.empty()) {
throw std::runtime_error(
"No personas available in personas.json for user generation");
}
const NamesByCountry names_by_country = JsonLoader::LoadNamesByCountry(
"forenames-by-country.json", "surnames-by-country.json");
const NamesByCountry names_by_country =
curated_data_service_->LoadNamesByCountry(
"forenames-by-country.json", "surnames-by-country.json");
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
@@ -99,8 +98,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
const auto generate_record =
[this, &rng, &skipped_count](
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
try {
std::unordered_set<std::string> used_email_local_parts;
@@ -126,19 +125,19 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
};
const auto export_record = [this,
&export_failed_count](const UserRecord& record) {
&export_failed_count](const UserRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
}
};
@@ -172,11 +171,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
}
if (export_failed_count > 0) {
@@ -186,4 +185,4 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
"generated user/users to SQLite",
export_failed_count)});
}
}
}

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
View File

@@ -23,7 +23,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
const std::filesystem::path locations_path = "locations.json";
auto all_locations = JsonLoader::LoadLocations(locations_path);
auto all_locations = curated_data_service_->LoadLocations(locations_path);
const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count),

41
tooling/pipeline/src/main.cc
View File

@@ -24,7 +24,9 @@
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_model/models.h"
#include "json_handling/json_loader.h"
#include "llama_backend_state.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service.h"
#include "services/datetime/timer.h"
@@ -47,7 +49,7 @@ int main(const int argc, char** argv) {
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
auto log_dispatcher = //
auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel);
@@ -104,21 +106,22 @@ int main(const int argc, char** argv) {
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<ICuratedDataService>().to<JsonLoader>(),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>());
@@ -142,7 +145,7 @@ int main(const int argc, char** argv) {
}),
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
@@ -162,7 +165,8 @@ int main(const int argc, char** argv) {
}),
di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory,
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
&log_producer
](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
@@ -173,14 +177,14 @@ int main(const int argc, char** argv) {
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} "
"top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} "
"top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
}
return std::make_unique<LlamaGenerator>(
options, model_path, log_producer,
@@ -204,7 +208,6 @@ int main(const int argc, char** argv) {
timer.Elapsed())});
return shutdown(EXIT_SUCCESS);
} catch (const std::exception& exception) {
const LogDTO log_entry{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
@@ -217,4 +220,4 @@ int main(const int argc, char** argv) {
return shutdown(EXIT_FAILURE);
}
}
}