AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-17 01:47:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

rename json loader

Browse Source
This commit is contained in:
Aaron Po
2026-06-27 15:16:34 -04:00
parent 9137f4bddb
commit d1ad1eb397
9 changed files with 158 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
View File

@@ -12,9 +12,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
const ApplicationOptions& app_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
application_options_(app_options) {}
std::unique_ptr<ICuratedDataService> curated_data_service,
const ApplicationOptions& application_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)),
application_options_(application_options) {
}

43
tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc
View File

@@ -18,14 +18,13 @@
#include "services/logging/logger.h"
namespace {
std::string Sanitize(std::string_view value) {
std::string out;
out.reserve(value.size());
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
@@ -70,8 +69,7 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
}
} // namespace
} // namespace
void BiergartenPipelineOrchestrator::GenerateUsers(
std::span<const EnrichedCity> cities) {
@@ -80,15 +78,16 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
.message = "=== SAMPLE USER GENERATION ==="});
const std::vector<UserPersona> personas =
JsonLoader::LoadPersonas("personas.json");
curated_data_service_->LoadPersonas("personas.json");
if (personas.empty()) {
throw std::runtime_error(
"No personas available in personas.json for user generation");
}
const NamesByCountry names_by_country = JsonLoader::LoadNamesByCountry(
"forenames-by-country.json", "surnames-by-country.json");
const NamesByCountry names_by_country =
curated_data_service_->LoadNamesByCountry(
"forenames-by-country.json", "surnames-by-country.json");
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
@@ -99,8 +98,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
const auto generate_record =
[this, &rng, &skipped_count](
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
try {
std::unordered_set<std::string> used_email_local_parts;
@@ -126,19 +125,19 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
};
const auto export_record = [this,
&export_failed_count](const UserRecord& record) {
&export_failed_count](const UserRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.location.city, record.location.country,
export_exception.what())});
}
};
@@ -172,11 +171,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
}
if (export_failed_count > 0) {
@@ -186,4 +185,4 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
"generated user/users to SQLite",
export_failed_count)});
}
}
}

2
tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
View File

@@ -23,7 +23,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
const std::filesystem::path locations_path = "locations.json";
auto all_locations = JsonLoader::LoadLocations(locations_path);
auto all_locations = curated_data_service_->LoadLocations(locations_path);
const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count),

41
tooling/pipeline/src/main.cc
View File

@@ -24,7 +24,9 @@
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_model/models.h"
#include "json_handling/json_loader.h"
#include "llama_backend_state.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service.h"
#include "services/datetime/timer.h"
@@ -47,7 +49,7 @@ int main(const int argc, char** argv) {
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
auto log_dispatcher = //
auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel);
@@ -104,21 +106,22 @@ int main(const int argc, char** argv) {
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<ICuratedDataService>().to<JsonLoader>(),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>());
@@ -142,7 +145,7 @@ int main(const int argc, char** argv) {
}),
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
@@ -162,7 +165,8 @@ int main(const int argc, char** argv) {
}),
di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory,
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
&log_producer
](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
@@ -173,14 +177,14 @@ int main(const int argc, char** argv) {
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} "
"top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} "
"top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
}
return std::make_unique<LlamaGenerator>(
options, model_path, log_producer,
@@ -204,7 +208,6 @@ int main(const int argc, char** argv) {
timer.Elapsed())});
return shutdown(EXIT_SUCCESS);
} catch (const std::exception& exception) {
const LogDTO log_entry{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
@@ -217,4 +220,4 @@ int main(const int argc, char** argv) {
return shutdown(EXIT_FAILURE);
}
}
}