diff --git a/tooling/pipeline-results-viewer/src/main.ts b/tooling/pipeline-results-viewer/src/main.ts index 334a551..75e70cd 100644 --- a/tooling/pipeline-results-viewer/src/main.ts +++ b/tooling/pipeline-results-viewer/src/main.ts @@ -1,20 +1,23 @@ import "95css/css/95css.css"; -import "./style.css"; import initSqlJs, { type Database } from "sql.js"; import sqlWasmUrl from "sql.js/dist/sql-wasm.wasm?url"; +import "./style.css"; const DB_URL = "/biergarten.sqlite"; const tableSelect = document.querySelector("#table-select")!; -const tableContainer = document.querySelector("#table-container")!; +const tableContainer = + document.querySelector("#table-container")!; const status = document.querySelector("#status")!; -function renderTable(db: Database, name: string): void { +const renderTable = (db: Database, name: string) => { const result = db.exec(`SELECT * FROM "${name}"`); tableContainer.replaceChildren(); if (result.length === 0) { - tableContainer.append(Object.assign(document.createElement("p"), { textContent: "No rows." })); + tableContainer.append( + Object.assign(document.createElement("p"), { textContent: "No rows." }), + ); return; } @@ -23,19 +26,23 @@ function renderTable(db: Database, name: string): void { const head = table.createTHead().insertRow(); for (const column of columns) { - head.append(Object.assign(document.createElement("th"), { textContent: column })); + const th = document.createElement("th"); + th.textContent = column; + head.append(th); } const body = table.createTBody(); for (const row of values) { const tr = body.insertRow(); for (const cell of row) { - tr.append(Object.assign(document.createElement("td"), { textContent: String(cell ?? "") })); + const td = document.createElement("td"); + td.textContent = String(cell ?? ""); + tr.append(td); } } tableContainer.append(table); -} +}; async function main(): Promise { status.textContent = "Loading database..."; @@ -47,14 +54,24 @@ async function main(): Promise { const tables = db - .exec("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")[0] + .exec( + `SELECT name + FROM sqlite_master + WHERE + type = 'table' + AND + name NOT LIKE 'sqlite_%' + ORDER BY name`, + )[0] ?.values.map((row) => String(row[0])) ?? []; for (const name of tables) { tableSelect.append(new Option(name, name)); } - tableSelect.addEventListener("change", () => renderTable(db, tableSelect.value)); + tableSelect.addEventListener("change", () => + renderTable(db, tableSelect.value), + ); if (tables.length > 0) { renderTable(db, tables[0]); diff --git a/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h b/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h index 0011881..5833b45 100644 --- a/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h +++ b/tooling/pipeline/includes/biergarten_pipeline_orchestrator.h @@ -15,6 +15,7 @@ #include "data_generation/data_generator.h" #include "data_model/generated_models.h" +#include "services/curated_data/curated_data_service.h" #include "services/database/export_service.h" #include "services/enrichment/enrichment_service.h" #include "services/logging/logger.h" @@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator { * @param generator Implementation (Llama or Mock) for brewery/user * generation. * @param exporter Database backend for persisting generated records. + * @param curated_data_service Loads curated location, persona, and name + * data used to seed generation. * @param application_options CLI configuration and paths. */ BiergartenPipelineOrchestrator( @@ -41,6 +44,7 @@ class BiergartenPipelineOrchestrator { std::unique_ptr context_service, std::unique_ptr generator, std::unique_ptr exporter, + std::unique_ptr curated_data_service, const ApplicationOptions& application_options); /** @@ -76,6 +80,7 @@ class BiergartenPipelineOrchestrator { * @brief Storage backend for generated brewery and user records. */ std::unique_ptr exporter_; + std::unique_ptr curated_data_service_; ApplicationOptions application_options_; @@ -114,4 +119,5 @@ class BiergartenPipelineOrchestrator { std::vector generated_breweries_; std::vector generated_users_; }; + #endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_ diff --git a/tooling/pipeline/includes/json_handling/json_loader.h b/tooling/pipeline/includes/json_handling/json_loader.h index 92236c6..45cb79f 100644 --- a/tooling/pipeline/includes/json_handling/json_loader.h +++ b/tooling/pipeline/includes/json_handling/json_loader.h @@ -3,33 +3,34 @@ /** * @file json_handling/json_loader.h - * @brief Loader API for curated location data. + * @brief JSON-backed implementation of ICuratedDataService. */ #include -#include #include #include "data_model/models.h" #include "data_model/names_by_country.h" -#include "services/logging/logger.h" +#include "services/curated_data/curated_data_service.h" /** - * @brief Loads curated world locations from a JSON file into memory. + * @brief Loads curated location, persona, and name data from JSON files. */ -class JsonLoader { +class JsonLoader final : public ICuratedDataService { public: + JsonLoader() = default; + /** * @brief Parses a JSON array file and returns all location records. */ - static std::vector LoadLocations( - const std::filesystem::path& filepath); + std::vector LoadLocations( + const std::filesystem::path& filepath) override; /** * @brief Parses a JSON array file and returns all persona records. */ - static std::vector LoadPersonas( - const std::filesystem::path& filepath); + std::vector LoadPersonas( + const std::filesystem::path& filepath) override; /** * @brief Parses the names-by-country fixture pair into a sampling-capable @@ -38,9 +39,9 @@ class JsonLoader { * @param forenames_filepath Path to forenames-by-country.json. * @param surnames_filepath Path to surnames-by-country.json. */ - static NamesByCountry LoadNamesByCountry( + NamesByCountry LoadNamesByCountry( const std::filesystem::path& forenames_filepath, - const std::filesystem::path& surnames_filepath); + const std::filesystem::path& surnames_filepath) override; }; #endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_ diff --git a/tooling/pipeline/includes/services/curated_data/curated_data_service.h b/tooling/pipeline/includes/services/curated_data/curated_data_service.h new file mode 100644 index 0000000..e2ddbc5 --- /dev/null +++ b/tooling/pipeline/includes/services/curated_data/curated_data_service.h @@ -0,0 +1,53 @@ +#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_ +#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_ + +/** + * @file services/curated_data/curated_data_service.h + * @brief Abstraction for loading curated location, persona, and name data. + */ + +#include +#include + +#include "data_model/models.h" +#include "data_model/names_by_country.h" + +/** + * @brief Interface for services that load curated data used to seed + * brewery/user generation. + */ +class ICuratedDataService { + public: + ICuratedDataService() = default; + virtual ~ICuratedDataService() = default; + + ICuratedDataService(const ICuratedDataService&) = delete; + ICuratedDataService& operator=(const ICuratedDataService&) = delete; + ICuratedDataService(ICuratedDataService&&) = delete; + ICuratedDataService& operator=(ICuratedDataService&&) = delete; + + /** + * @brief Loads all curated location records. + */ + virtual std::vector LoadLocations( + const std::filesystem::path& filepath) = 0; + + /** + * @brief Loads all curated persona records. + */ + virtual std::vector LoadPersonas( + const std::filesystem::path& filepath) = 0; + + /** + * @brief Loads the names-by-country fixture pair into a sampling-capable + * NamesByCountry. + * + * @param forenames_filepath Path to forenames-by-country.json. + * @param surnames_filepath Path to surnames-by-country.json. + */ + virtual NamesByCountry LoadNamesByCountry( + const std::filesystem::path& forenames_filepath, + const std::filesystem::path& surnames_filepath) = 0; +}; + +#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_ diff --git a/tooling/pipeline/run.sh b/tooling/pipeline/run.sh new file mode 100644 index 0000000..365fbdd --- /dev/null +++ b/tooling/pipeline/run.sh @@ -0,0 +1,8 @@ +./biergarten-pipeline \ + --model ../models/google_gemma-4-E4B-it-Q6_K.gguf \ + --temperature 1.0 \ + --top-p 0.95 \ + --top-k 64 \ + --n-ctx 8192 \ + --location-count 15 \ + --prompt-dir ../prompts diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc index 9c0a0ea..7fa70e7 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc @@ -12,9 +12,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator( std::unique_ptr context_service, std::unique_ptr generator, std::unique_ptr exporter, - const ApplicationOptions& app_options) - : logger_(std::move(logger)), - context_service_(std::move(context_service)), - generator_(std::move(generator)), - exporter_(std::move(exporter)), - application_options_(app_options) {} + std::unique_ptr curated_data_service, + const ApplicationOptions& application_options) + : logger_(std::move(logger)), + context_service_(std::move(context_service)), + generator_(std::move(generator)), + exporter_(std::move(exporter)), + curated_data_service_(std::move(curated_data_service)), + application_options_(application_options) { +} \ No newline at end of file diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc index 8a562c3..3592d86 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/generate_users.cc @@ -18,14 +18,13 @@ #include "services/logging/logger.h" namespace { - std::string Sanitize(std::string_view value) { std::string out; out.reserve(value.size()); for (const char character : value) { if (std::isalnum(static_cast(character)) != 0) { out.push_back(static_cast( - std::tolower(static_cast(character)))); + std::tolower(static_cast(character)))); } } return out; @@ -70,8 +69,7 @@ std::string GenerateDateOfBirth(std::mt19937& rng) { static_cast(birth_ymd.month()), static_cast(birth_ymd.day())); } - -} // namespace +} // namespace void BiergartenPipelineOrchestrator::GenerateUsers( std::span cities) { @@ -80,15 +78,16 @@ void BiergartenPipelineOrchestrator::GenerateUsers( .message = "=== SAMPLE USER GENERATION ==="}); const std::vector personas = - JsonLoader::LoadPersonas("personas.json"); + curated_data_service_->LoadPersonas("personas.json"); if (personas.empty()) { throw std::runtime_error( "No personas available in personas.json for user generation"); } - const NamesByCountry names_by_country = JsonLoader::LoadNamesByCountry( - "forenames-by-country.json", "surnames-by-country.json"); + const NamesByCountry names_by_country = + curated_data_service_->LoadNamesByCountry( + "forenames-by-country.json", "surnames-by-country.json"); std::mt19937 rng(std::random_device{}()); std::uniform_int_distribution persona_dist(0, personas.size() - 1); @@ -99,8 +98,8 @@ void BiergartenPipelineOrchestrator::GenerateUsers( const auto generate_record = [this, &rng, &skipped_count]( - const EnrichedCity& city, const UserPersona& persona, - const Name& sampled_name) -> std::optional { + const EnrichedCity& city, const UserPersona& persona, + const Name& sampled_name) -> std::optional { try { std::unordered_set used_email_local_parts; @@ -126,19 +125,19 @@ void BiergartenPipelineOrchestrator::GenerateUsers( }; const auto export_record = [this, - &export_failed_count](const UserRecord& record) { + &export_failed_count](const UserRecord& record) { try { exporter_->ProcessRecord(record); } catch (const std::exception& export_exception) { ++export_failed_count; logger_->Log( - {.level = LogLevel::Warn, - .phase = PipelinePhase::UserGeneration, - .message = std::format("[Pipeline] Generated user for '{}' ({}) but " - "SQLite export failed: {}", - record.location.city, record.location.country, - export_exception.what())}); + {.level = LogLevel::Warn, + .phase = PipelinePhase::UserGeneration, + .message = std::format("[Pipeline] Generated user for '{}' ({}) but " + "SQLite export failed: {}", + record.location.city, record.location.country, + export_exception.what())}); } }; @@ -172,11 +171,11 @@ void BiergartenPipelineOrchestrator::GenerateUsers( if (skipped_count > 0) { logger_->Log( - {.level = LogLevel::Warn, - .phase = PipelinePhase::UserGeneration, - .message = std::format( - "[Pipeline] Skipped {} city/cities during user generation", - skipped_count)}); + {.level = LogLevel::Warn, + .phase = PipelinePhase::UserGeneration, + .message = std::format( + "[Pipeline] Skipped {} city/cities during user generation", + skipped_count)}); } if (export_failed_count > 0) { @@ -186,4 +185,4 @@ void BiergartenPipelineOrchestrator::GenerateUsers( "generated user/users to SQLite", export_failed_count)}); } -} +} \ No newline at end of file diff --git a/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc b/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc index 5aad4be..441dd8b 100644 --- a/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc +++ b/tooling/pipeline/src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc @@ -23,7 +23,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() { const std::filesystem::path locations_path = "locations.json"; - auto all_locations = JsonLoader::LoadLocations(locations_path); + auto all_locations = curated_data_service_->LoadLocations(locations_path); const size_t sample_count = std::min( static_cast(application_options_.pipeline.location_count), diff --git a/tooling/pipeline/src/main.cc b/tooling/pipeline/src/main.cc index f13e844..7e3a604 100644 --- a/tooling/pipeline/src/main.cc +++ b/tooling/pipeline/src/main.cc @@ -24,7 +24,9 @@ #include "data_generation/mock_generator.h" #include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h" #include "data_model/models.h" +#include "json_handling/json_loader.h" #include "llama_backend_state.h" +#include "services/curated_data/curated_data_service.h" #include "services/database/export_service.h" #include "services/database/sqlite_export_service.h" #include "services/datetime/timer.h" @@ -47,7 +49,7 @@ int main(const int argc, char** argv) { spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v"); BoundedChannel log_channel(kLogMaxCount); - auto log_dispatcher = // + auto log_dispatcher = // std::make_unique(log_channel); std::shared_ptr log_producer = std::make_shared(log_channel); @@ -104,21 +106,22 @@ int main(const int argc, char** argv) { di::bind().to(options), di::bind().to(model_path), di::bind().to(), + di::bind().to(), di::bind().to([options, log_producer] { if (options.generator.use_mocked) { { log_producer->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = "Prompt formatter: none (mock mode)"}); + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = "Prompt formatter: none (mock mode)"}); } return std::unique_ptr(nullptr); } { log_producer->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = "Prompt formatter: Gemma4JinjaPromptFormatter"}); + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = "Prompt formatter: Gemma4JinjaPromptFormatter"}); } return std::unique_ptr( std::make_unique()); @@ -142,7 +145,7 @@ int main(const int argc, char** argv) { }), di::bind().to( [options, &log_producer]( - const auto& inj) -> std::unique_ptr { + const auto& inj) -> std::unique_ptr { if (options.generator.use_mocked) { { log_producer->Log({.level = LogLevel::Info, @@ -162,7 +165,8 @@ int main(const int argc, char** argv) { }), di::bind().to( [&options, &model_path, &sampling, &prompt_directory, - &log_producer](const auto& inj) -> std::unique_ptr { + &log_producer + ](const auto& inj) -> std::unique_ptr { if (options.generator.use_mocked) { { log_producer->Log({.level = LogLevel::Info, @@ -173,14 +177,14 @@ int main(const int argc, char** argv) { } { log_producer->Log( - {.level = LogLevel::Info, - .phase = PipelinePhase::Startup, - .message = std::format( - "Generator: LlamaGenerator | model={} | " - "temp={:.2f} " - "top_p={:.2f} top_k={} n_ctx={} seed={}", - model_path, sampling.temperature, sampling.top_p, - sampling.top_k, sampling.n_ctx, sampling.seed)}); + {.level = LogLevel::Info, + .phase = PipelinePhase::Startup, + .message = std::format( + "Generator: LlamaGenerator | model={} | " + "temp={:.2f} " + "top_p={:.2f} top_k={} n_ctx={} seed={}", + model_path, sampling.temperature, sampling.top_p, + sampling.top_k, sampling.n_ctx, sampling.seed)}); } return std::make_unique( options, model_path, log_producer, @@ -204,7 +208,6 @@ int main(const int argc, char** argv) { timer.Elapsed())}); return shutdown(EXIT_SUCCESS); - } catch (const std::exception& exception) { const LogDTO log_entry{.level = LogLevel::Error, .phase = PipelinePhase::Teardown, @@ -217,4 +220,4 @@ int main(const int argc, char** argv) { return shutdown(EXIT_FAILURE); } -} +} \ No newline at end of file