mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-07-16 09:37:23 +00:00
rename json loader
This commit is contained in:
@@ -1,20 +1,23 @@
|
||||
import "95css/css/95css.css";
|
||||
import "./style.css";
|
||||
import initSqlJs, { type Database } from "sql.js";
|
||||
import sqlWasmUrl from "sql.js/dist/sql-wasm.wasm?url";
|
||||
import "./style.css";
|
||||
|
||||
const DB_URL = "/biergarten.sqlite";
|
||||
|
||||
const tableSelect = document.querySelector<HTMLSelectElement>("#table-select")!;
|
||||
const tableContainer = document.querySelector<HTMLDivElement>("#table-container")!;
|
||||
const tableContainer =
|
||||
document.querySelector<HTMLDivElement>("#table-container")!;
|
||||
const status = document.querySelector<HTMLParagraphElement>("#status")!;
|
||||
|
||||
function renderTable(db: Database, name: string): void {
|
||||
const renderTable = (db: Database, name: string) => {
|
||||
const result = db.exec(`SELECT * FROM "${name}"`);
|
||||
tableContainer.replaceChildren();
|
||||
|
||||
if (result.length === 0) {
|
||||
tableContainer.append(Object.assign(document.createElement("p"), { textContent: "No rows." }));
|
||||
tableContainer.append(
|
||||
Object.assign(document.createElement("p"), { textContent: "No rows." }),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -23,19 +26,23 @@ function renderTable(db: Database, name: string): void {
|
||||
|
||||
const head = table.createTHead().insertRow();
|
||||
for (const column of columns) {
|
||||
head.append(Object.assign(document.createElement("th"), { textContent: column }));
|
||||
const th = document.createElement("th");
|
||||
th.textContent = column;
|
||||
head.append(th);
|
||||
}
|
||||
|
||||
const body = table.createTBody();
|
||||
for (const row of values) {
|
||||
const tr = body.insertRow();
|
||||
for (const cell of row) {
|
||||
tr.append(Object.assign(document.createElement("td"), { textContent: String(cell ?? "") }));
|
||||
const td = document.createElement("td");
|
||||
td.textContent = String(cell ?? "");
|
||||
tr.append(td);
|
||||
}
|
||||
}
|
||||
|
||||
tableContainer.append(table);
|
||||
}
|
||||
};
|
||||
|
||||
async function main(): Promise<void> {
|
||||
status.textContent = "Loading database...";
|
||||
@@ -47,14 +54,24 @@ async function main(): Promise<void> {
|
||||
|
||||
const tables =
|
||||
db
|
||||
.exec("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")[0]
|
||||
.exec(
|
||||
`SELECT name
|
||||
FROM sqlite_master
|
||||
WHERE
|
||||
type = 'table'
|
||||
AND
|
||||
name NOT LIKE 'sqlite_%'
|
||||
ORDER BY name`,
|
||||
)[0]
|
||||
?.values.map((row) => String(row[0])) ?? [];
|
||||
|
||||
for (const name of tables) {
|
||||
tableSelect.append(new Option(name, name));
|
||||
}
|
||||
|
||||
tableSelect.addEventListener("change", () => renderTable(db, tableSelect.value));
|
||||
tableSelect.addEventListener("change", () =>
|
||||
renderTable(db, tableSelect.value),
|
||||
);
|
||||
|
||||
if (tables.length > 0) {
|
||||
renderTable(db, tables[0]);
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_model/generated_models.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
#include "services/logging/logger.h"
|
||||
@@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator {
|
||||
* @param generator Implementation (Llama or Mock) for brewery/user
|
||||
* generation.
|
||||
* @param exporter Database backend for persisting generated records.
|
||||
* @param curated_data_service Loads curated location, persona, and name
|
||||
* data used to seed generation.
|
||||
* @param application_options CLI configuration and paths.
|
||||
*/
|
||||
BiergartenPipelineOrchestrator(
|
||||
@@ -41,6 +44,7 @@ class BiergartenPipelineOrchestrator {
|
||||
std::unique_ptr<IEnrichmentService> context_service,
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||
const ApplicationOptions& application_options);
|
||||
|
||||
/**
|
||||
@@ -76,6 +80,7 @@ class BiergartenPipelineOrchestrator {
|
||||
* @brief Storage backend for generated brewery and user records.
|
||||
*/
|
||||
std::unique_ptr<IExportService> exporter_;
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service_;
|
||||
|
||||
ApplicationOptions application_options_;
|
||||
|
||||
@@ -114,4 +119,5 @@ class BiergartenPipelineOrchestrator {
|
||||
std::vector<BreweryRecord> generated_breweries_;
|
||||
std::vector<UserRecord> generated_users_;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
|
||||
|
||||
@@ -3,33 +3,34 @@
|
||||
|
||||
/**
|
||||
* @file json_handling/json_loader.h
|
||||
* @brief Loader API for curated location data.
|
||||
* @brief JSON-backed implementation of ICuratedDataService.
|
||||
*/
|
||||
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "data_model/models.h"
|
||||
#include "data_model/names_by_country.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
|
||||
/**
|
||||
* @brief Loads curated world locations from a JSON file into memory.
|
||||
* @brief Loads curated location, persona, and name data from JSON files.
|
||||
*/
|
||||
class JsonLoader {
|
||||
class JsonLoader final : public ICuratedDataService {
|
||||
public:
|
||||
JsonLoader() = default;
|
||||
|
||||
/**
|
||||
* @brief Parses a JSON array file and returns all location records.
|
||||
*/
|
||||
static std::vector<Location> LoadLocations(
|
||||
const std::filesystem::path& filepath);
|
||||
std::vector<Location> LoadLocations(
|
||||
const std::filesystem::path& filepath) override;
|
||||
|
||||
/**
|
||||
* @brief Parses a JSON array file and returns all persona records.
|
||||
*/
|
||||
static std::vector<UserPersona> LoadPersonas(
|
||||
const std::filesystem::path& filepath);
|
||||
std::vector<UserPersona> LoadPersonas(
|
||||
const std::filesystem::path& filepath) override;
|
||||
|
||||
/**
|
||||
* @brief Parses the names-by-country fixture pair into a sampling-capable
|
||||
@@ -38,9 +39,9 @@ class JsonLoader {
|
||||
* @param forenames_filepath Path to forenames-by-country.json.
|
||||
* @param surnames_filepath Path to surnames-by-country.json.
|
||||
*/
|
||||
static NamesByCountry LoadNamesByCountry(
|
||||
NamesByCountry LoadNamesByCountry(
|
||||
const std::filesystem::path& forenames_filepath,
|
||||
const std::filesystem::path& surnames_filepath);
|
||||
const std::filesystem::path& surnames_filepath) override;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
|
||||
|
||||
/**
|
||||
* @file services/curated_data/curated_data_service.h
|
||||
* @brief Abstraction for loading curated location, persona, and name data.
|
||||
*/
|
||||
|
||||
#include <filesystem>
|
||||
#include <vector>
|
||||
|
||||
#include "data_model/models.h"
|
||||
#include "data_model/names_by_country.h"
|
||||
|
||||
/**
|
||||
* @brief Interface for services that load curated data used to seed
|
||||
* brewery/user generation.
|
||||
*/
|
||||
class ICuratedDataService {
|
||||
public:
|
||||
ICuratedDataService() = default;
|
||||
virtual ~ICuratedDataService() = default;
|
||||
|
||||
ICuratedDataService(const ICuratedDataService&) = delete;
|
||||
ICuratedDataService& operator=(const ICuratedDataService&) = delete;
|
||||
ICuratedDataService(ICuratedDataService&&) = delete;
|
||||
ICuratedDataService& operator=(ICuratedDataService&&) = delete;
|
||||
|
||||
/**
|
||||
* @brief Loads all curated location records.
|
||||
*/
|
||||
virtual std::vector<Location> LoadLocations(
|
||||
const std::filesystem::path& filepath) = 0;
|
||||
|
||||
/**
|
||||
* @brief Loads all curated persona records.
|
||||
*/
|
||||
virtual std::vector<UserPersona> LoadPersonas(
|
||||
const std::filesystem::path& filepath) = 0;
|
||||
|
||||
/**
|
||||
* @brief Loads the names-by-country fixture pair into a sampling-capable
|
||||
* NamesByCountry.
|
||||
*
|
||||
* @param forenames_filepath Path to forenames-by-country.json.
|
||||
* @param surnames_filepath Path to surnames-by-country.json.
|
||||
*/
|
||||
virtual NamesByCountry LoadNamesByCountry(
|
||||
const std::filesystem::path& forenames_filepath,
|
||||
const std::filesystem::path& surnames_filepath) = 0;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
|
||||
8
tooling/pipeline/run.sh
Normal file
8
tooling/pipeline/run.sh
Normal file
@@ -0,0 +1,8 @@
|
||||
./biergarten-pipeline \
|
||||
--model ../models/google_gemma-4-E4B-it-Q6_K.gguf \
|
||||
--temperature 1.0 \
|
||||
--top-p 0.95 \
|
||||
--top-k 64 \
|
||||
--n-ctx 8192 \
|
||||
--location-count 15 \
|
||||
--prompt-dir ../prompts
|
||||
@@ -12,9 +12,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
|
||||
std::unique_ptr<IEnrichmentService> context_service,
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
const ApplicationOptions& app_options)
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||
const ApplicationOptions& application_options)
|
||||
: logger_(std::move(logger)),
|
||||
context_service_(std::move(context_service)),
|
||||
generator_(std::move(generator)),
|
||||
exporter_(std::move(exporter)),
|
||||
application_options_(app_options) {}
|
||||
curated_data_service_(std::move(curated_data_service)),
|
||||
application_options_(application_options) {
|
||||
}
|
||||
@@ -18,7 +18,6 @@
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
namespace {
|
||||
|
||||
std::string Sanitize(std::string_view value) {
|
||||
std::string out;
|
||||
out.reserve(value.size());
|
||||
@@ -70,7 +69,6 @@ std::string GenerateDateOfBirth(std::mt19937& rng) {
|
||||
static_cast<unsigned>(birth_ymd.month()),
|
||||
static_cast<unsigned>(birth_ymd.day()));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
@@ -80,14 +78,15 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
||||
.message = "=== SAMPLE USER GENERATION ==="});
|
||||
|
||||
const std::vector<UserPersona> personas =
|
||||
JsonLoader::LoadPersonas("personas.json");
|
||||
curated_data_service_->LoadPersonas("personas.json");
|
||||
|
||||
if (personas.empty()) {
|
||||
throw std::runtime_error(
|
||||
"No personas available in personas.json for user generation");
|
||||
}
|
||||
|
||||
const NamesByCountry names_by_country = JsonLoader::LoadNamesByCountry(
|
||||
const NamesByCountry names_by_country =
|
||||
curated_data_service_->LoadNamesByCountry(
|
||||
"forenames-by-country.json", "surnames-by-country.json");
|
||||
|
||||
std::mt19937 rng(std::random_device{}());
|
||||
|
||||
@@ -23,7 +23,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
|
||||
|
||||
const std::filesystem::path locations_path = "locations.json";
|
||||
|
||||
auto all_locations = JsonLoader::LoadLocations(locations_path);
|
||||
auto all_locations = curated_data_service_->LoadLocations(locations_path);
|
||||
|
||||
const size_t sample_count = std::min(
|
||||
static_cast<size_t>(application_options_.pipeline.location_count),
|
||||
|
||||
@@ -24,7 +24,9 @@
|
||||
#include "data_generation/mock_generator.h"
|
||||
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
|
||||
#include "data_model/models.h"
|
||||
#include "json_handling/json_loader.h"
|
||||
#include "llama_backend_state.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/database/sqlite_export_service.h"
|
||||
#include "services/datetime/timer.h"
|
||||
@@ -104,6 +106,7 @@ int main(const int argc, char** argv) {
|
||||
di::bind<ApplicationOptions>().to(options),
|
||||
di::bind<std::string>().to(model_path),
|
||||
di::bind<IExportService>().to<SqliteExportService>(),
|
||||
di::bind<ICuratedDataService>().to<JsonLoader>(),
|
||||
di::bind<IPromptFormatter>().to([options, log_producer] {
|
||||
if (options.generator.use_mocked) {
|
||||
{
|
||||
@@ -162,7 +165,8 @@ int main(const int argc, char** argv) {
|
||||
}),
|
||||
di::bind<DataGenerator>().to(
|
||||
[&options, &model_path, &sampling, &prompt_directory,
|
||||
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
|
||||
&log_producer
|
||||
](const auto& inj) -> std::unique_ptr<DataGenerator> {
|
||||
if (options.generator.use_mocked) {
|
||||
{
|
||||
log_producer->Log({.level = LogLevel::Info,
|
||||
@@ -204,7 +208,6 @@ int main(const int argc, char** argv) {
|
||||
timer.Elapsed())});
|
||||
|
||||
return shutdown(EXIT_SUCCESS);
|
||||
|
||||
} catch (const std::exception& exception) {
|
||||
const LogDTO log_entry{.level = LogLevel::Error,
|
||||
.phase = PipelinePhase::Teardown,
|
||||
|
||||
Reference in New Issue
Block a user