remove puml

This commit is contained in:
Aaron Po
2026-08-31 20:14:01 -04:00
parent f4471c7fd7
commit 12af9cfee2
20 changed files with 92 additions and 3570 deletions

View File

@@ -1,280 +0,0 @@
@startuml
!include ../biergarten-weizen-theme.puml
skinparam style strictuml
skinparam defaultFontSize 14
skinparam titleFontSize 20
skinparam ActivityBarColor #628A5B
title The Biergarten Data Pipeline (Current — Synchronous Data Path)
|#F2F6F0|main.cc|
start
:Create BoundedChannel<LogEntry> log_channel;
:Spawn log dispatcher thread\n(LogDispatcher::Run());
note right
The only concurrency in the current pipeline:
a dedicated thread drains log_channel and
forwards entries to spdlog for the entire run.
Joined during shutdown after log_channel closes.
end note
:ParseArguments(argc, argv);
if (Are arguments valid?) then (no)
:Log error / usage info;
stop
else (yes)
endif
:Init OpenSSL global state & LlamaBackendState;
:di::make_injector(...);
note right
Binds ILogger, IEnrichmentService, DataGenerator,
IExportService, IPromptFormatter, WebClient based
on --mocked vs. model-backed mode.
end note
:injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
:BiergartenPipelineOrchestrator::Run();
note right
Run() itself is synchronous — sampling, enrichment,
generation, and export all happen on this thread.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
:Initialize SQLite export;
|#E0EAE0|SqliteExportService|
:GetUtcTimestamp() from SystemDateTimeProvider;
:Initialize();
note right
Builds a fresh biergarten_seed_<UTC datetime>.sqlite filename
Appends a numeric suffix if the timestamp already exists
Opens DB Connection
Executes Schema DDL
Begins Transaction
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
:QueryLocations();
|#E2EBDC|JsonLoader|
:JsonLoader::LoadLocations("locations.json");
:std::ranges::sample(all_locations, --location-count);
note right
--location-count defaults to 10.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
while (For each sampled Location?) is (Remaining cities)
|#DCE8D8|IEnrichmentService|
:GetLocationContext(loc);
note right
WikipediaEnrichmentService fetches a generic
"brewing" extract and a "beer in {country}" extract
(not a city/region-specific page). MockEnrichmentService
(--mocked) returns an empty string instead.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
if (Lookup failed?) then (yes)
:Log warning, skip city;
else (no)
:Store EnrichedCity{Location, region_context};
endif
endwhile (Done)
|#EAF0E8|BiergartenPipelineOrchestrator|
:GenerateUsers(enriched_cities);
|#E2EBDC|JsonLoader|
:LoadPersonas("personas.json");
:LoadForenamesByCountry("forenames-by-country.json");
:LoadSurnamesByCountry("surnames-by-country.json");
note right
Each call is cached after its first parse.
MockCuratedDataService (--mocked) returns a
fixed in-memory dataset instead and skips
these three JSON files entirely.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
while (For each EnrichedCity?) is (Remaining cities)
:SampleName(forenames_by_country,\n surnames_by_country, location.iso3166_1);
if (Names available for country?) then (no)
:Log warning, skip city;
else (yes)
:Sample UserPersona (uniform random);
|#E5EDE1|DataGenerator|
if (Generator Mode) then (MockGenerator)
:DeterministicHash & Format;
elseif (Generator Mode) then (LlamaGenerator)
:prompt_directory_->Load("USER_GENERATION");
note right
Resolves to USER_GENERATION.md inside --prompt-dir.
end note
repeat
:Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
:ValidateUserJson(raw, user);
if (Is JSON Valid?) then (yes)
break
else (no)
:Attempt++;
:Append validation error to retry prompt;
endif
repeat while (Attempt < 3?) is (yes)
else (OpenAIGenerator)
:prompt_directory_->Load("USER_GENERATION");
note right
Same prompt file as LlamaGenerator - content
is backend-agnostic.
end note
repeat
:CallChatCompletionsApi(system_prompt, user_prompt,\nkUserJsonSchema, max_tokens);
:ValidateUserJson(raw, user);
if (Is JSON Valid?) then (yes)
break
else (no)
:Attempt++;
:Append validation error to retry prompt;
endif
repeat while (Attempt < 2?) is (yes)
note right
Structured Outputs (response_format:
json_schema, strict: true) already
guarantees schema shape, so only 2
attempts are needed - retries here cover
transient failures / placeholder text,
not malformed JSON.
end note
endif
|#EAF0E8|BiergartenPipelineOrchestrator|
if (Generation successful?) then (yes)
:GetRandomCoordsWithinRange(city_centre, 5km, rng);
:BuildEmail(name, used_email_local_parts);
:GenerateDateOfBirth(rng);
note right
email and date_of_birth are generated
programmatically, never LLM-authored.
Generation + coordinate jitter run inside
one try/catch per record. UserRecord.address
now carries {city, longitude, latitude}.
end note
|#E0EAE0|SqliteExportService|
:ProcessRecord(UserRecord);
if (Location in cache?) then (yes)
:Reuse location_id;
else (no)
:Insert Location & Cache ID;
endif
:Insert User (FK: location_id);
if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator|
:Log warning "SQLite export failed";
else (no)
endif
else (no)
:Log warning "Generation failed, skipping...";
endif
endif
endwhile (Done)
|#EAF0E8|BiergartenPipelineOrchestrator|
:GenerateBreweries(enriched_cities);
|#E5EDE1|DataGenerator|
while (For each EnrichedCity?) is (Remaining cities)
if (Generator Mode) then (MockGenerator)
:DeterministicHash & Format;
elseif (Generator Mode) then (LlamaGenerator)
:PrepareRegionContext;
:prompt_directory_->Load("BREWERY_GENERATION");
note right
Resolves to BREWERY_GENERATION.md inside --prompt-dir.
end note
repeat
:Infer(system_prompt, user_prompt, max_tokens, kBreweryJsonGrammar);
:ValidateBreweryJson(raw, brewery);
if (Is JSON Valid?) then (yes)
break
else (no)
:Attempt++;
:Append validation error to retry prompt;
endif
repeat while (Attempt < 3?) is (yes)
note right
max_tokens is fixed across retries —
it is not raised on truncation.
end note
else (OpenAIGenerator)
:PrepareRegionContext;
:prompt_directory_->Load("BREWERY_GENERATION");
note right
Same prompt file as LlamaGenerator - content
is backend-agnostic.
end note
repeat
:CallChatCompletionsApi(system_prompt, user_prompt,\nkBreweryJsonSchema, max_tokens);
:ValidateBreweryJson(raw, brewery);
if (Is JSON Valid?) then (yes)
break
else (no)
:Attempt++;
:Append validation error to retry prompt;
endif
repeat while (Attempt < 2?) is (yes)
note right
Structured Outputs already guarantees schema
shape, so only 2 attempts are needed - retries
here cover transient failures / placeholder
text, not malformed JSON.
end note
endif
|#EAF0E8|BiergartenPipelineOrchestrator|
if (Generation successful?) then (yes)
:GetRandomCoordsWithinRange(city_centre, 5km, rng);
note right
Generation + coordinate jitter run inside one
try/catch per record. BreweryRecord.address
now carries {city, longitude, latitude}.
end note
|#E0EAE0|SqliteExportService|
:ProcessRecord(GeneratedBrewery);
if (Location in cache?) then (yes)
:Reuse location_id;
else (no)
:Insert Location & Cache ID;
endif
:Insert Brewery (FK: location_id);
if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator|
:Log warning "SQLite export failed";
note right
Data loss is prevented per-record.
The pipeline continues running.
end note
else (no)
endif
else (no)
:Log warning "Generation failed, skipping...";
endif
|#E5EDE1|DataGenerator|
endwhile (Done)
|#E0EAE0|SqliteExportService|
:Finalize();
note right
Commits Transaction
Closes Database Connection
end note
|#F2F6F0|main.cc|
:Close log_channel;
:Join log dispatcher thread;
:Return 0;
stop
@enduml

View File

@@ -1,282 +0,0 @@
@startuml
!include ../biergarten-weizen-theme.puml
skinparam style strictuml
skinparam defaultFontSize 14
skinparam titleFontSize 20
title The Biergarten Data Pipeline - Class Diagram
class BiergartenPipelineOrchestrator {
- logger_ : std::shared_ptr<ILogger>
- context_service_ : std::unique_ptr<IEnrichmentService>
- generator_ : std::unique_ptr<DataGenerator>
- exporter_ : std::unique_ptr<IExportService>
- curated_data_service_ : std::unique_ptr<ICuratedDataService>
- application_options_ : ApplicationOptions
- generated_breweries_ : std::vector<BreweryRecord>
- generated_users_ : std::vector<UserRecord>
+ Run() : bool
- QueryLocations() : std::vector<City>
- GenerateBreweries(cities : std::span<const EnrichedCity>) : void
- GenerateUsers(cities : std::span<const EnrichedCity>) : void
- LogResults() const : void
}
note right of BiergartenPipelineOrchestrator
GenerateBreweries and GenerateUsers both call the
free function GetRandomCoordsWithinRange()
(biergarten_pipeline_orchestrator/coords.h) to jitter
a coordinate pair uniformly within 5km of the city
centre for each generated record's address.
end note
class LogLevel <<enumeration>> {
Debug
Info
Warn
Error
}
class PipelinePhase <<enumeration>> {
Startup
Enrichment
UserGeneration
BreweryAndBeerGeneration
CheckinGeneration
RatingGeneration
FollowGeneration
Teardown
}
struct LogDTO {
+ level : LogLevel
+ phase : PipelinePhase
+ message : std::string
}
struct LogEntry {
+ timestamp : std::chrono::system_clock::time_point
+ origin : std::source_location
+ thread_id : std::thread::id
+ level : LogLevel
+ phase : PipelinePhase
+ message : std::string
}
interface ILogger <<interface>> {
+ Log(payload : LogDTO) : void
- {abstract} DoLog(entry : LogEntry) : void
}
class LogProducer {
- channel_ : BoundedChannel<LogEntry>&
- DoLog(entry : LogEntry) : void
}
class LogDispatcher {
- channel_ : BoundedChannel<LogEntry>&
+ Run() : void
- ToSpdlogLevel(level) : spdlog::level::level_enum
}
interface IEnrichmentService <<interface>> {
+ GetLocationContext(loc : const Location&) : std::string
}
class MockEnrichmentService {
+ GetLocationContext(loc : const Location&) : std::string
}
class WikipediaEnrichmentService {
- client_ : std::unique_ptr<WebClient>
- extract_cache_ : std::unordered_map<std::string, std::string>
+ GetLocationContext(loc : const Location&) : std::string
- FetchExtract(query : std::string_view) : std::string
}
interface WebClient <<interface>> {
+ Get(url : const std::string&) : std::string
+ UrlEncode(value : const std::string&) : std::string
}
class HttpWebClient {
+ Get(url : const std::string&) : std::string
+ UrlEncode(value : const std::string&) : std::string
}
interface DataGenerator <<interface>> {
+ GenerateBrewery(city : const EnrichedCity&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
}
class MockGenerator {
+ GenerateBrewery(city : const EnrichedCity&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
- DeterministicHash(location : const Location&) : size_t
- DeterministicHash(location : const Location&, persona : const UserPersona&, name : const Name&) : size_t
}
class LlamaGenerator {
- model_ : ModelHandle
- context_ : ContextHandle
- prompt_formatter_ : std::unique_ptr<IPromptFormatter>
- prompt_directory_ : std::unique_ptr<IPromptDirectory>
- rng_ : std::mt19937
+ GenerateBrewery(...) : BreweryResult
+ GenerateUser(...) : UserResult
- Load(model_path : const std::string&) : void
- Infer(...) : std::string
- InferFormatted(...) : std::string
}
class OpenAIGenerator {
- api_key_ : std::string
- model_ : std::string
- logger_ : std::shared_ptr<ILogger>
- prompt_directory_ : std::unique_ptr<IPromptDirectory>
- web_client_ : std::unique_ptr<WebClient>
+ GenerateBrewery(city : const EnrichedCity&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
- CallChatCompletionsApi(...) : std::string
}
note right of OpenAIGenerator
Chat Completions with Structured Outputs
(response_format: json_schema, strict: true)
guarantees schema-valid JSON, replacing
llama.cpp's GBNF grammar constraint -
no IPromptFormatter needed.
end note
interface IPromptFormatter <<interface>> {
+ Format(system_prompt : std::string_view, user_prompt : std::string_view) : std::string
}
class Gemma4JinjaPromptFormatter {
+ Format(system_prompt : std::string_view, user_prompt : std::string_view) : std::string
}
interface IPromptDirectory <<interface>> {
+ Load(key : std::string_view) : std::string
}
class PromptDirectory {
- prompt_dir_ : std::filesystem::path
- cache_ : std::unordered_map<std::string, std::string>
+ Load(key : std::string_view) : std::string
}
interface ICuratedDataService <<interface>> {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
class JsonLoader {
- cache_ : cache
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
note right of JsonLoader
Each Load* method memoizes its result in
cache_ on first call; later calls on the
same instance return the cached value
without re-parsing.
end note
class MockCuratedDataService {
- locations_ : std::vector<Location>
- personas_ : std::vector<UserPersona>
- forenames_by_country_ : std::unordered_map<std::string, forename_list>
- surnames_by_country_ : std::unordered_map<std::string, surname_list>
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
note right of MockCuratedDataService
Fixed 4-location / 3-persona dataset
(US, DE, FR, BE) used in --mocked runs;
filepath arguments are ignored.
end note
interface IExportService <<interface>> {
+ Initialize() : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+ ProcessRecord(user : const UserRecord&) : uint64_t
+ Finalize() : void
}
class SqliteExportService {
- date_time_provider_ : std::unique_ptr<IDateTimeProvider>
- run_timestamp_utc_ : std::string
- database_path_ : std::filesystem::path
- db_handle_ : SqliteDatabaseHandle
- insert_location_stmt_ : SqliteStatementHandle
- insert_brewery_stmt_ : SqliteStatementHandle
- insert_user_stmt_ : SqliteStatementHandle
- transaction_open_ : bool
- location_cache_ : std::unordered_map<std::string, sqlite3_int64>
+ Initialize() : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+ ProcessRecord(user : const UserRecord&) : uint64_t
+ Finalize() : void
- InitializeSchema() : void
- ResolveLocationId(location : const Location&) : sqlite3_int64
}
interface IDateTimeProvider <<interface>> {
+ GetUtcTimestamp() : std::string
}
class SystemDateTimeProvider {
+ GetUtcTimestamp() : std::string
}
' Structural Relationships / Dependency Injection
BiergartenPipelineOrchestrator *-- ILogger : owns
BiergartenPipelineOrchestrator *-- IEnrichmentService : owns
BiergartenPipelineOrchestrator *-- DataGenerator : owns
BiergartenPipelineOrchestrator *-- IExportService : owns
BiergartenPipelineOrchestrator *-- ICuratedDataService : owns
LogEntry *-- LogLevel
LogEntry *-- PipelinePhase
LogDTO *-- LogLevel
LogDTO *-- PipelinePhase
ILogger <|.. LogProducer : implements
LogProducer ..> LogEntry : emits
LogDispatcher ..> LogEntry : consumes
IEnrichmentService <|.. WikipediaEnrichmentService : implements
IEnrichmentService <|.. MockEnrichmentService : implements
WikipediaEnrichmentService *-- WebClient : owns
WebClient <|.. HttpWebClient : implements
DataGenerator <|.. MockGenerator : implements
DataGenerator <|.. LlamaGenerator : implements
DataGenerator <|.. OpenAIGenerator : implements
LlamaGenerator *-- IPromptFormatter : uses
LlamaGenerator *-- IPromptDirectory : uses
OpenAIGenerator *-- IPromptDirectory : uses
OpenAIGenerator *-- WebClient : uses
IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements
IPromptDirectory <|.. PromptDirectory : implements
ICuratedDataService <|.. JsonLoader : implements
ICuratedDataService <|.. MockCuratedDataService : implements
IExportService <|.. SqliteExportService : implements
SqliteExportService *-- IDateTimeProvider : owns
IDateTimeProvider <|.. SystemDateTimeProvider : implements
@enduml