mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-15 01:54:43 +00:00
remove puml
This commit is contained in:
Aaron Po
@@ -1,56 +0,0 @@
|
||||
' skinparam shadowing false
|
||||
' skinparam backgroundColor #FCFCF7
|
||||
' skinparam defaultFontName "DM Sans"
|
||||
' skinparam defaultFontColor #14180C
|
||||
' skinparam titleFontName "Volkhov"
|
||||
' skinparam titleFontColor #14180C
|
||||
' skinparam ArrowColor #656F33
|
||||
' skinparam NoteBackgroundColor #DBEEDD
|
||||
' skinparam NoteFontColor #14180C
|
||||
|
||||
' skinparam NoteBorderColor #4A5837
|
||||
' skinparam SwimlaneBorderColor #4A5837
|
||||
' skinparam SwimlaneBorderThickness 1
|
||||
' skinparam activityStartColor #EBECE3
|
||||
' skinparam activityEndColor #4A5837
|
||||
' skinparam activityStopColor #4A5837
|
||||
' skinparam ActivityBackgroundColor #EBECE3
|
||||
' skinparam ActivityBorderColor #4A5837
|
||||
' skinparam ActivityDiamondBackgroundColor #CBD2B5
|
||||
' skinparam ActivityDiamondBorderColor #4A5837
|
||||
' skinparam packageStyle rectangle
|
||||
' skinparam packageBackgroundColor #F1F3EA
|
||||
' skinparam packageBorderColor #4A5837
|
||||
' skinparam packageFontColor #14180C
|
||||
' skinparam classBackgroundColor #EBECE3
|
||||
' skinparam classBorderColor #4A5837
|
||||
' skinparam classFontColor #14180C
|
||||
' skinparam classAttributeFontColor #3F4724
|
||||
' skinparam classStereotypeFontColor #4A5837
|
||||
' skinparam interfaceBackgroundColor #DBEEDD
|
||||
' skinparam interfaceBorderColor #4A5837
|
||||
' skinparam interfaceFontColor #14180C
|
||||
' skinparam enumBackgroundColor #E4E6D8
|
||||
' skinparam enumBorderColor #4A5837
|
||||
' skinparam enumFontColor #14180C
|
||||
' skinparam ParticipantBackgroundColor #EBECE3
|
||||
' skinparam ParticipantBorderColor #4A5837
|
||||
' skinparam ParticipantFontColor #14180C
|
||||
' skinparam ActorBackgroundColor #DBEEDD
|
||||
' skinparam ActorBorderColor #4A5837
|
||||
' skinparam ActorFontColor #14180C
|
||||
' skinparam LifeLineBorderColor #4A5837
|
||||
' skinparam LifeLineBackgroundColor #F1F3EA
|
||||
' skinparam DatabaseBackgroundColor #EBECE3
|
||||
' skinparam DatabaseBorderColor #4A5837
|
||||
' skinparam DatabaseFontColor #14180C
|
||||
' skinparam SequenceGroupBackgroundColor #DBEEDD
|
||||
' skinparam SequenceGroupBorderColor #4A5837
|
||||
' skinparam SequenceGroupFontColor #14180C
|
||||
' skinparam SequenceGroupBodyBackgroundColor transparent
|
||||
' skinparam SequenceDividerBackgroundColor #DBEEDD
|
||||
' skinparam SequenceDividerBorderColor #4A5837
|
||||
' skinparam SequenceDividerFontColor #14180C
|
||||
' skinparam SequenceBoxBackgroundColor #F1F3EA
|
||||
' skinparam SequenceBoxBorderColor #4A5837
|
||||
' skinparam SequenceBoxFontColor #14180C
|
||||
@@ -1,280 +0,0 @@
|
||||
@startuml
|
||||
!include ../biergarten-weizen-theme.puml
|
||||
skinparam style strictuml
|
||||
skinparam defaultFontSize 14
|
||||
skinparam titleFontSize 20
|
||||
skinparam ActivityBarColor #628A5B
|
||||
|
||||
title The Biergarten Data Pipeline (Current — Synchronous Data Path)
|
||||
|
||||
|#F2F6F0|main.cc|
|
||||
start
|
||||
:Create BoundedChannel<LogEntry> log_channel;
|
||||
:Spawn log dispatcher thread\n(LogDispatcher::Run());
|
||||
note right
|
||||
The only concurrency in the current pipeline:
|
||||
a dedicated thread drains log_channel and
|
||||
forwards entries to spdlog for the entire run.
|
||||
Joined during shutdown after log_channel closes.
|
||||
end note
|
||||
:ParseArguments(argc, argv);
|
||||
if (Are arguments valid?) then (no)
|
||||
:Log error / usage info;
|
||||
stop
|
||||
else (yes)
|
||||
endif
|
||||
|
||||
:Init OpenSSL global state & LlamaBackendState;
|
||||
:di::make_injector(...);
|
||||
note right
|
||||
Binds ILogger, IEnrichmentService, DataGenerator,
|
||||
IExportService, IPromptFormatter, WebClient based
|
||||
on --mocked vs. model-backed mode.
|
||||
end note
|
||||
:injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
|
||||
:BiergartenPipelineOrchestrator::Run();
|
||||
note right
|
||||
Run() itself is synchronous — sampling, enrichment,
|
||||
generation, and export all happen on this thread.
|
||||
end note
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
:Initialize SQLite export;
|
||||
|
||||
|#E0EAE0|SqliteExportService|
|
||||
:GetUtcTimestamp() from SystemDateTimeProvider;
|
||||
:Initialize();
|
||||
note right
|
||||
Builds a fresh biergarten_seed_<UTC datetime>.sqlite filename
|
||||
Appends a numeric suffix if the timestamp already exists
|
||||
Opens DB Connection
|
||||
Executes Schema DDL
|
||||
Begins Transaction
|
||||
end note
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
:QueryLocations();
|
||||
|
||||
|#E2EBDC|JsonLoader|
|
||||
:JsonLoader::LoadLocations("locations.json");
|
||||
:std::ranges::sample(all_locations, --location-count);
|
||||
note right
|
||||
--location-count defaults to 10.
|
||||
end note
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
while (For each sampled Location?) is (Remaining cities)
|
||||
|#DCE8D8|IEnrichmentService|
|
||||
:GetLocationContext(loc);
|
||||
note right
|
||||
WikipediaEnrichmentService fetches a generic
|
||||
"brewing" extract and a "beer in {country}" extract
|
||||
(not a city/region-specific page). MockEnrichmentService
|
||||
(--mocked) returns an empty string instead.
|
||||
end note
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
if (Lookup failed?) then (yes)
|
||||
:Log warning, skip city;
|
||||
else (no)
|
||||
:Store EnrichedCity{Location, region_context};
|
||||
endif
|
||||
endwhile (Done)
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
:GenerateUsers(enriched_cities);
|
||||
|
||||
|#E2EBDC|JsonLoader|
|
||||
:LoadPersonas("personas.json");
|
||||
:LoadForenamesByCountry("forenames-by-country.json");
|
||||
:LoadSurnamesByCountry("surnames-by-country.json");
|
||||
note right
|
||||
Each call is cached after its first parse.
|
||||
MockCuratedDataService (--mocked) returns a
|
||||
fixed in-memory dataset instead and skips
|
||||
these three JSON files entirely.
|
||||
end note
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
while (For each EnrichedCity?) is (Remaining cities)
|
||||
:SampleName(forenames_by_country,\n surnames_by_country, location.iso3166_1);
|
||||
if (Names available for country?) then (no)
|
||||
:Log warning, skip city;
|
||||
else (yes)
|
||||
:Sample UserPersona (uniform random);
|
||||
|
||||
|#E5EDE1|DataGenerator|
|
||||
if (Generator Mode) then (MockGenerator)
|
||||
:DeterministicHash & Format;
|
||||
elseif (Generator Mode) then (LlamaGenerator)
|
||||
:prompt_directory_->Load("USER_GENERATION");
|
||||
note right
|
||||
Resolves to USER_GENERATION.md inside --prompt-dir.
|
||||
end note
|
||||
repeat
|
||||
:Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
|
||||
:ValidateUserJson(raw, user);
|
||||
if (Is JSON Valid?) then (yes)
|
||||
break
|
||||
else (no)
|
||||
:Attempt++;
|
||||
:Append validation error to retry prompt;
|
||||
endif
|
||||
repeat while (Attempt < 3?) is (yes)
|
||||
else (OpenAIGenerator)
|
||||
:prompt_directory_->Load("USER_GENERATION");
|
||||
note right
|
||||
Same prompt file as LlamaGenerator - content
|
||||
is backend-agnostic.
|
||||
end note
|
||||
repeat
|
||||
:CallChatCompletionsApi(system_prompt, user_prompt,\nkUserJsonSchema, max_tokens);
|
||||
:ValidateUserJson(raw, user);
|
||||
if (Is JSON Valid?) then (yes)
|
||||
break
|
||||
else (no)
|
||||
:Attempt++;
|
||||
:Append validation error to retry prompt;
|
||||
endif
|
||||
repeat while (Attempt < 2?) is (yes)
|
||||
note right
|
||||
Structured Outputs (response_format:
|
||||
json_schema, strict: true) already
|
||||
guarantees schema shape, so only 2
|
||||
attempts are needed - retries here cover
|
||||
transient failures / placeholder text,
|
||||
not malformed JSON.
|
||||
end note
|
||||
endif
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
if (Generation successful?) then (yes)
|
||||
:GetRandomCoordsWithinRange(city_centre, 5km, rng);
|
||||
:BuildEmail(name, used_email_local_parts);
|
||||
:GenerateDateOfBirth(rng);
|
||||
note right
|
||||
email and date_of_birth are generated
|
||||
programmatically, never LLM-authored.
|
||||
Generation + coordinate jitter run inside
|
||||
one try/catch per record. UserRecord.address
|
||||
now carries {city, longitude, latitude}.
|
||||
end note
|
||||
|
||||
|#E0EAE0|SqliteExportService|
|
||||
:ProcessRecord(UserRecord);
|
||||
if (Location in cache?) then (yes)
|
||||
:Reuse location_id;
|
||||
else (no)
|
||||
:Insert Location & Cache ID;
|
||||
endif
|
||||
:Insert User (FK: location_id);
|
||||
|
||||
if (Exception caught during insert?) then (yes)
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
:Log warning "SQLite export failed";
|
||||
else (no)
|
||||
endif
|
||||
else (no)
|
||||
:Log warning "Generation failed, skipping...";
|
||||
endif
|
||||
endif
|
||||
endwhile (Done)
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
:GenerateBreweries(enriched_cities);
|
||||
|
||||
|#E5EDE1|DataGenerator|
|
||||
while (For each EnrichedCity?) is (Remaining cities)
|
||||
if (Generator Mode) then (MockGenerator)
|
||||
:DeterministicHash & Format;
|
||||
elseif (Generator Mode) then (LlamaGenerator)
|
||||
:PrepareRegionContext;
|
||||
:prompt_directory_->Load("BREWERY_GENERATION");
|
||||
note right
|
||||
Resolves to BREWERY_GENERATION.md inside --prompt-dir.
|
||||
end note
|
||||
repeat
|
||||
:Infer(system_prompt, user_prompt, max_tokens, kBreweryJsonGrammar);
|
||||
:ValidateBreweryJson(raw, brewery);
|
||||
if (Is JSON Valid?) then (yes)
|
||||
break
|
||||
else (no)
|
||||
:Attempt++;
|
||||
:Append validation error to retry prompt;
|
||||
endif
|
||||
repeat while (Attempt < 3?) is (yes)
|
||||
note right
|
||||
max_tokens is fixed across retries —
|
||||
it is not raised on truncation.
|
||||
end note
|
||||
else (OpenAIGenerator)
|
||||
:PrepareRegionContext;
|
||||
:prompt_directory_->Load("BREWERY_GENERATION");
|
||||
note right
|
||||
Same prompt file as LlamaGenerator - content
|
||||
is backend-agnostic.
|
||||
end note
|
||||
repeat
|
||||
:CallChatCompletionsApi(system_prompt, user_prompt,\nkBreweryJsonSchema, max_tokens);
|
||||
:ValidateBreweryJson(raw, brewery);
|
||||
if (Is JSON Valid?) then (yes)
|
||||
break
|
||||
else (no)
|
||||
:Attempt++;
|
||||
:Append validation error to retry prompt;
|
||||
endif
|
||||
repeat while (Attempt < 2?) is (yes)
|
||||
note right
|
||||
Structured Outputs already guarantees schema
|
||||
shape, so only 2 attempts are needed - retries
|
||||
here cover transient failures / placeholder
|
||||
text, not malformed JSON.
|
||||
end note
|
||||
endif
|
||||
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
if (Generation successful?) then (yes)
|
||||
:GetRandomCoordsWithinRange(city_centre, 5km, rng);
|
||||
note right
|
||||
Generation + coordinate jitter run inside one
|
||||
try/catch per record. BreweryRecord.address
|
||||
now carries {city, longitude, latitude}.
|
||||
end note
|
||||
|
||||
|#E0EAE0|SqliteExportService|
|
||||
:ProcessRecord(GeneratedBrewery);
|
||||
if (Location in cache?) then (yes)
|
||||
:Reuse location_id;
|
||||
else (no)
|
||||
:Insert Location & Cache ID;
|
||||
endif
|
||||
:Insert Brewery (FK: location_id);
|
||||
|
||||
if (Exception caught during insert?) then (yes)
|
||||
|#EAF0E8|BiergartenPipelineOrchestrator|
|
||||
:Log warning "SQLite export failed";
|
||||
note right
|
||||
Data loss is prevented per-record.
|
||||
The pipeline continues running.
|
||||
end note
|
||||
else (no)
|
||||
endif
|
||||
else (no)
|
||||
:Log warning "Generation failed, skipping...";
|
||||
endif
|
||||
|#E5EDE1|DataGenerator|
|
||||
endwhile (Done)
|
||||
|
||||
|#E0EAE0|SqliteExportService|
|
||||
:Finalize();
|
||||
note right
|
||||
Commits Transaction
|
||||
Closes Database Connection
|
||||
end note
|
||||
|
||||
|#F2F6F0|main.cc|
|
||||
:Close log_channel;
|
||||
:Join log dispatcher thread;
|
||||
:Return 0;
|
||||
stop
|
||||
|
||||
@enduml
|
||||
@@ -1,282 +0,0 @@
|
||||
@startuml
|
||||
!include ../biergarten-weizen-theme.puml
|
||||
skinparam style strictuml
|
||||
skinparam defaultFontSize 14
|
||||
skinparam titleFontSize 20
|
||||
|
||||
title The Biergarten Data Pipeline - Class Diagram
|
||||
|
||||
class BiergartenPipelineOrchestrator {
|
||||
- logger_ : std::shared_ptr<ILogger>
|
||||
- context_service_ : std::unique_ptr<IEnrichmentService>
|
||||
- generator_ : std::unique_ptr<DataGenerator>
|
||||
- exporter_ : std::unique_ptr<IExportService>
|
||||
- curated_data_service_ : std::unique_ptr<ICuratedDataService>
|
||||
- application_options_ : ApplicationOptions
|
||||
- generated_breweries_ : std::vector<BreweryRecord>
|
||||
- generated_users_ : std::vector<UserRecord>
|
||||
+ Run() : bool
|
||||
- QueryLocations() : std::vector<City>
|
||||
- GenerateBreweries(cities : std::span<const EnrichedCity>) : void
|
||||
- GenerateUsers(cities : std::span<const EnrichedCity>) : void
|
||||
- LogResults() const : void
|
||||
}
|
||||
|
||||
note right of BiergartenPipelineOrchestrator
|
||||
GenerateBreweries and GenerateUsers both call the
|
||||
free function GetRandomCoordsWithinRange()
|
||||
(biergarten_pipeline_orchestrator/coords.h) to jitter
|
||||
a coordinate pair uniformly within 5km of the city
|
||||
centre for each generated record's address.
|
||||
end note
|
||||
|
||||
class LogLevel <<enumeration>> {
|
||||
Debug
|
||||
Info
|
||||
Warn
|
||||
Error
|
||||
}
|
||||
|
||||
class PipelinePhase <<enumeration>> {
|
||||
Startup
|
||||
Enrichment
|
||||
UserGeneration
|
||||
BreweryAndBeerGeneration
|
||||
CheckinGeneration
|
||||
RatingGeneration
|
||||
FollowGeneration
|
||||
Teardown
|
||||
}
|
||||
|
||||
struct LogDTO {
|
||||
+ level : LogLevel
|
||||
+ phase : PipelinePhase
|
||||
+ message : std::string
|
||||
}
|
||||
|
||||
struct LogEntry {
|
||||
+ timestamp : std::chrono::system_clock::time_point
|
||||
+ origin : std::source_location
|
||||
+ thread_id : std::thread::id
|
||||
+ level : LogLevel
|
||||
+ phase : PipelinePhase
|
||||
+ message : std::string
|
||||
}
|
||||
|
||||
interface ILogger <<interface>> {
|
||||
+ Log(payload : LogDTO) : void
|
||||
- {abstract} DoLog(entry : LogEntry) : void
|
||||
}
|
||||
|
||||
class LogProducer {
|
||||
- channel_ : BoundedChannel<LogEntry>&
|
||||
- DoLog(entry : LogEntry) : void
|
||||
}
|
||||
|
||||
class LogDispatcher {
|
||||
- channel_ : BoundedChannel<LogEntry>&
|
||||
+ Run() : void
|
||||
- ToSpdlogLevel(level) : spdlog::level::level_enum
|
||||
}
|
||||
|
||||
interface IEnrichmentService <<interface>> {
|
||||
+ GetLocationContext(loc : const Location&) : std::string
|
||||
}
|
||||
|
||||
class MockEnrichmentService {
|
||||
+ GetLocationContext(loc : const Location&) : std::string
|
||||
}
|
||||
|
||||
class WikipediaEnrichmentService {
|
||||
- client_ : std::unique_ptr<WebClient>
|
||||
- extract_cache_ : std::unordered_map<std::string, std::string>
|
||||
+ GetLocationContext(loc : const Location&) : std::string
|
||||
- FetchExtract(query : std::string_view) : std::string
|
||||
}
|
||||
|
||||
interface WebClient <<interface>> {
|
||||
+ Get(url : const std::string&) : std::string
|
||||
+ UrlEncode(value : const std::string&) : std::string
|
||||
}
|
||||
|
||||
class HttpWebClient {
|
||||
+ Get(url : const std::string&) : std::string
|
||||
+ UrlEncode(value : const std::string&) : std::string
|
||||
}
|
||||
|
||||
interface DataGenerator <<interface>> {
|
||||
+ GenerateBrewery(city : const EnrichedCity&) : BreweryResult
|
||||
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
|
||||
}
|
||||
|
||||
class MockGenerator {
|
||||
+ GenerateBrewery(city : const EnrichedCity&) : BreweryResult
|
||||
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
|
||||
- DeterministicHash(location : const Location&) : size_t
|
||||
- DeterministicHash(location : const Location&, persona : const UserPersona&, name : const Name&) : size_t
|
||||
}
|
||||
|
||||
class LlamaGenerator {
|
||||
- model_ : ModelHandle
|
||||
- context_ : ContextHandle
|
||||
- prompt_formatter_ : std::unique_ptr<IPromptFormatter>
|
||||
- prompt_directory_ : std::unique_ptr<IPromptDirectory>
|
||||
- rng_ : std::mt19937
|
||||
+ GenerateBrewery(...) : BreweryResult
|
||||
+ GenerateUser(...) : UserResult
|
||||
- Load(model_path : const std::string&) : void
|
||||
- Infer(...) : std::string
|
||||
- InferFormatted(...) : std::string
|
||||
}
|
||||
|
||||
class OpenAIGenerator {
|
||||
- api_key_ : std::string
|
||||
- model_ : std::string
|
||||
- logger_ : std::shared_ptr<ILogger>
|
||||
- prompt_directory_ : std::unique_ptr<IPromptDirectory>
|
||||
- web_client_ : std::unique_ptr<WebClient>
|
||||
+ GenerateBrewery(city : const EnrichedCity&) : BreweryResult
|
||||
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
|
||||
- CallChatCompletionsApi(...) : std::string
|
||||
}
|
||||
|
||||
note right of OpenAIGenerator
|
||||
Chat Completions with Structured Outputs
|
||||
(response_format: json_schema, strict: true)
|
||||
guarantees schema-valid JSON, replacing
|
||||
llama.cpp's GBNF grammar constraint -
|
||||
no IPromptFormatter needed.
|
||||
end note
|
||||
|
||||
interface IPromptFormatter <<interface>> {
|
||||
+ Format(system_prompt : std::string_view, user_prompt : std::string_view) : std::string
|
||||
}
|
||||
|
||||
class Gemma4JinjaPromptFormatter {
|
||||
+ Format(system_prompt : std::string_view, user_prompt : std::string_view) : std::string
|
||||
}
|
||||
|
||||
interface IPromptDirectory <<interface>> {
|
||||
+ Load(key : std::string_view) : std::string
|
||||
}
|
||||
|
||||
class PromptDirectory {
|
||||
- prompt_dir_ : std::filesystem::path
|
||||
- cache_ : std::unordered_map<std::string, std::string>
|
||||
+ Load(key : std::string_view) : std::string
|
||||
}
|
||||
|
||||
interface ICuratedDataService <<interface>> {
|
||||
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
|
||||
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
|
||||
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
|
||||
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
|
||||
}
|
||||
|
||||
class JsonLoader {
|
||||
- cache_ : cache
|
||||
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
|
||||
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
|
||||
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
|
||||
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
|
||||
}
|
||||
|
||||
note right of JsonLoader
|
||||
Each Load* method memoizes its result in
|
||||
cache_ on first call; later calls on the
|
||||
same instance return the cached value
|
||||
without re-parsing.
|
||||
end note
|
||||
|
||||
class MockCuratedDataService {
|
||||
- locations_ : std::vector<Location>
|
||||
- personas_ : std::vector<UserPersona>
|
||||
- forenames_by_country_ : std::unordered_map<std::string, forename_list>
|
||||
- surnames_by_country_ : std::unordered_map<std::string, surname_list>
|
||||
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
|
||||
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
|
||||
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
|
||||
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
|
||||
}
|
||||
|
||||
note right of MockCuratedDataService
|
||||
Fixed 4-location / 3-persona dataset
|
||||
(US, DE, FR, BE) used in --mocked runs;
|
||||
filepath arguments are ignored.
|
||||
end note
|
||||
|
||||
interface IExportService <<interface>> {
|
||||
+ Initialize() : void
|
||||
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
|
||||
+ ProcessRecord(user : const UserRecord&) : uint64_t
|
||||
+ Finalize() : void
|
||||
}
|
||||
|
||||
class SqliteExportService {
|
||||
- date_time_provider_ : std::unique_ptr<IDateTimeProvider>
|
||||
- run_timestamp_utc_ : std::string
|
||||
- database_path_ : std::filesystem::path
|
||||
- db_handle_ : SqliteDatabaseHandle
|
||||
- insert_location_stmt_ : SqliteStatementHandle
|
||||
- insert_brewery_stmt_ : SqliteStatementHandle
|
||||
- insert_user_stmt_ : SqliteStatementHandle
|
||||
- transaction_open_ : bool
|
||||
- location_cache_ : std::unordered_map<std::string, sqlite3_int64>
|
||||
+ Initialize() : void
|
||||
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
|
||||
+ ProcessRecord(user : const UserRecord&) : uint64_t
|
||||
+ Finalize() : void
|
||||
- InitializeSchema() : void
|
||||
- ResolveLocationId(location : const Location&) : sqlite3_int64
|
||||
}
|
||||
|
||||
interface IDateTimeProvider <<interface>> {
|
||||
+ GetUtcTimestamp() : std::string
|
||||
}
|
||||
|
||||
class SystemDateTimeProvider {
|
||||
+ GetUtcTimestamp() : std::string
|
||||
}
|
||||
|
||||
' Structural Relationships / Dependency Injection
|
||||
BiergartenPipelineOrchestrator *-- ILogger : owns
|
||||
BiergartenPipelineOrchestrator *-- IEnrichmentService : owns
|
||||
BiergartenPipelineOrchestrator *-- DataGenerator : owns
|
||||
BiergartenPipelineOrchestrator *-- IExportService : owns
|
||||
BiergartenPipelineOrchestrator *-- ICuratedDataService : owns
|
||||
|
||||
LogEntry *-- LogLevel
|
||||
LogEntry *-- PipelinePhase
|
||||
LogDTO *-- LogLevel
|
||||
LogDTO *-- PipelinePhase
|
||||
ILogger <|.. LogProducer : implements
|
||||
LogProducer ..> LogEntry : emits
|
||||
LogDispatcher ..> LogEntry : consumes
|
||||
|
||||
IEnrichmentService <|.. WikipediaEnrichmentService : implements
|
||||
IEnrichmentService <|.. MockEnrichmentService : implements
|
||||
WikipediaEnrichmentService *-- WebClient : owns
|
||||
|
||||
WebClient <|.. HttpWebClient : implements
|
||||
|
||||
DataGenerator <|.. MockGenerator : implements
|
||||
DataGenerator <|.. LlamaGenerator : implements
|
||||
DataGenerator <|.. OpenAIGenerator : implements
|
||||
|
||||
LlamaGenerator *-- IPromptFormatter : uses
|
||||
LlamaGenerator *-- IPromptDirectory : uses
|
||||
|
||||
OpenAIGenerator *-- IPromptDirectory : uses
|
||||
OpenAIGenerator *-- WebClient : uses
|
||||
|
||||
IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements
|
||||
IPromptDirectory <|.. PromptDirectory : implements
|
||||
|
||||
ICuratedDataService <|.. JsonLoader : implements
|
||||
ICuratedDataService <|.. MockCuratedDataService : implements
|
||||
|
||||
IExportService <|.. SqliteExportService : implements
|
||||
SqliteExportService *-- IDateTimeProvider : owns
|
||||
IDateTimeProvider <|.. SystemDateTimeProvider : implements
|
||||
|
||||
@enduml
|
||||
@@ -1,372 +0,0 @@
|
||||
@startuml biergarten_activity
|
||||
!include ../biergarten-weizen-theme.puml
|
||||
skinparam defaultFontSize 13
|
||||
skinparam titleFontSize 20
|
||||
|
||||
title The Biergarten Data Pipeline — Activity Diagram
|
||||
|
||||
|Main|
|
||||
start
|
||||
:ParseArguments(argc, argv);
|
||||
if (Invalid args?) then (yes)
|
||||
:spdlog::error;
|
||||
stop
|
||||
else (no)
|
||||
endif
|
||||
:Init OpenSSL global state & LlamaBackendState;
|
||||
:Build DI injector;
|
||||
|
||||
:Initialize SqliteExportService;
|
||||
note right
|
||||
Opens SQLite connection.
|
||||
(Transactions are now managed
|
||||
per-phase via batching).
|
||||
end note
|
||||
|
||||
:Create BoundedChannel<LogEntry> log_ch;
|
||||
:Spawn Log Worker thread;
|
||||
note right
|
||||
Log worker drains log_ch for the
|
||||
entire pipeline lifetime.
|
||||
All workers emit LogEntry structs
|
||||
via PipelineLogger -- never spdlog directly.
|
||||
end note
|
||||
|
||||
:BiergartenPipelineOrchestrator::Run();
|
||||
|BiergartenPipelineOrchestrator::Run()|
|
||||
|
||||
fork
|
||||
:JsonLoader::LoadBeerStyles("beer-styles.json");
|
||||
:EnrichmentService::PreWarmBeerStyleCache(beer_styles);
|
||||
fork again
|
||||
:JsonLoader::LoadLocations("locations.json");
|
||||
:EnrichmentService::PreWarmLocationCache(sampled_locations);
|
||||
end fork
|
||||
fork
|
||||
:JsonLoader::LoadForenamesByCountry(\n "forenames-by-country.json");
|
||||
fork again
|
||||
:JsonLoader::LoadSurnamesByCountry(\n "surnames-by-country.json");
|
||||
fork again
|
||||
:JsonLoader::LoadPersonas("personas.json");
|
||||
end fork
|
||||
note right
|
||||
Each call is memoized after its first parse
|
||||
(implemented today). MockCuratedDataService
|
||||
returns a fixed in-memory dataset instead
|
||||
under --mocked, skipping these JSON files.
|
||||
end note
|
||||
|
||||
' ═══════════════════════════════════════════
|
||||
' PHASE 0 — USER GENERATION
|
||||
' ═══════════════════════════════════════════
|
||||
|Orchestrator|
|
||||
:RunUserPhase(sampled_locations);
|
||||
:Create BoundedChannels\n(loc_ch, exp_ch);
|
||||
|
||||
fork
|
||||
|Orchestrator|
|
||||
:Loop: Send Locations -> loc_ch;
|
||||
:Close loc_ch;
|
||||
note right
|
||||
Producer closes loc_ch.
|
||||
LLM Worker while loop
|
||||
terminates on empty + closed.
|
||||
end note
|
||||
fork again
|
||||
|LLM Worker|
|
||||
while (loc_ch has items?) is (yes)
|
||||
:Receive Location;
|
||||
|
||||
:GetLocationContextFromCache(location);
|
||||
note right
|
||||
Guaranteed cache hit from startup.
|
||||
end note
|
||||
|
||||
:IPersonaSelectionStrategy::SelectPersona(\n personas_palette_);
|
||||
note right
|
||||
Guaranteed cache hit from startup.
|
||||
Returns a Persona struct carrying
|
||||
style_affinities, abv_range,
|
||||
ibu_preference, checkin_weight.
|
||||
end note
|
||||
|
||||
:SampleName(forenames_by_country,\n surnames_by_country, location.iso3166_1, rng);
|
||||
note right
|
||||
Deterministic lookup -- no LLM involved.
|
||||
Free function (implemented today in
|
||||
generate_users.cc), not a class method.
|
||||
Name selected from pre-keyed maps
|
||||
and passed into the generation prompt.
|
||||
Skips the city if either map has no
|
||||
entry for iso3166_1.
|
||||
end note
|
||||
|
||||
:GenerateUser(enriched_city, persona, sampled_name)\nvia DataGenerator;
|
||||
note right
|
||||
LLM receives: EnrichedCity context + persona
|
||||
description + sampled name. Generates
|
||||
bio and preference signals grounded
|
||||
in locale and persona.
|
||||
end note
|
||||
|
||||
:PipelineLogger::Log(Info, UserGeneration,\n city, user_id, "llm");
|
||||
:Send GeneratedUser -> exp_ch;
|
||||
endwhile (no)
|
||||
:Close exp_ch;
|
||||
note right
|
||||
Producer closes exp_ch.
|
||||
SQLite Worker while loop
|
||||
terminates on empty + closed.
|
||||
end note
|
||||
fork again
|
||||
|SQLite Worker|
|
||||
:BEGIN TRANSACTION;
|
||||
while (exp_ch has items?) is (yes)
|
||||
:Receive GeneratedUser;
|
||||
:ProcessUser(user);
|
||||
:PipelineLogger::Log(Info, UserGeneration,\n city, user_id, "sqlite");
|
||||
:Append -> user_pool_;
|
||||
if (Batch size reached?) then (yes)
|
||||
:COMMIT & BEGIN;
|
||||
else (no)
|
||||
endif
|
||||
endwhile (no)
|
||||
:COMMIT (Final);
|
||||
end fork
|
||||
|
||||
|Orchestrator|
|
||||
:Join LLM Worker, SQLite Worker;
|
||||
|
||||
' ═══════════════════════════════════════════
|
||||
' PHASE 1a — BREWERY GENERATION
|
||||
' ═══════════════════════════════════════════
|
||||
:RunBreweryPhase(sampled_locations);
|
||||
:Create BoundedChannels\n(loc_ch, exp_ch);
|
||||
|
||||
fork
|
||||
|Orchestrator|
|
||||
:Loop: Sample User from user_pool_
|
||||
and pair with Location;
|
||||
:Send BreweryTask(Location, User) -> loc_ch;
|
||||
:Close loc_ch;
|
||||
fork again
|
||||
|LLM Worker|
|
||||
while (loc_ch has items?) is (yes)
|
||||
:Receive BreweryTask(Location, User);
|
||||
|
||||
:GetLocationContextFromCache(task.location);
|
||||
note right
|
||||
Guaranteed cache hit from startup.
|
||||
end note
|
||||
|
||||
:GenerateBrewery(enriched_city, context, task.user)\nvia DataGenerator;
|
||||
note right
|
||||
KV cache stays warm.
|
||||
Brewery is linked to the sampled owner_user_id.
|
||||
end note
|
||||
:PipelineLogger::Log(Info,\n BreweryGeneration,\n city, brewery_id, "llm");
|
||||
:Send GeneratedBrewery -> exp_ch;
|
||||
endwhile (no)
|
||||
:Close exp_ch;
|
||||
fork again
|
||||
|SQLite Worker|
|
||||
:BEGIN TRANSACTION;
|
||||
while (exp_ch has items?) is (yes)
|
||||
:Receive GeneratedBrewery;
|
||||
:ProcessBrewery(brewery);
|
||||
:PipelineLogger::Log(Info,\n BreweryGeneration,\n city, brewery_id, "sqlite");
|
||||
:Append -> brewery_pool_;
|
||||
if (Batch size reached?) then (yes)
|
||||
:COMMIT & BEGIN;
|
||||
else (no)
|
||||
endif
|
||||
endwhile (no)
|
||||
:COMMIT (Final);
|
||||
end fork
|
||||
|
||||
|Orchestrator|
|
||||
:Join LLM Worker, SQLite Worker;
|
||||
note right
|
||||
brewery_pool_ is now fully populated.
|
||||
Phase 1b may begin.
|
||||
end note
|
||||
|
||||
' ═══════════════════════════════════════════
|
||||
' PHASE 1b — BEER GENERATION
|
||||
' ═══════════════════════════════════════════
|
||||
:RunBeerPhase();
|
||||
:Create BoundedChannels\n(brew_ch, exp_ch);
|
||||
|
||||
fork
|
||||
|Orchestrator|
|
||||
:Loop: Send Breweries -> brew_ch;
|
||||
:Close brew_ch;
|
||||
fork again
|
||||
|LLM Worker|
|
||||
while (brew_ch has items?) is (yes)
|
||||
:Receive GeneratedBrewery;
|
||||
:IBeerSelectionStrategy::SelectStyles(\n brewery, beer_style_palette_);
|
||||
|
||||
while (For each selected BeerStyle?) is (remaining)
|
||||
:GetStyleContextFromCache(style);
|
||||
note right
|
||||
Guaranteed cache hit from startup.
|
||||
KV cache stays warm across all
|
||||
beer generations -- system prompt
|
||||
does not change within this phase.
|
||||
end note
|
||||
:GenerateBeer(brewery, style_context)\nvia DataGenerator;
|
||||
:Attach GeneratedBeer to bundle;
|
||||
endwhile (done)
|
||||
|
||||
:PipelineLogger::Log(Info,\n BeerGeneration,\n city, brewery_id, "llm");
|
||||
:Send BeersBundle -> exp_ch;
|
||||
endwhile (no)
|
||||
:Close exp_ch;
|
||||
fork again
|
||||
|SQLite Worker|
|
||||
:BEGIN TRANSACTION;
|
||||
while (exp_ch has items?) is (yes)
|
||||
:Receive BeersBundle;
|
||||
while (For each beer in bundle?) is (remaining)
|
||||
:Set beer.brewery_id from bundle;
|
||||
:ProcessBeer(beer);
|
||||
:Append -> beer_pool_;
|
||||
endwhile (done)
|
||||
:PipelineLogger::Log(Info,\n BeerGeneration,\n city, brewery_id, "sqlite");
|
||||
if (Batch size reached?) then (yes)
|
||||
:COMMIT & BEGIN;
|
||||
else (no)
|
||||
endif
|
||||
endwhile (no)
|
||||
:COMMIT (Final);
|
||||
end fork
|
||||
|
||||
|Orchestrator|
|
||||
:Join LLM Worker, SQLite Worker;
|
||||
note right
|
||||
Both brewery_pool_ and beer_pool_
|
||||
are now completely populated.
|
||||
Checkin and Follow phases may
|
||||
now run in parallel.
|
||||
end note
|
||||
|
||||
' ═══════════════════════════════════════════
|
||||
' PHASE 2 — CHECKIN + FOLLOW GENERATION
|
||||
' (parallel — both depend only on user_pool_
|
||||
' and brewery_pool_ being fully populated)
|
||||
' ═══════════════════════════════════════════
|
||||
fork
|
||||
|Orchestrator|
|
||||
:RunCheckinPhase();
|
||||
:ICheckinDistributionStrategy::\nAssignActivityWeights(user_pool_);
|
||||
note right
|
||||
Weights seeded from each user's
|
||||
persona.checkin_weight. J-curve profile
|
||||
emerges from persona distribution.
|
||||
end note
|
||||
|
||||
:BEGIN TRANSACTION;
|
||||
while (For each GeneratedUser in user_pool_?) is (remaining)
|
||||
:CheckinsForUser(user, brewery_pool_.size());
|
||||
while (For each checkin index?) is (remaining)
|
||||
:TimestampFor(user, index);
|
||||
:Select brewery from brewery_pool_;
|
||||
:GenerateCheckin(user, brewery, timestamp)\nvia DataGenerator;
|
||||
:ProcessCheckin(checkin);
|
||||
:PipelineLogger::Log(Info, CheckinGeneration,\n nullopt, checkin_id, "sqlite");
|
||||
:Append -> checkin_pool_;
|
||||
if (Batch size reached?) then (yes)
|
||||
:COMMIT & BEGIN;
|
||||
else (no)
|
||||
endif
|
||||
endwhile (done)
|
||||
endwhile (done)
|
||||
:COMMIT (Final);
|
||||
|
||||
fork again
|
||||
|Orchestrator|
|
||||
:RunFollowPhase();
|
||||
:IFollowGenerationStrategy::\nAssignFollowWeights(user_pool_);
|
||||
note right
|
||||
For RandomFollowStrategy, weights
|
||||
are uniform. For ActivityWeightedFollowStrategy,
|
||||
weights derived from user.activity_weight
|
||||
so high-activity users attract more followers.
|
||||
end note
|
||||
|
||||
:BEGIN TRANSACTION;
|
||||
:IFollowGenerationStrategy::\nGenerateFollows(user_pool_);
|
||||
note right
|
||||
Self-follow constraint (follower_id != followed_id)
|
||||
enforced here and at the DB schema level.
|
||||
end note
|
||||
while (For each GeneratedFollow?) is (remaining)
|
||||
:ProcessFollow(follow);
|
||||
:PipelineLogger::Log(Info, FollowGeneration,\n nullopt, follower_id, "sqlite");
|
||||
:Append -> follow_pool_;
|
||||
if (Batch size reached?) then (yes)
|
||||
:COMMIT & BEGIN;
|
||||
else (no)
|
||||
endif
|
||||
endwhile (done)
|
||||
:COMMIT (Final);
|
||||
|
||||
end fork
|
||||
|
||||
|Orchestrator|
|
||||
:Join CheckinPhase, FollowPhase;
|
||||
note right
|
||||
checkin_pool_ and follow_pool_
|
||||
are now fully populated.
|
||||
Rating phase may begin.
|
||||
end note
|
||||
|
||||
' ═══════════════════════════════════════════
|
||||
' PHASE 3 — RATING GENERATION
|
||||
' ═══════════════════════════════════════════
|
||||
:RunRatingPhase();
|
||||
note right
|
||||
Beer selection biased by
|
||||
user.persona.style_affinities and abv_range.
|
||||
Rating skew modulated per persona.
|
||||
end note
|
||||
|
||||
:BEGIN TRANSACTION;
|
||||
while (For each GeneratedCheckin in checkin_pool_?) is (remaining)
|
||||
:Match brewery_id, select beer from beer_pool_\n(same brewery_id, biased by persona affinities);
|
||||
if (Beer exists for brewery?) then (yes)
|
||||
:GenerateRating(user, beer, checkin_id)\nvia DataGenerator;
|
||||
:ProcessRating(rating);
|
||||
:PipelineLogger::Log(Info, RatingGeneration,\n nullopt, rating_id, "sqlite");
|
||||
if (Batch size reached?) then (yes)
|
||||
:COMMIT & BEGIN;
|
||||
else (no)
|
||||
endif
|
||||
else (no)
|
||||
:PipelineLogger::Log(Warn, RatingGeneration,\n nullopt, brewery_id, "sqlite");
|
||||
:Skip -- brewery has no beers;
|
||||
endif
|
||||
endwhile (done)
|
||||
:COMMIT (Final);
|
||||
|
||||
' ═══════════════════════════════════════════
|
||||
' TEARDOWN
|
||||
' ═══════════════════════════════════════════
|
||||
|Orchestrator|
|
||||
:Finalize SqliteExportService;
|
||||
note right
|
||||
Safely closes the DB connection.
|
||||
end note
|
||||
:Close log_ch;
|
||||
|
||||
|Main|
|
||||
:spdlog::info "Pipeline complete in X ms";
|
||||
:Join Log Worker;
|
||||
note right
|
||||
Drain guarantees no LogEntry is
|
||||
dropped at shutdown.
|
||||
end note
|
||||
stop
|
||||
|
||||
@enduml
|
||||
@@ -1,646 +0,0 @@
|
||||
@startuml class_diagram
|
||||
|
||||
' ==========================================
|
||||
' CONFIGURATION & STYLING
|
||||
' ==========================================
|
||||
!include ../biergarten-weizen-theme.puml
|
||||
skinparam classAttributeFontSize 9
|
||||
skinparam defaultFontSize 25
|
||||
skinparam titleFontSize 30
|
||||
|
||||
title Biergarten Data Pipeline — Class Diagram
|
||||
|
||||
package "Domain: Models" {
|
||||
|
||||
class Location {
|
||||
+ city : std::string
|
||||
+ state_province : std::string
|
||||
+ iso3166_2 : std::string
|
||||
+ country : std::string
|
||||
+ iso3166_1 : std::string
|
||||
+ local_languages : std::vector<std::string>
|
||||
+ latitude : double
|
||||
+ longitude : double
|
||||
}
|
||||
|
||||
class LocationContext {
|
||||
+ text : std::string
|
||||
+ completeness : Completeness
|
||||
+ char_count : size_t
|
||||
}
|
||||
|
||||
enum Completeness {
|
||||
Full
|
||||
Partial
|
||||
Absent
|
||||
}
|
||||
|
||||
class EnrichedCity {
|
||||
+ location : Location
|
||||
+ context : LocationContext
|
||||
}
|
||||
|
||||
class BeerStyle {
|
||||
+ name : std::string
|
||||
+ description : std::string
|
||||
+ min_abv : float
|
||||
+ max_abv : float
|
||||
+ min_ibu : int
|
||||
+ max_ibu : int
|
||||
}
|
||||
|
||||
class BreweryResult {
|
||||
+ name_en : std::string
|
||||
+ description_en : std::string
|
||||
+ name_local : std::string
|
||||
+ description_local : std::string
|
||||
}
|
||||
|
||||
class BeerResult {
|
||||
+ name_en : std::string
|
||||
+ description_en : std::string
|
||||
+ name_local : std::string
|
||||
+ description_local : std::string
|
||||
+ style : std::string
|
||||
+ abv : float
|
||||
+ ibu : int
|
||||
}
|
||||
|
||||
class UserResult {
|
||||
+ first_name : std::string
|
||||
+ last_name : std::string
|
||||
+ gender : std::string
|
||||
+ username : std::string
|
||||
+ bio : std::string
|
||||
+ activity_weight : float
|
||||
}
|
||||
|
||||
class Name {
|
||||
+ first_name : std::string
|
||||
+ last_name : std::string
|
||||
+ gender : std::string
|
||||
}
|
||||
|
||||
class ForenameEntry {
|
||||
+ name : std::string
|
||||
+ gender : std::string
|
||||
}
|
||||
|
||||
note right of ForenameEntry
|
||||
forename_list = std::unordered_set<ForenameEntry>
|
||||
surname_list = std::unordered_set<std::string>
|
||||
(aliases declared in curated_data_service.h).
|
||||
ICuratedDataService::LoadForenamesByCountry() /
|
||||
LoadSurnamesByCountry() each return a flat
|
||||
std::unordered_map<std::string, forename_list>
|
||||
or <std::string, surname_list> keyed by ISO
|
||||
3166-1 country code -- no NamesByCountry
|
||||
wrapper class. SampleName(forenames_by_country,
|
||||
surnames_by_country, iso3166_1, rng) is a free
|
||||
function, not a method, so it can be called by
|
||||
any per-city worker without owning the maps.
|
||||
Sourced from popular-names-by-country-dataset
|
||||
(CC0) -- see ETHICS-AND-KNOWN-ISSUES.md. Gender
|
||||
is preserved per forename so it can be threaded
|
||||
through to persona/bio generation later; surnames
|
||||
carry no gender in the source data. Pairing
|
||||
happens at sample time, not at fixture-authoring
|
||||
time, so no information from the source dataset
|
||||
is discarded up front.
|
||||
end note
|
||||
|
||||
ForenameEntry ..> Name : SampleName() produces
|
||||
|
||||
class CheckinResult {
|
||||
+ checked_in_at : std::string
|
||||
+ note : std::string
|
||||
}
|
||||
|
||||
class RatingResult {
|
||||
+ score : float
|
||||
+ note : std::string
|
||||
}
|
||||
|
||||
class GenerationMetadata {
|
||||
+ generation_id : uint64_t
|
||||
+ generated_time : std::string
|
||||
+ context_provided : bool
|
||||
+ generated_with : std::string
|
||||
}
|
||||
|
||||
class GeneratedBrewery {
|
||||
+ brewery_id : uint64_t
|
||||
+ location : Location
|
||||
+ brewery : BreweryResult
|
||||
+ context_completeness : LocationContext::Completeness
|
||||
+ metadata : GenerationMetadata
|
||||
}
|
||||
|
||||
class GeneratedBeer {
|
||||
+ beer_id : uint64_t
|
||||
+ brewery_id : uint64_t
|
||||
+ location : Location
|
||||
+ style : BeerStyle
|
||||
+ beer : BeerResult
|
||||
+ metadata : GenerationMetadata
|
||||
}
|
||||
|
||||
class GeneratedUser {
|
||||
+ user_id : uint64_t
|
||||
+ location : Location
|
||||
+ user : UserResult
|
||||
+ email : std::string
|
||||
+ date_of_birth : std::string
|
||||
+ password : std::string
|
||||
+ metadata : GenerationMetadata
|
||||
}
|
||||
|
||||
note right of GeneratedUser
|
||||
email, date_of_birth, and password are
|
||||
programmatically generated by the orchestrator
|
||||
(not LLM-authored) so a downstream auth-account
|
||||
seeding consumer can register real accounts from
|
||||
this export. See ROADMAP.md §1.
|
||||
end note
|
||||
|
||||
class GeneratedCheckin {
|
||||
+ checkin_id : uint64_t
|
||||
+ user_id : uint64_t
|
||||
+ brewery_id : uint64_t
|
||||
+ checkin : CheckinResult
|
||||
+ metadata : GenerationMetadata
|
||||
}
|
||||
|
||||
class GeneratedRating {
|
||||
+ user_id : uint64_t
|
||||
+ beer_id : uint64_t
|
||||
+ checkin_id : uint64_t
|
||||
+ rating : RatingResult
|
||||
+ metadata : GenerationMetadata
|
||||
}
|
||||
|
||||
class GeneratedFollow {
|
||||
+ follower_id : uint64_t
|
||||
+ followed_id : uint64_t
|
||||
+ metadata : GenerationMetadata
|
||||
}
|
||||
|
||||
class UserPersona {
|
||||
+ name: std::string
|
||||
+ description: std::string
|
||||
+ style_affinities: std::vector<std::string>
|
||||
}
|
||||
|
||||
LocationContext *-- Completeness
|
||||
}
|
||||
|
||||
package "Domain: Application Configuration" {
|
||||
class SamplingOptions {
|
||||
+ temperature: float = 1.0F
|
||||
+ top_p: float = 0.95F
|
||||
+ top_k: uint32_t = 64
|
||||
+ n_ctx: uint32_t = 8192
|
||||
+ seed: int = -1
|
||||
}
|
||||
|
||||
class GeneratorOptions {
|
||||
+ model_path: std::filesystem::path
|
||||
+ use_mocked: bool = false
|
||||
+ sampling: std::optional<SamplingOptions>
|
||||
}
|
||||
|
||||
class PipelineOptions {
|
||||
+ output_path: std::filesystem::path
|
||||
+ log_path: std::filesystem::path
|
||||
}
|
||||
|
||||
class ApplicationOptions {
|
||||
+ generator: GeneratorOptions
|
||||
+ pipeline: PipelineOptions
|
||||
}
|
||||
|
||||
ApplicationOptions *-- GeneratorOptions
|
||||
ApplicationOptions *-- PipelineOptions
|
||||
GeneratorOptions o-- SamplingOptions
|
||||
}
|
||||
|
||||
package "Domain: Policy" {
|
||||
|
||||
interface ContextStrategy <<interface>> {
|
||||
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
||||
+ MaxContextChars() : size_t
|
||||
}
|
||||
|
||||
class BreweryContextStrategy {
|
||||
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
||||
+ MaxContextChars() : size_t
|
||||
}
|
||||
|
||||
class BeerContextStrategy {
|
||||
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
||||
+ MaxContextChars() : size_t
|
||||
}
|
||||
|
||||
interface SamplingStrategy <<interface>> {
|
||||
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
|
||||
}
|
||||
|
||||
class UniformSamplingStrategy {
|
||||
- sample_size_ : size_t
|
||||
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
|
||||
}
|
||||
|
||||
interface BeerSelectionStrategy <<interface>> {
|
||||
+ SelectStyles(brewery : const GeneratedBrewery&,\n palette : std::span<const BeerStyle>) : std::vector<BeerStyle>
|
||||
}
|
||||
|
||||
class RandomBeerSelectionStrategy {
|
||||
- rng_ : std::mt19937
|
||||
- min_beers_ : size_t
|
||||
- max_beers_ : size_t
|
||||
+ SelectStyles(brewery : const GeneratedBrewery&,\n palette : std::span<const BeerStyle>) : std::vector<BeerStyle>
|
||||
}
|
||||
|
||||
interface CheckinDistributionStrategy <<interface>> {
|
||||
+ AssignActivityWeights(users : std::vector<GeneratedUser>&) : void
|
||||
+ CheckinsForUser(user : const GeneratedUser&,\n brewery_count : size_t) : size_t
|
||||
+ TimestampFor(user : const GeneratedUser&,\n index : size_t) : std::string
|
||||
}
|
||||
|
||||
class JCurveCheckinStrategy {
|
||||
- rng_ : std::mt19937
|
||||
+ AssignActivityWeights(users : std::vector<GeneratedUser>&) : void
|
||||
+ CheckinsForUser(user : const GeneratedUser&,\n brewery_count : size_t) : size_t
|
||||
+ TimestampFor(user : const GeneratedUser&,\n index : size_t) : std::string
|
||||
}
|
||||
|
||||
class RandomCheckinStrategy {
|
||||
- rng_ : std::mt19937
|
||||
- min_checkins_ : size_t
|
||||
- max_checkins_ : size_t
|
||||
+ AssignActivityWeights(users : std::vector<GeneratedUser>&) : void
|
||||
+ CheckinsForUser(user : const GeneratedUser&,\n brewery_count : size_t) : size_t
|
||||
+ TimestampFor(user : const GeneratedUser&,\n index : size_t) : std::string
|
||||
}
|
||||
|
||||
interface FollowGenerationStrategy <<interface>> {
|
||||
+ GenerateFollows(users : const std::vector<GeneratedUser>&) : std::vector<GeneratedFollow>
|
||||
}
|
||||
|
||||
class RandomFollowStrategy {
|
||||
- rng_ : std::mt19937
|
||||
- min_follows_ : size_t
|
||||
- max_follows_ : size_t
|
||||
+ GenerateFollows(users : const std::vector<GeneratedUser>&) : std::vector<GeneratedFollow>
|
||||
}
|
||||
|
||||
class ActivityWeightedFollowStrategy {
|
||||
- rng_ : std::mt19937
|
||||
- min_follows_ : size_t
|
||||
- max_follows_ : size_t
|
||||
+ GenerateFollows(users : const std::vector<GeneratedUser>&) : std::vector<GeneratedFollow>
|
||||
}
|
||||
}
|
||||
|
||||
package "Infrastructure: Logging" {
|
||||
enum LogLevel {
|
||||
Debug
|
||||
Info
|
||||
Warn
|
||||
Error
|
||||
}
|
||||
|
||||
enum PipelinePhase {
|
||||
Startup
|
||||
UserGeneration
|
||||
BreweryAndBeerGeneration
|
||||
CheckinGeneration
|
||||
RatingGeneration
|
||||
FollowGeneration
|
||||
Teardown
|
||||
}
|
||||
|
||||
class LogEntry {
|
||||
+ timestamp : std::chrono::system_clock::time_point
|
||||
+ level : LogLevel
|
||||
+ phase : PipelinePhase
|
||||
+ message : std::string
|
||||
+ worker : std::optional<std::string>
|
||||
}
|
||||
|
||||
interface ILogger <<interface>> {
|
||||
+ Log(entry : const LogEntry&) : void
|
||||
}
|
||||
|
||||
class LogProducer {
|
||||
- channel_ : BoundedChannel<LogEntry>&
|
||||
+ Log(entry : const LogEntry&) : void
|
||||
}
|
||||
|
||||
class LogDispatcher {
|
||||
- channel_ : BoundedChannel<LogEntry>&
|
||||
+ Run() : void
|
||||
- ToSpdlogLevel(level) : spdlog::level::level_enum
|
||||
}
|
||||
|
||||
LogEntry *-- LogLevel
|
||||
LogEntry *-- PipelinePhase
|
||||
ILogger <|.. LogProducer
|
||||
LogProducer ..> LogEntry : emits
|
||||
LogDispatcher ..> LogEntry : consumes
|
||||
}
|
||||
|
||||
package "Infrastructure: Pipeline Channel" {
|
||||
|
||||
class "BoundedChannel<T>" as BoundedChannel {
|
||||
- queue_ : std::queue<T>
|
||||
- mutex_ : std::mutex
|
||||
- not_full_ : std::condition_variable
|
||||
- not_empty_ : std::condition_variable
|
||||
- capacity_ : size_t
|
||||
- closed_ : bool
|
||||
+ Send(item : T) : void
|
||||
+ Receive() : std::optional<T>
|
||||
+ Close() : void
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
package "Infrastructure: Data Preloading" {
|
||||
|
||||
interface ICuratedDataService <<interface>> {
|
||||
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
|
||||
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
|
||||
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
|
||||
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
|
||||
+ LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
|
||||
}
|
||||
|
||||
class JsonLoader {
|
||||
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
|
||||
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
|
||||
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
|
||||
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
|
||||
+ LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
|
||||
}
|
||||
|
||||
note right of JsonLoader
|
||||
Each Load* call is memoized after its first
|
||||
parse -- implemented today, ahead of the rest
|
||||
of this package (LoadBeerStyles is still planned).
|
||||
end note
|
||||
|
||||
class MockCuratedDataService {
|
||||
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
|
||||
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
|
||||
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
|
||||
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
|
||||
+ LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
|
||||
}
|
||||
|
||||
note right of MockCuratedDataService
|
||||
Fixed in-memory dataset used in --mocked mode,
|
||||
implemented today for Locations/Personas/
|
||||
Forenames/Surnames; would need a LoadBeerStyles
|
||||
fixture too once that method is implemented.
|
||||
end note
|
||||
|
||||
}
|
||||
|
||||
package "Infrastructure: Enrichment" {
|
||||
|
||||
interface EnrichmentService <<interface>> {
|
||||
+ GetLocationContext(loc : const Location&,\n strategy : const ContextStrategy&) : LocationContext
|
||||
}
|
||||
|
||||
class WikipediaService {
|
||||
- client_ : std::unique_ptr<WebClient>
|
||||
- extract_cache_ : std::unordered_map<std::string, std::string>
|
||||
+ GetLocationContext(loc : const Location&,\n strategy : const ContextStrategy&) : LocationContext
|
||||
- FetchExtract(query : std::string_view) : std::string
|
||||
}
|
||||
|
||||
interface WebClient <<interface>> {
|
||||
+ Get(url : const std::string&) : std::string
|
||||
+ UrlEncode(value : const std::string&) : std::string
|
||||
}
|
||||
|
||||
class HttpWebClient {
|
||||
+ Get(url : const std::string&) : std::string
|
||||
+ UrlEncode(value : const std::string&) : std::string
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
package "Infrastructure: Prompting" {
|
||||
|
||||
interface IPromptDirectory <<interface>> {
|
||||
+ Load(key : std::string_view) : std::string
|
||||
}
|
||||
|
||||
class PromptDirectory {
|
||||
- prompt_dir_ : std::filesystem::path
|
||||
- cache_ : std::unordered_map<std::string, std::string>
|
||||
+ PromptDirectory(prompt_dir : const std::filesystem::path&)
|
||||
+ Load(key : std::string_view) : std::string
|
||||
}
|
||||
|
||||
IPromptDirectory <|.. PromptDirectory
|
||||
}
|
||||
|
||||
package "Infrastructure: Data Generation" {
|
||||
|
||||
interface DataGenerator <<interface>> {
|
||||
+ GenerateBrewery(location : const Location&,\n context : const LocationContext&) : BreweryResult
|
||||
+ GenerateBeer(brewery_id : uint64_t,\n location : const Location&,\n context : const LocationContext&,\n style : const BeerStyle&) : BeerResult
|
||||
+ GenerateUser(city : const EnrichedCity&,\n persona : const UserPersona&,\n name : const Name&) : UserResult
|
||||
+ GenerateCheckin(user : const GeneratedUser&,\n brewery : const GeneratedBrewery&,\n timestamp : const std::string&) : CheckinResult
|
||||
+ GenerateRating(user : const GeneratedUser&,\n beer : const GeneratedBeer&,\n checkin_id : uint64_t) : RatingResult
|
||||
}
|
||||
|
||||
class MockGenerator {
|
||||
+ GenerateBrewery(...) : BreweryResult
|
||||
+ GenerateBeer(...) : BeerResult
|
||||
+ GenerateUser(...) : UserResult
|
||||
+ GenerateCheckin(...) : CheckinResult
|
||||
+ GenerateRating(...) : RatingResult
|
||||
- DeterministicHash(location : const Location&) : size_t
|
||||
}
|
||||
|
||||
class LlamaGenerator {
|
||||
- model_ : ModelHandle
|
||||
- context_ : ContextHandle
|
||||
- prompt_formatter_ : std::unique_ptr<PromptFormatter>
|
||||
- prompt_directory_ : std::unique_ptr<IPromptDirectory>
|
||||
- rng_ : std::mt19937
|
||||
+ GenerateBrewery(...) : BreweryResult
|
||||
+ GenerateBeer(...) : BeerResult
|
||||
+ GenerateUser(...) : UserResult
|
||||
+ GenerateCheckin(...) : CheckinResult
|
||||
+ GenerateRating(...) : RatingResult
|
||||
- Load(opts : const GeneratorOptions&) : void
|
||||
- Infer(system_prompt, user_prompt,\n max_tokens, grammar) : std::string
|
||||
- ValidateModelArchitecture() : void
|
||||
}
|
||||
|
||||
interface PromptFormatter <<interface>> {
|
||||
+ Format(system_prompt : std::string_view,\n user_prompt : std::string_view) : std::string
|
||||
+ ExpectedArchitecture() : std::string_view
|
||||
}
|
||||
|
||||
class Gemma4JinjaPromptFormatter {
|
||||
+ Format(...) : std::string
|
||||
+ ExpectedArchitecture() : std::string_view
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
package "Infrastructure: Data Export" {
|
||||
|
||||
interface ExportService <<interface>> {
|
||||
+ Initialize() : void
|
||||
+ ProcessBrewery(brewery : const GeneratedBrewery&) : uint64_t
|
||||
+ ProcessBeer(beer : const GeneratedBeer&) : uint64_t
|
||||
+ ProcessUser(user : const GeneratedUser&) : uint64_t
|
||||
+ ProcessCheckin(checkin : const GeneratedCheckin&) : uint64_t
|
||||
+ ProcessRating(rating : const GeneratedRating&) : void
|
||||
+ ProcessFollow(follow : const GeneratedFollow&) : void
|
||||
+ Finalize() : void
|
||||
}
|
||||
|
||||
class SqliteExportService {
|
||||
- date_time_provider_ : std::unique_ptr<DateTimeProvider>
|
||||
- db_handle_ : SqliteDatabaseHandle
|
||||
- insert_location_stmt_ : SqliteStatementHandle
|
||||
- insert_brewery_stmt_ : SqliteStatementHandle
|
||||
- insert_beer_stmt_ : SqliteStatementHandle
|
||||
- insert_user_stmt_ : SqliteStatementHandle
|
||||
- insert_checkin_stmt_ : SqliteStatementHandle
|
||||
- insert_rating_stmt_ : SqliteStatementHandle
|
||||
- insert_follow_stmt_ : SqliteStatementHandle
|
||||
- transaction_open_ : bool
|
||||
- location_cache_ : std::unordered_map<std::string, uint64_t>
|
||||
- brewery_cache_ : std::unordered_map<std::string, uint64_t>
|
||||
+ Initialize() : void
|
||||
+ ProcessRecord(brewery : const GeneratedBrewery&) : uint64_t
|
||||
+ ProcessRecord(beer : const GeneratedBeer&) : uint64_t
|
||||
+ ProcessRecord(user : const GeneratedUser&) : uint64_t
|
||||
+ ProcessRecord(checkin : const GeneratedCheckin&) : uint64_t
|
||||
+ ProcessRecord(rating : const GeneratedRating&) : void
|
||||
+ ProcessRecord(follow : const GeneratedFollow&) : void
|
||||
+ Finalize() : void
|
||||
- InitializeSchema() : void
|
||||
- PrepareStatements() : void
|
||||
- RollbackAndCloseNoThrow() : void
|
||||
- FinalizeStatements() : void
|
||||
}
|
||||
|
||||
interface DateTimeProvider <<interface>> {
|
||||
+ GetUtcTimestamp() : std::string
|
||||
}
|
||||
|
||||
class SystemDateTimeProvider {
|
||||
+ GetUtcTimestamp() : std::string
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class BiergartenPipelineOrchestrator {
|
||||
- curated_data_service_ : std::unique_ptr<ICuratedDataService>
|
||||
- enrichment_service_ : std::unique_ptr<EnrichmentService>
|
||||
- generator_ : std::unique_ptr<DataGenerator>
|
||||
- logger_ : std::unique_ptr<Logger>
|
||||
- exporter_ : std::unique_ptr<ExportService>
|
||||
- brewery_context_strategy_ : std::unique_ptr<ContextStrategy>
|
||||
- sampling_strategy_ : std::unique_ptr<SamplingStrategy>
|
||||
- beer_selection_strategy_ : std::unique_ptr<BeerSelectionStrategy>
|
||||
- checkin_strategy_ : std::unique_ptr<CheckinDistributionStrategy>
|
||||
- follow_strategy_ : std::unique_ptr<FollowGenerationStrategy>
|
||||
- beer_style_palette_ : std::vector<BeerStyle>
|
||||
- options_ : ApplicationOptions
|
||||
--
|
||||
- user_pool_ : std::vector<GeneratedUser>
|
||||
- brewery_pool_ : std::vector<GeneratedBrewery>
|
||||
- beer_pool_ : std::vector<GeneratedBeer>
|
||||
- checkin_pool_ : std::vector<GeneratedCheckin>
|
||||
- follow_pool_ : std::vector<GeneratedFollow>
|
||||
--
|
||||
+ Run() : bool
|
||||
- RunUserPhase(locations : const std::vector<Location>&) : void
|
||||
- RunBreweryAndBeerPhase(locations : const std::vector<Location>&) : void
|
||||
- RunCheckinPhase() : void
|
||||
- RunRatingPhase() : void
|
||||
- RunFollowPhase() : void
|
||||
}
|
||||
|
||||
' --- Orchestration Aggregations (Services & Strategies) ---
|
||||
BiergartenPipelineOrchestrator *-- ICuratedDataService
|
||||
BiergartenPipelineOrchestrator *-- EnrichmentService
|
||||
BiergartenPipelineOrchestrator *-- DataGenerator
|
||||
BiergartenPipelineOrchestrator *-- ExportService
|
||||
BiergartenPipelineOrchestrator *-- CheckinDistributionStrategy
|
||||
BiergartenPipelineOrchestrator *-- FollowGenerationStrategy
|
||||
BiergartenPipelineOrchestrator *-- SamplingStrategy
|
||||
BiergartenPipelineOrchestrator *-- BeerSelectionStrategy
|
||||
BiergartenPipelineOrchestrator *-- ApplicationOptions
|
||||
BiergartenPipelineOrchestrator *-- Logger
|
||||
|
||||
' --- Orchestration Aggregations (Data Pools) ---
|
||||
BiergartenPipelineOrchestrator *-- "0..*" GeneratedUser : user_pool_
|
||||
BiergartenPipelineOrchestrator *-- "0..*" GeneratedBrewery : brewery_pool_
|
||||
BiergartenPipelineOrchestrator *-- "0..*" GeneratedBeer : beer_pool_
|
||||
BiergartenPipelineOrchestrator *-- "0..*" GeneratedCheckin : checkin_pool_
|
||||
BiergartenPipelineOrchestrator *-- "0..*" GeneratedFollow : follow_pool_
|
||||
|
||||
' --- Interfaces & Implementations ---
|
||||
ICuratedDataService <|.. JsonLoader
|
||||
ICuratedDataService <|.. MockCuratedDataService
|
||||
Logger <|.. PipelineLogger
|
||||
ContextStrategy <|.. BreweryContextStrategy
|
||||
ContextStrategy <|.. BeerContextStrategy
|
||||
SamplingStrategy <|.. UniformSamplingStrategy
|
||||
BeerSelectionStrategy <|.. RandomBeerSelectionStrategy
|
||||
CheckinDistributionStrategy <|.. JCurveCheckinStrategy
|
||||
CheckinDistributionStrategy <|.. RandomCheckinStrategy
|
||||
FollowGenerationStrategy <|.. RandomFollowStrategy
|
||||
FollowGenerationStrategy <|.. ActivityWeightedFollowStrategy
|
||||
EnrichmentService <|.. WikipediaService
|
||||
WebClient <|.. HttpWebClient
|
||||
DataGenerator <|.. MockGenerator
|
||||
DataGenerator <|.. LlamaGenerator
|
||||
PromptFormatter <|.. Gemma4JinjaPromptFormatter
|
||||
ExportService <|.. SqliteExportService
|
||||
DateTimeProvider <|.. SystemDateTimeProvider
|
||||
|
||||
' --- Service Compositions & Dependencies ---
|
||||
WikipediaService *-- WebClient
|
||||
WikipediaService ..> ContextStrategy
|
||||
LlamaGenerator *-- PromptFormatter
|
||||
LlamaGenerator *-- IPromptDirectory
|
||||
LlamaGenerator ..> GeneratorOptions
|
||||
SqliteExportService *-- DateTimeProvider
|
||||
|
||||
' --- Cross-Component Aggregations (Held References) ---
|
||||
PipelineLogger o-- BoundedChannel : logs to
|
||||
LogWorker o-- BoundedChannel : drains from
|
||||
|
||||
' --- Domain Containment ---
|
||||
EnrichedCity *-- Location
|
||||
EnrichedCity *-- LocationContext
|
||||
GeneratedBrewery *-- Location
|
||||
GeneratedBrewery *-- BreweryResult
|
||||
GeneratedBrewery *-- GenerationMetadata
|
||||
GeneratedBeer *-- Location
|
||||
GeneratedBeer *-- BeerStyle
|
||||
GeneratedBeer *-- BeerResult
|
||||
GeneratedBeer *-- GenerationMetadata
|
||||
GeneratedUser *-- Location
|
||||
GeneratedUser *-- UserResult
|
||||
GeneratedUser *-- GenerationMetadata
|
||||
GeneratedCheckin *-- CheckinResult
|
||||
GeneratedCheckin *-- GenerationMetadata
|
||||
GeneratedRating *-- RatingResult
|
||||
GeneratedRating *-- GenerationMetadata
|
||||
GeneratedFollow *-- GenerationMetadata
|
||||
|
||||
@enduml
|
||||
Reference in New Issue
Block a user