mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-06-01 01:54:00 +00:00
Compare commits
2 Commits
d40ce34363
...
4331865281
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4331865281 | ||
|
|
8f5471d96c |
34
pipeline/diagrams/biergarten-weizen-theme.puml
Normal file
34
pipeline/diagrams/biergarten-weizen-theme.puml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
skinparam shadowing false
|
||||||
|
skinparam backgroundColor #FCFCF7
|
||||||
|
skinparam defaultFontName "DM Sans"
|
||||||
|
skinparam defaultFontColor #14180C
|
||||||
|
skinparam titleFontName "Volkhov"
|
||||||
|
skinparam titleFontColor #14180C
|
||||||
|
skinparam ArrowColor #656F33
|
||||||
|
skinparam NoteBackgroundColor #DBEEDD
|
||||||
|
skinparam NoteFontColor #14180C
|
||||||
|
skinparam NoteBorderColor #4A5837
|
||||||
|
skinparam SwimlaneBorderColor #4A5837
|
||||||
|
skinparam SwimlaneBorderThickness 1
|
||||||
|
skinparam activityStartColor #EBECE3
|
||||||
|
skinparam activityEndColor #4A5837
|
||||||
|
skinparam activityStopColor #4A5837
|
||||||
|
skinparam ActivityBackgroundColor #EBECE3
|
||||||
|
skinparam ActivityBorderColor #4A5837
|
||||||
|
skinparam ActivityDiamondBackgroundColor #CBD2B5
|
||||||
|
skinparam ActivityDiamondBorderColor #4A5837
|
||||||
|
skinparam packageStyle rectangle
|
||||||
|
skinparam packageBackgroundColor #F1F3EA
|
||||||
|
skinparam packageBorderColor #4A5837
|
||||||
|
skinparam packageFontColor #14180C
|
||||||
|
skinparam classBackgroundColor #EBECE3
|
||||||
|
skinparam classBorderColor #4A5837
|
||||||
|
skinparam classFontColor #14180C
|
||||||
|
skinparam classAttributeFontColor #3F4724
|
||||||
|
skinparam classStereotypeFontColor #4A5837
|
||||||
|
skinparam interfaceBackgroundColor #DBEEDD
|
||||||
|
skinparam interfaceBorderColor #4A5837
|
||||||
|
skinparam interfaceFontColor #14180C
|
||||||
|
skinparam enumBackgroundColor #E4E6D8
|
||||||
|
skinparam enumBorderColor #4A5837
|
||||||
|
skinparam enumFontColor #14180C
|
||||||
@@ -1,26 +1,7 @@
|
|||||||
@startuml biergarten_activity
|
@startuml biergarten_activity
|
||||||
skinparam defaultFontName "DM Sans"
|
!include ../biergarten-weizen-theme.puml
|
||||||
skinparam defaultFontSize 13
|
skinparam defaultFontSize 13
|
||||||
skinparam titleFontName "Volkhov"
|
|
||||||
skinparam titleFontSize 20
|
skinparam titleFontSize 20
|
||||||
skinparam backgroundColor #FCFCF7
|
|
||||||
skinparam defaultFontColor #14180C
|
|
||||||
skinparam titleFontColor #14180C
|
|
||||||
skinparam ArrowColor #656F33
|
|
||||||
skinparam activityStartColor #EBECE3
|
|
||||||
skinparam activityEndColor #4A5837
|
|
||||||
skinparam activityStopColor #4A5837
|
|
||||||
skinparam ActivityBackgroundColor #EBECE3
|
|
||||||
skinparam ActivityBorderColor #4A5837
|
|
||||||
skinparam ActivityDiamondBackgroundColor #CBD2B5
|
|
||||||
skinparam ActivityDiamondBorderColor #4A5837
|
|
||||||
skinparam NoteBackgroundColor #DBEEDD
|
|
||||||
skinparam NoteFontColor #14180C
|
|
||||||
skinparam NoteBorderColor #4A5837
|
|
||||||
skinparam SwimlaneBorderColor #4A5837
|
|
||||||
skinparam SwimlaneBorderThickness 1
|
|
||||||
skinparam monochrome reverse
|
|
||||||
|
|
||||||
|
|
||||||
title The Biergarten Data Pipeline — Activity Diagram
|
title The Biergarten Data Pipeline — Activity Diagram
|
||||||
|
|
||||||
@@ -35,12 +16,11 @@ endif
|
|||||||
:Init CurlGlobalState & LlamaBackendState;
|
:Init CurlGlobalState & LlamaBackendState;
|
||||||
:Build DI injector;
|
:Build DI injector;
|
||||||
|
|
||||||
|
|
||||||
:Initialize SqliteExportService;
|
:Initialize SqliteExportService;
|
||||||
note right
|
note right
|
||||||
Opens SQLite connection.
|
Opens SQLite connection.
|
||||||
Begins a single transaction
|
(Transactions are now managed
|
||||||
covering all five fixture types.
|
per-phase via batching).
|
||||||
end note
|
end note
|
||||||
|
|
||||||
:Create BoundedChannel<LogEntry> log_ch;
|
:Create BoundedChannel<LogEntry> log_ch;
|
||||||
@@ -49,47 +29,51 @@ note right
|
|||||||
Log worker drains log_ch for the
|
Log worker drains log_ch for the
|
||||||
entire pipeline lifetime.
|
entire pipeline lifetime.
|
||||||
All workers emit LogEntry structs
|
All workers emit LogEntry structs
|
||||||
via PipelineLogger — never spdlog directly.
|
via PipelineLogger -- never spdlog directly.
|
||||||
end note
|
end note
|
||||||
|
|
||||||
:BiergartenPipelineOrchestrator::Run();
|
:BiergartenPipelineOrchestrator::Run();
|
||||||
|BiergartenPipelineOrchestrator::Run()|
|
|BiergartenPipelineOrchestrator::Run()|
|
||||||
:JsonLoader::LoadLocations("locations.json");
|
|
||||||
:JsonLoader::LoadBeerStyles("beer-styles.json");
|
|
||||||
:JsonLoader::LoadPersonas("personas.json");
|
|
||||||
:JsonLoader::LoadNamesByCountry("names-by-country.json");
|
|
||||||
|
|
||||||
:EnrichmentService::PreWarmBeerStyleCache(beer_styles);
|
|
||||||
note right
|
|
||||||
Beer styles do not need location context.
|
|
||||||
Wikipedia summaries for the entire palette are
|
|
||||||
fetched and cached globally at startup.
|
|
||||||
end note
|
|
||||||
|
|
||||||
:EnrichmentService::PreWarmPersonaCache(personas);
|
|
||||||
note right
|
|
||||||
Persona descriptions do not need location context.
|
|
||||||
All persona lookups are resolved and cached
|
|
||||||
globally at startup.
|
|
||||||
end note
|
|
||||||
|
|
||||||
|
fork
|
||||||
|
:JsonLoader::LoadBeerStyles("beer-styles.json");
|
||||||
|
:EnrichmentService::PreWarmBeerStyleCache(beer_styles);
|
||||||
|
fork again
|
||||||
|
:JsonLoader::LoadLocations("locations.json");
|
||||||
|
:EnrichmentService::PreWarmLocationCache(sampled_locations);
|
||||||
|
end fork
|
||||||
|
fork
|
||||||
|
:JsonLoader::LoadNamesByCountry("names-by-country.json");
|
||||||
|
fork again
|
||||||
|
:JsonLoader::LoadPersonas("personas.json");
|
||||||
|
end fork
|
||||||
|
|
||||||
' ═══════════════════════════════════════════
|
' ═══════════════════════════════════════════
|
||||||
' PHASE 0 — USER GENERATION
|
' PHASE 0 — USER GENERATION
|
||||||
' ═══════════════════════════════════════════
|
' ═══════════════════════════════════════════
|
||||||
|Orchestrator|
|
|Orchestrator|
|
||||||
:RunUserPhase(sampled_locations);
|
:RunUserPhase(sampled_locations);
|
||||||
:Create BoundedChannels\n(loc_ch, llm_ch, exp_ch);
|
:Create BoundedChannels\n(loc_ch, exp_ch);
|
||||||
|
|
||||||
fork
|
fork
|
||||||
|Orchestrator|
|
|Orchestrator|
|
||||||
:Loop: Send Locations → loc_ch;
|
:Loop: Send Locations -> loc_ch;
|
||||||
:Close loc_ch;
|
:Close loc_ch;
|
||||||
|
note right
|
||||||
|
Producer closes loc_ch.
|
||||||
|
LLM Worker while loop
|
||||||
|
terminates on empty + closed.
|
||||||
|
end note
|
||||||
fork again
|
fork again
|
||||||
|LLM Worker|
|
|LLM Worker|
|
||||||
while (loc_ch has items?) is (yes)
|
while (loc_ch has items?) is (yes)
|
||||||
:Receive Location;
|
:Receive Location;
|
||||||
|
|
||||||
|
:GetLocationContextFromCache(location);
|
||||||
|
note right
|
||||||
|
Guaranteed cache hit from startup.
|
||||||
|
end note
|
||||||
|
|
||||||
:IPersonaSelectionStrategy::SelectPersona(\n personas_palette_);
|
:IPersonaSelectionStrategy::SelectPersona(\n personas_palette_);
|
||||||
note right
|
note right
|
||||||
Guaranteed cache hit from startup.
|
Guaranteed cache hit from startup.
|
||||||
@@ -100,94 +84,150 @@ fork again
|
|||||||
|
|
||||||
:NamesByCountry::SampleName(\n location.iso3166_1);
|
:NamesByCountry::SampleName(\n location.iso3166_1);
|
||||||
note right
|
note right
|
||||||
Deterministic lookup — no LLM involved.
|
Deterministic lookup -- no LLM involved.
|
||||||
Name selected from pre-keyed table
|
Name selected from pre-keyed table
|
||||||
and passed into the generation prompt.
|
and passed into the generation prompt.
|
||||||
end note
|
end note
|
||||||
|
|
||||||
:GenerateUser(location, persona, sampled_name)\nvia DataGenerator;
|
:GenerateUser(enriched_city, persona, sampled_name)\nvia DataGenerator;
|
||||||
note right
|
note right
|
||||||
LLM receives: Location fields + persona
|
LLM receives: EnrichedCity context + persona
|
||||||
description + sampled name. Generates
|
description + sampled name. Generates
|
||||||
bio and preference signals grounded
|
bio and preference signals grounded
|
||||||
in locale and persona.
|
in locale and persona.
|
||||||
end note
|
end note
|
||||||
|
|
||||||
:PipelineLogger::Log(Info, UserGeneration,\n city, user_id, "llm");
|
:PipelineLogger::Log(Info, UserGeneration,\n city, user_id, "llm");
|
||||||
:Send GeneratedUser → llm_ch;
|
:Send GeneratedUser -> exp_ch;
|
||||||
endwhile (no)
|
endwhile (no)
|
||||||
:Close llm_ch;
|
:Close exp_ch;
|
||||||
|
note right
|
||||||
|
Producer closes exp_ch.
|
||||||
|
SQLite Worker while loop
|
||||||
|
terminates on empty + closed.
|
||||||
|
end note
|
||||||
fork again
|
fork again
|
||||||
|SQLite Worker|
|
|SQLite Worker|
|
||||||
while (llm_ch has items?) is (yes)
|
:BEGIN TRANSACTION;
|
||||||
|
while (exp_ch has items?) is (yes)
|
||||||
:Receive GeneratedUser;
|
:Receive GeneratedUser;
|
||||||
:ProcessUser(user) → sqlite3_int64;
|
:ProcessUser(user);
|
||||||
:PipelineLogger::Log(Info, UserGeneration,\n city, user_id, "sqlite");
|
:PipelineLogger::Log(Info, UserGeneration,\n city, user_id, "sqlite");
|
||||||
:Append → user_pool_;
|
:Append -> user_pool_;
|
||||||
|
if (Batch size reached?) then (yes)
|
||||||
|
:COMMIT & BEGIN;
|
||||||
|
else (no)
|
||||||
|
endif
|
||||||
endwhile (no)
|
endwhile (no)
|
||||||
|
:COMMIT (Final);
|
||||||
end fork
|
end fork
|
||||||
|
|
||||||
|Orchestrator|
|
|Orchestrator|
|
||||||
:Join LLM Worker, SQLite Worker;
|
:Join LLM Worker, SQLite Worker;
|
||||||
|
|
||||||
' ═══════════════════════════════════════════
|
' ═══════════════════════════════════════════
|
||||||
' PHASE 1 — BREWERY & BEER GENERATION
|
' PHASE 1a — BREWERY GENERATION
|
||||||
' ═══════════════════════════════════════════
|
' ═══════════════════════════════════════════
|
||||||
:RunBreweryAndBeerPhase(sampled_locations);
|
:RunBreweryPhase(sampled_locations);
|
||||||
:Create BoundedChannels\n(loc_ch, llm_ch, exp_ch);
|
:Create BoundedChannels\n(loc_ch, exp_ch);
|
||||||
|
|
||||||
fork
|
fork
|
||||||
|Orchestrator|
|
|Orchestrator|
|
||||||
:Loop: Send Locations → loc_ch;
|
:Loop: Sample User from user_pool_
|
||||||
|
and pair with Location;
|
||||||
|
:Send BreweryTask(Location, User) -> loc_ch;
|
||||||
:Close loc_ch;
|
:Close loc_ch;
|
||||||
fork again
|
fork again
|
||||||
|Enrichment Workers (xN)|
|
|LLM Worker|
|
||||||
while (loc_ch has items?) is (yes)
|
while (loc_ch has items?) is (yes)
|
||||||
:Receive Location;
|
:Receive BreweryTask(Location, User);
|
||||||
:GetLocationContext(location,\nBreweryContextStrategy);
|
|
||||||
:PipelineLogger::Log(Info,\n BreweryAndBeerGeneration,\n city, nullopt, "enrichment");
|
:GetLocationContextFromCache(task.location);
|
||||||
:Send EnrichedCity → llm_ch;
|
note right
|
||||||
|
Guaranteed cache hit from startup.
|
||||||
|
end note
|
||||||
|
|
||||||
|
:GenerateBrewery(enriched_city, context, task.user)\nvia DataGenerator;
|
||||||
|
note right
|
||||||
|
KV cache stays warm.
|
||||||
|
Brewery is linked to the sampled owner_user_id.
|
||||||
|
end note
|
||||||
|
:PipelineLogger::Log(Info,\n BreweryGeneration,\n city, brewery_id, "llm");
|
||||||
|
:Send GeneratedBrewery -> exp_ch;
|
||||||
endwhile (no)
|
endwhile (no)
|
||||||
|
:Close exp_ch;
|
||||||
|
fork again
|
||||||
|
|SQLite Worker|
|
||||||
|
:BEGIN TRANSACTION;
|
||||||
|
while (exp_ch has items?) is (yes)
|
||||||
|
:Receive GeneratedBrewery;
|
||||||
|
:ProcessBrewery(brewery);
|
||||||
|
:PipelineLogger::Log(Info,\n BreweryGeneration,\n city, brewery_id, "sqlite");
|
||||||
|
:Append -> brewery_pool_;
|
||||||
|
if (Batch size reached?) then (yes)
|
||||||
|
:COMMIT & BEGIN;
|
||||||
|
else (no)
|
||||||
|
endif
|
||||||
|
endwhile (no)
|
||||||
|
:COMMIT (Final);
|
||||||
|
end fork
|
||||||
|
|
||||||
|
|Orchestrator|
|
||||||
|
:Join LLM Worker, SQLite Worker;
|
||||||
|
note right
|
||||||
|
brewery_pool_ is now fully populated.
|
||||||
|
Phase 1b may begin.
|
||||||
|
end note
|
||||||
|
|
||||||
|
' ═══════════════════════════════════════════
|
||||||
|
' PHASE 1b — BEER GENERATION
|
||||||
|
' ═══════════════════════════════════════════
|
||||||
|
:RunBeerPhase();
|
||||||
|
:Create BoundedChannels\n(brew_ch, exp_ch);
|
||||||
|
|
||||||
|
fork
|
||||||
|Orchestrator|
|
|Orchestrator|
|
||||||
:Join Enrichment Workers;
|
:Loop: Send Breweries -> brew_ch;
|
||||||
:Close llm_ch;
|
:Close brew_ch;
|
||||||
fork again
|
fork again
|
||||||
|LLM Worker|
|
|LLM Worker|
|
||||||
while (llm_ch has items?) is (yes)
|
while (brew_ch has items?) is (yes)
|
||||||
:Receive EnrichedCity;
|
:Receive GeneratedBrewery;
|
||||||
|
|
||||||
:GenerateBrewery(location, context)\nvia DataGenerator;
|
|
||||||
|
|
||||||
:IBeerSelectionStrategy::SelectStyles(\n brewery, beer_style_palette_);
|
:IBeerSelectionStrategy::SelectStyles(\n brewery, beer_style_palette_);
|
||||||
|
|
||||||
while (For each selected BeerStyle?) is (remaining)
|
while (For each selected BeerStyle?) is (remaining)
|
||||||
:GetStyleContextFromCache(style);
|
:GetStyleContextFromCache(style);
|
||||||
note right
|
note right
|
||||||
Guaranteed cache hit from startup.
|
Guaranteed cache hit from startup.
|
||||||
|
KV cache stays warm across all
|
||||||
|
beer generations -- system prompt
|
||||||
|
does not change within this phase.
|
||||||
end note
|
end note
|
||||||
:GenerateBeer(brewery, style_context)\nvia DataGenerator;
|
:GenerateBeer(brewery, style_context)\nvia DataGenerator;
|
||||||
:Attach GeneratedBeer to Brewery bundle;
|
:Attach GeneratedBeer to bundle;
|
||||||
endwhile (done)
|
endwhile (done)
|
||||||
|
|
||||||
:PipelineLogger::Log(Info,\n BreweryAndBeerGeneration,\n city, brewery_id, "llm");
|
:PipelineLogger::Log(Info,\n BeerGeneration,\n city, brewery_id, "llm");
|
||||||
:Send BreweryWithBeers Bundle → exp_ch;
|
:Send BeersBundle -> exp_ch;
|
||||||
endwhile (no)
|
endwhile (no)
|
||||||
:Close exp_ch;
|
:Close exp_ch;
|
||||||
fork again
|
fork again
|
||||||
|SQLite Worker|
|
|SQLite Worker|
|
||||||
|
:BEGIN TRANSACTION;
|
||||||
while (exp_ch has items?) is (yes)
|
while (exp_ch has items?) is (yes)
|
||||||
:Receive BreweryWithBeers Bundle;
|
:Receive BeersBundle;
|
||||||
:ProcessBrewery(brewery) → brewery_id;
|
|
||||||
:Append → brewery_pool_;
|
|
||||||
|
|
||||||
while (For each beer in bundle?) is (remaining)
|
while (For each beer in bundle?) is (remaining)
|
||||||
:Set beer.brewery_id = brewery_id;
|
:Set beer.brewery_id from bundle;
|
||||||
:ProcessBeer(beer) → sqlite3_int64;
|
:ProcessBeer(beer);
|
||||||
:Append → beer_pool_;
|
:Append -> beer_pool_;
|
||||||
endwhile (done)
|
endwhile (done)
|
||||||
|
:PipelineLogger::Log(Info,\n BeerGeneration,\n city, brewery_id, "sqlite");
|
||||||
:PipelineLogger::Log(Info,\n BreweryAndBeerGeneration,\n city, brewery_id, "sqlite");
|
if (Batch size reached?) then (yes)
|
||||||
|
:COMMIT & BEGIN;
|
||||||
|
else (no)
|
||||||
|
endif
|
||||||
endwhile (no)
|
endwhile (no)
|
||||||
|
:COMMIT (Final);
|
||||||
end fork
|
end fork
|
||||||
|
|
||||||
|Orchestrator|
|
|Orchestrator|
|
||||||
@@ -208,17 +248,23 @@ note right
|
|||||||
emerges from persona distribution.
|
emerges from persona distribution.
|
||||||
end note
|
end note
|
||||||
|
|
||||||
|
:BEGIN TRANSACTION;
|
||||||
while (For each GeneratedUser in user_pool_?) is (remaining)
|
while (For each GeneratedUser in user_pool_?) is (remaining)
|
||||||
:CheckinsForUser(user, brewery_pool_.size());
|
:CheckinsForUser(user, brewery_pool_.size());
|
||||||
while (For each checkin index?) is (remaining)
|
while (For each checkin index?) is (remaining)
|
||||||
:TimestampFor(user, index);
|
:TimestampFor(user, index);
|
||||||
:Select brewery from brewery_pool_;
|
:Select brewery from brewery_pool_;
|
||||||
:GenerateCheckin(user, brewery, timestamp)\nvia DataGenerator;
|
:GenerateCheckin(user, brewery, timestamp)\nvia DataGenerator;
|
||||||
:ProcessCheckin(checkin) → sqlite3_int64;
|
:ProcessCheckin(checkin);
|
||||||
:PipelineLogger::Log(Info, CheckinGeneration,\n nullopt, checkin_id, "sqlite");
|
:PipelineLogger::Log(Info, CheckinGeneration,\n nullopt, checkin_id, "sqlite");
|
||||||
:Append → checkin_pool_;
|
:Append -> checkin_pool_;
|
||||||
|
if (Batch size reached?) then (yes)
|
||||||
|
:COMMIT & BEGIN;
|
||||||
|
else (no)
|
||||||
|
endif
|
||||||
endwhile (done)
|
endwhile (done)
|
||||||
endwhile (done)
|
endwhile (done)
|
||||||
|
:COMMIT (Final);
|
||||||
|
|
||||||
' ═══════════════════════════════════════════
|
' ═══════════════════════════════════════════
|
||||||
' PHASE 3 — RATING GENERATION
|
' PHASE 3 — RATING GENERATION
|
||||||
@@ -230,33 +276,41 @@ note right
|
|||||||
Rating skew modulated per persona.
|
Rating skew modulated per persona.
|
||||||
end note
|
end note
|
||||||
|
|
||||||
|
:BEGIN TRANSACTION;
|
||||||
while (For each GeneratedCheckin in checkin_pool_?) is (remaining)
|
while (For each GeneratedCheckin in checkin_pool_?) is (remaining)
|
||||||
:Match brewery_id → select beer from beer_pool_\n(same brewery_id, biased by persona affinities);
|
:Match brewery_id, select beer from beer_pool_\n(same brewery_id, biased by persona affinities);
|
||||||
if (Beer exists for brewery?) then (yes)
|
if (Beer exists for brewery?) then (yes)
|
||||||
:GenerateRating(user, beer, checkin_id)\nvia DataGenerator;
|
:GenerateRating(user, beer, checkin_id)\nvia DataGenerator;
|
||||||
:ProcessRating(rating);
|
:ProcessRating(rating);
|
||||||
:PipelineLogger::Log(Info, RatingGeneration,\n nullopt, rating_id, "sqlite");
|
:PipelineLogger::Log(Info, RatingGeneration,\n nullopt, rating_id, "sqlite");
|
||||||
|
if (Batch size reached?) then (yes)
|
||||||
|
:COMMIT & BEGIN;
|
||||||
|
else (no)
|
||||||
|
endif
|
||||||
else (no)
|
else (no)
|
||||||
:PipelineLogger::Log(Warn, RatingGeneration,\n nullopt, brewery_id, "sqlite");
|
:PipelineLogger::Log(Warn, RatingGeneration,\n nullopt, brewery_id, "sqlite");
|
||||||
:Skip — brewery has no beers;
|
:Skip -- brewery has no beers;
|
||||||
endif
|
endif
|
||||||
endwhile (done)
|
endwhile (done)
|
||||||
|
:COMMIT (Final);
|
||||||
|
|
||||||
' ═══════════════════════════════════════════
|
' ═══════════════════════════════════════════
|
||||||
' TEARDOWN
|
' TEARDOWN
|
||||||
' ═══════════════════════════════════════════
|
' ═══════════════════════════════════════════
|
||||||
|Main|
|
|Orchestrator|
|
||||||
:Finalize SqliteExportService;
|
:Finalize SqliteExportService;
|
||||||
note right
|
note right
|
||||||
COMMIT covers all five fixture types.
|
Safely closes the DB connection.
|
||||||
end note
|
end note
|
||||||
:Close log_ch;
|
:Close log_ch;
|
||||||
|
|
||||||
|
|Main|
|
||||||
|
:spdlog::info "Pipeline complete in X ms";
|
||||||
:Join Log Worker;
|
:Join Log Worker;
|
||||||
note right
|
note right
|
||||||
Drain guarantees no LogEntry is
|
Drain guarantees no LogEntry is
|
||||||
dropped at shutdown.
|
dropped at shutdown.
|
||||||
end note
|
end note
|
||||||
:spdlog::info "Pipeline complete in X ms";
|
|
||||||
stop
|
stop
|
||||||
|
|
||||||
@enduml
|
@enduml
|
||||||
|
|||||||
@@ -1,51 +1,14 @@
|
|||||||
@startuml future_possible_architecture
|
@startuml
|
||||||
|
|
||||||
' ==========================================
|
' ==========================================
|
||||||
' CONFIGURATION & STYLING
|
' CONFIGURATION & STYLING
|
||||||
' ==========================================
|
' ==========================================
|
||||||
left to right direction
|
!include ../biergarten-weizen-theme.puml
|
||||||
skinparam linetype ortho
|
skinparam classAttributeFontSize 9
|
||||||
|
skinparam defaultFontSize 25
|
||||||
|
skinparam titleFontSize 30
|
||||||
|
|
||||||
' --- Typography ---
|
package "Domain: Models" {
|
||||||
skinparam defaultFontName "DM Sans"
|
|
||||||
skinparam defaultFontSize 14
|
|
||||||
skinparam titleFontName "Volkhov"
|
|
||||||
skinparam titleFontSize 20
|
|
||||||
|
|
||||||
' --- Global Colors ---
|
|
||||||
skinparam backgroundColor #FCFCF7
|
|
||||||
skinparam defaultFontColor #14180C
|
|
||||||
skinparam titleFontColor #14180C
|
|
||||||
skinparam ArrowColor #656F33
|
|
||||||
|
|
||||||
skinparam class {
|
|
||||||
BackgroundColor #EBECE3
|
|
||||||
HeaderBackgroundColor #CBD2B5
|
|
||||||
BorderColor #4A5837
|
|
||||||
ArrowColor #656F33
|
|
||||||
FontColor #14180C
|
|
||||||
}
|
|
||||||
|
|
||||||
skinparam package {
|
|
||||||
BackgroundColor #DBEEDD
|
|
||||||
BorderColor #4A5837
|
|
||||||
FontColor #14180C
|
|
||||||
}
|
|
||||||
|
|
||||||
skinparam note {
|
|
||||||
BackgroundColor #DBEEDD
|
|
||||||
BorderColor #4A5837
|
|
||||||
FontColor #14180C
|
|
||||||
}
|
|
||||||
|
|
||||||
skinparam monochrome reverse
|
|
||||||
|
|
||||||
title The Biergarten Data Pipeline — Planned Architecture
|
|
||||||
|
|
||||||
' ==========================================
|
|
||||||
' DOMAIN MODELS
|
|
||||||
' ==========================================
|
|
||||||
package "Domain Models" {
|
|
||||||
|
|
||||||
class Location {
|
class Location {
|
||||||
+ city : std::string
|
+ city : std::string
|
||||||
@@ -62,8 +25,9 @@ package "Domain Models" {
|
|||||||
+ text : std::string
|
+ text : std::string
|
||||||
+ completeness : Completeness
|
+ completeness : Completeness
|
||||||
+ char_count : size_t
|
+ char_count : size_t
|
||||||
--
|
}
|
||||||
<<enum>> Completeness
|
|
||||||
|
enum Completeness {
|
||||||
Full
|
Full
|
||||||
Partial
|
Partial
|
||||||
Absent
|
Absent
|
||||||
@@ -156,6 +120,11 @@ package "Domain Models" {
|
|||||||
+ generated_at : std::string
|
+ generated_at : std::string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
LocationContext *-- Completeness
|
||||||
|
}
|
||||||
|
|
||||||
|
package "Domain: Application Configuration"{
|
||||||
class SamplingOptions {
|
class SamplingOptions {
|
||||||
+ temperature : float = 1.0F
|
+ temperature : float = 1.0F
|
||||||
+ top_p : float = 0.95F
|
+ top_p : float = 0.95F
|
||||||
@@ -184,15 +153,60 @@ package "Domain Models" {
|
|||||||
ApplicationOptions *-- GeneratorOptions
|
ApplicationOptions *-- GeneratorOptions
|
||||||
ApplicationOptions *-- PipelineOptions
|
ApplicationOptions *-- PipelineOptions
|
||||||
GeneratorOptions *-- SamplingOptions
|
GeneratorOptions *-- SamplingOptions
|
||||||
LocationContext *-- Completeness
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
package "Domain: Policy" {
|
||||||
|
|
||||||
' ==========================================
|
interface ContextStrategy <<interface>> {
|
||||||
' LOGGING
|
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
||||||
' ==========================================
|
+ MaxContextChars() : size_t
|
||||||
package "Logging" {
|
}
|
||||||
|
|
||||||
|
class BreweryContextStrategy {
|
||||||
|
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
||||||
|
+ MaxContextChars() : size_t
|
||||||
|
}
|
||||||
|
|
||||||
|
class BeerContextStrategy {
|
||||||
|
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
||||||
|
+ MaxContextChars() : size_t
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SamplingStrategy <<interface>> {
|
||||||
|
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
|
||||||
|
}
|
||||||
|
|
||||||
|
class UniformSamplingStrategy {
|
||||||
|
- sample_size_ : size_t
|
||||||
|
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BeerSelectionStrategy <<interface>> {
|
||||||
|
+ SelectStyles(brewery : const GeneratedBrewery&,\n palette : std::span<const BeerStyle>) : std::vector<BeerStyle>
|
||||||
|
}
|
||||||
|
|
||||||
|
class RandomBeerSelectionStrategy {
|
||||||
|
- rng_ : std::mt19937
|
||||||
|
- min_beers_ : size_t
|
||||||
|
- max_beers_ : size_t
|
||||||
|
+ SelectStyles(brewery : const GeneratedBrewery&,\n palette : std::span<const BeerStyle>) : std::vector<BeerStyle>
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CheckinDistributionStrategy <<interface>> {
|
||||||
|
+ AssignActivityWeights(users : std::vector<GeneratedUser>&) : void
|
||||||
|
+ CheckinsForUser(user : const GeneratedUser&,\n brewery_count : size_t) : size_t
|
||||||
|
+ TimestampFor(user : const GeneratedUser&,\n index : size_t) : std::string
|
||||||
|
}
|
||||||
|
|
||||||
|
class JCurveCheckinStrategy {
|
||||||
|
- rng_ : std::mt19937
|
||||||
|
+ AssignActivityWeights(users : std::vector<GeneratedUser>&) : void
|
||||||
|
+ CheckinsForUser(user : const GeneratedUser&,\n brewery_count : size_t) : size_t
|
||||||
|
+ TimestampFor(user : const GeneratedUser&,\n index : size_t) : std::string
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
package "Infrastructure: Logging" {
|
||||||
enum LogLevel {
|
enum LogLevel {
|
||||||
Debug
|
Debug
|
||||||
Info
|
Info
|
||||||
@@ -243,67 +257,23 @@ package "Logging" {
|
|||||||
LogWorker ..> LogEntry : consumes
|
LogWorker ..> LogEntry : consumes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
package "Infrastructure: Pipeline Channel" {
|
||||||
|
|
||||||
' ==========================================
|
class "BoundedChannel<T>" as BoundedChannel {
|
||||||
' DOMAIN POLICY
|
- queue_ : std::queue<T>
|
||||||
' ==========================================
|
- mutex_ : std::mutex
|
||||||
package "Domain Policy" {
|
- not_full_ : std::condition_variable
|
||||||
|
- not_empty_ : std::condition_variable
|
||||||
interface ContextStrategy <<interface>> {
|
- capacity_ : size_t
|
||||||
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
- closed_ : bool
|
||||||
+ MaxContextChars() : size_t
|
+ Send(item : T) : void
|
||||||
}
|
+ Receive() : std::optional<T>
|
||||||
|
+ Close() : void
|
||||||
class BreweryContextStrategy {
|
|
||||||
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
|
||||||
+ MaxContextChars() : size_t
|
|
||||||
}
|
|
||||||
|
|
||||||
class BeerContextStrategy {
|
|
||||||
+ QueriesFor(loc : const Location&) : std::vector<std::string>
|
|
||||||
+ MaxContextChars() : size_t
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SamplingStrategy <<interface>> {
|
|
||||||
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
|
|
||||||
}
|
|
||||||
|
|
||||||
class UniformSamplingStrategy {
|
|
||||||
- sample_size_ : size_t
|
|
||||||
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BeerSelectionStrategy <<interface>> {
|
|
||||||
+ SelectStyles(brewery : const GeneratedBrewery&,\n palette : std::span<const BeerStyle>) : std::vector<BeerStyle>
|
|
||||||
}
|
|
||||||
|
|
||||||
class RandomBeerSelectionStrategy {
|
|
||||||
- rng_ : std::mt19937
|
|
||||||
- min_beers_ : size_t
|
|
||||||
- max_beers_ : size_t
|
|
||||||
+ SelectStyles(brewery : const GeneratedBrewery&,\n palette : std::span<const BeerStyle>) : std::vector<BeerStyle>
|
|
||||||
}
|
|
||||||
|
|
||||||
interface CheckinDistributionStrategy <<interface>> {
|
|
||||||
+ AssignActivityWeights(users : std::vector<GeneratedUser>&) : void
|
|
||||||
+ CheckinsForUser(user : const GeneratedUser&,\n brewery_count : size_t) : size_t
|
|
||||||
+ TimestampFor(user : const GeneratedUser&,\n index : size_t) : std::string
|
|
||||||
}
|
|
||||||
|
|
||||||
class JCurveCheckinStrategy {
|
|
||||||
- rng_ : std::mt19937
|
|
||||||
+ AssignActivityWeights(users : std::vector<GeneratedUser>&) : void
|
|
||||||
+ CheckinsForUser(user : const GeneratedUser&,\n brewery_count : size_t) : size_t
|
|
||||||
+ TimestampFor(user : const GeneratedUser&,\n index : size_t) : std::string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
package "Infrastructure: Data Preloading" {
|
||||||
' ==========================================
|
|
||||||
' ORCHESTRATION
|
|
||||||
' ==========================================
|
|
||||||
package "Orchestration" {
|
|
||||||
|
|
||||||
interface DataPreloader <<interface>> {
|
interface DataPreloader <<interface>> {
|
||||||
+ LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
|
+ LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
|
||||||
@@ -312,38 +282,6 @@ package "Orchestration" {
|
|||||||
+ LoadNamesByCountry(filepath : const std::filesystem::path&) : NamesByCountry
|
+ LoadNamesByCountry(filepath : const std::filesystem::path&) : NamesByCountry
|
||||||
}
|
}
|
||||||
|
|
||||||
class BiergartenPipelineOrchestrator {
|
|
||||||
- preloader_ : std::unique_ptr<DataPreloader>
|
|
||||||
- enrichment_service_ : std::unique_ptr<EnrichmentService>
|
|
||||||
- generator_ : std::unique_ptr<DataGenerator>
|
|
||||||
- logger_ : std::unique_ptr<Logger>
|
|
||||||
- exporter_ : std::unique_ptr<ExportService>
|
|
||||||
- brewery_context_strategy_ : std::unique_ptr<ContextStrategy>
|
|
||||||
- sampling_strategy_ : std::unique_ptr<SamplingStrategy>
|
|
||||||
- beer_selection_strategy_ : std::unique_ptr<BeerSelectionStrategy>
|
|
||||||
- checkin_strategy_ : std::unique_ptr<CheckinDistributionStrategy>
|
|
||||||
- beer_style_palette_ : std::vector<BeerStyle>
|
|
||||||
- options_ : ApplicationOptions
|
|
||||||
--
|
|
||||||
- user_pool_ : std::vector<GeneratedUser>
|
|
||||||
- brewery_pool_ : std::vector<GeneratedBrewery>
|
|
||||||
- beer_pool_ : std::vector<GeneratedBeer>
|
|
||||||
- checkin_pool_ : std::vector<GeneratedCheckin>
|
|
||||||
--
|
|
||||||
+ Run() : bool
|
|
||||||
- RunUserPhase(locations : const std::vector<Location>&) : void
|
|
||||||
- RunBreweryAndBeerPhase(locations : const std::vector<Location>&) : void
|
|
||||||
- RunCheckinPhase() : void
|
|
||||||
- RunRatingPhase() : void
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
' ==========================================
|
|
||||||
' INFRASTRUCTURE: PRELOADING
|
|
||||||
' ==========================================
|
|
||||||
package "Infrastructure: Preloading" {
|
|
||||||
|
|
||||||
class JsonLoader {
|
class JsonLoader {
|
||||||
+ LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
|
+ LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
|
||||||
+ LoadBeerStyles(filepath : const std::filesystem::path&) : std::vector<BeerStyle>
|
+ LoadBeerStyles(filepath : const std::filesystem::path&) : std::vector<BeerStyle>
|
||||||
@@ -353,10 +291,6 @@ package "Infrastructure: Preloading" {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
' ==========================================
|
|
||||||
' INFRASTRUCTURE: ENRICHMENT
|
|
||||||
' ==========================================
|
|
||||||
package "Infrastructure: Enrichment" {
|
package "Infrastructure: Enrichment" {
|
||||||
|
|
||||||
interface EnrichmentService <<interface>> {
|
interface EnrichmentService <<interface>> {
|
||||||
@@ -382,11 +316,7 @@ package "Infrastructure: Enrichment" {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
package "Infrastructure: Data Generation" {
|
||||||
' ==========================================
|
|
||||||
' INFRASTRUCTURE: GENERATION
|
|
||||||
' ==========================================
|
|
||||||
package "Infrastructure: Generation" {
|
|
||||||
|
|
||||||
interface DataGenerator <<interface>> {
|
interface DataGenerator <<interface>> {
|
||||||
+ GenerateBrewery(location : const Location&,\n context : const LocationContext&) : BreweryResult
|
+ GenerateBrewery(location : const Location&,\n context : const LocationContext&) : BreweryResult
|
||||||
@@ -432,31 +362,7 @@ package "Infrastructure: Generation" {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
package "Infrastructure: Data Export" {
|
||||||
' ==========================================
|
|
||||||
' INFRASTRUCTURE: PIPELINE CHANNEL
|
|
||||||
' ==========================================
|
|
||||||
package "Infrastructure: Pipeline Channel" {
|
|
||||||
|
|
||||||
class "BoundedChannel<T>" as BoundedChannel {
|
|
||||||
- queue_ : std::queue<T>
|
|
||||||
- mutex_ : std::mutex
|
|
||||||
- not_full_ : std::condition_variable
|
|
||||||
- not_empty_ : std::condition_variable
|
|
||||||
- capacity_ : size_t
|
|
||||||
- closed_ : bool
|
|
||||||
+ Send(item : T) : void
|
|
||||||
+ Receive() : std::optional<T>
|
|
||||||
+ Close() : void
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
' ==========================================
|
|
||||||
' INFRASTRUCTURE: EXPORT
|
|
||||||
' ==========================================
|
|
||||||
package "Infrastructure: Export" {
|
|
||||||
|
|
||||||
interface ExportService <<interface>> {
|
interface ExportService <<interface>> {
|
||||||
+ Initialize() : void
|
+ Initialize() : void
|
||||||
@@ -504,9 +410,31 @@ package "Infrastructure: Export" {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
' ==========================================
|
|
||||||
' GLOBAL RELATIONSHIPS
|
class BiergartenPipelineOrchestrator {
|
||||||
' ==========================================
|
- preloader_ : std::unique_ptr<DataPreloader>
|
||||||
|
- enrichment_service_ : std::unique_ptr<EnrichmentService>
|
||||||
|
- generator_ : std::unique_ptr<DataGenerator>
|
||||||
|
- logger_ : std::unique_ptr<Logger>
|
||||||
|
- exporter_ : std::unique_ptr<ExportService>
|
||||||
|
- brewery_context_strategy_ : std::unique_ptr<ContextStrategy>
|
||||||
|
- sampling_strategy_ : std::unique_ptr<SamplingStrategy>
|
||||||
|
- beer_selection_strategy_ : std::unique_ptr<BeerSelectionStrategy>
|
||||||
|
- checkin_strategy_ : std::unique_ptr<CheckinDistributionStrategy>
|
||||||
|
- beer_style_palette_ : std::vector<BeerStyle>
|
||||||
|
- options_ : ApplicationOptions
|
||||||
|
--
|
||||||
|
- user_pool_ : std::vector<GeneratedUser>
|
||||||
|
- brewery_pool_ : std::vector<GeneratedBrewery>
|
||||||
|
- beer_pool_ : std::vector<GeneratedBeer>
|
||||||
|
- checkin_pool_ : std::vector<GeneratedCheckin>
|
||||||
|
--
|
||||||
|
+ Run() : bool
|
||||||
|
- RunUserPhase(locations : const std::vector<Location>&) : void
|
||||||
|
- RunBreweryAndBeerPhase(locations : const std::vector<Location>&) : void
|
||||||
|
- RunCheckinPhase() : void
|
||||||
|
- RunRatingPhase() : void
|
||||||
|
}
|
||||||
|
|
||||||
' --- Orchestration Aggregations (Services & Strategies) ---
|
' --- Orchestration Aggregations (Services & Strategies) ---
|
||||||
BiergartenPipelineOrchestrator *-- DataPreloader
|
BiergartenPipelineOrchestrator *-- DataPreloader
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
1
pipeline/diagrams/planned/output/class.svg
Normal file
1
pipeline/diagrams/planned/output/class.svg
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user