mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-15 10:04:44 +00:00
restruc
This commit is contained in:
@@ -3,5 +3,5 @@
|
||||
Each generation phase runs a producer/consumer trio concurrently: an **Orchestrator** producer feeds a bounded channel, an **LLM Worker** consumes it and generates records (closing its output channel when the input is drained and closed), and a **SQLite Worker** consumes that output channel and commits in batches. The Checkin and Follow phases run in parallel with each other once the user and brewery pools are populated.
|
||||
|
||||
```mermaid
|
||||
--8<-- "diagrams/pipeline/planned/activity.mmd"
|
||||
--8<-- "pipeline/diagrams/planned/activity.mmd"
|
||||
```
|
||||
|
||||
194
docs/pipeline/diagrams/planned/activity.mmd
Normal file
194
docs/pipeline/diagrams/planned/activity.mmd
Normal file
@@ -0,0 +1,194 @@
|
||||
flowchart TD
|
||||
Start([start]) --> ParseArgs["Main: ParseArguments(argc, argv)"]
|
||||
ParseArgs --> InvalidArgs{Invalid args?}
|
||||
InvalidArgs -- yes --> LogErr["spdlog::error"] --> Stop1([stop])
|
||||
InvalidArgs -- no --> InitOpenSSL["Main: Init OpenSSL global state and LlamaBackendState"]
|
||||
InitOpenSSL --> BuildDI["Main: Build DI injector"]
|
||||
BuildDI --> InitSqlite["Main: Initialize SqliteExportService<br/>opens SQLite connection; transactions now managed per-phase via batching"]
|
||||
InitSqlite --> LogSetup["Main: create BoundedChannel log_ch, spawn Log Worker thread<br/>drains log_ch for the pipeline lifetime; all workers emit LogEntry via PipelineLogger, never spdlog directly"]
|
||||
LogSetup --> OrchRun["BiergartenPipelineOrchestrator::Run()"]
|
||||
|
||||
OrchRun --> PL1["JsonLoader.LoadBeerStyles → EnrichmentService.PreWarmBeerStyleCache"]
|
||||
OrchRun --> PL2["JsonLoader.LoadLocations → EnrichmentService.PreWarmLocationCache"]
|
||||
OrchRun --> PL3["JsonLoader.LoadForenamesByCountry"]
|
||||
OrchRun --> PL4["JsonLoader.LoadSurnamesByCountry"]
|
||||
OrchRun --> PL5["JsonLoader.LoadPersonas"]
|
||||
PL1 & PL2 & PL3 & PL4 & PL5 --> UserPhase["Orchestrator: RunUserPhase(sampled_locations)<br/>each Load* call memoized after first parse; MockCuratedDataService (--mocked) returns a fixed in-memory dataset instead, skipping these JSON files"]
|
||||
|
||||
UserPhase --> UCreate["Create BoundedChannels (loc_ch, exp_ch)"]
|
||||
|
||||
subgraph USERGEN["Phase 0 — User Generation (concurrent)"]
|
||||
direction LR
|
||||
subgraph U_PROD["Orchestrator (producer)"]
|
||||
UP1["Loop: send Location → loc_ch"]
|
||||
UP1 --> UP2["Close loc_ch"]
|
||||
end
|
||||
subgraph U_LLM["LLM Worker"]
|
||||
UL1{"loc_ch has items?"}
|
||||
UL1 -- yes --> UL2["Receive Location"]
|
||||
UL2 --> UL3["GetLocationContextFromCache(location) — guaranteed cache hit from startup"]
|
||||
UL3 --> UL4["IPersonaSelectionStrategy.SelectPersona(personas_palette_) — guaranteed cache hit; returns Persona{style_affinities, abv_range, ibu_preference, checkin_weight}"]
|
||||
UL4 --> UL5["SampleName(forenames_by_country, surnames_by_country, iso3166_1, rng)<br/>deterministic, no LLM; free function; skips the city if either map has no entry for iso3166_1"]
|
||||
UL5 --> UL6["GenerateUser(enriched_city, persona, sampled_name) via DataGenerator<br/>LLM receives EnrichedCity context + persona description + sampled name"]
|
||||
UL6 --> UL7["PipelineLogger.Log(Info, UserGeneration, city, user_id, llm)"]
|
||||
UL7 --> UL8["Send GeneratedUser → exp_ch"]
|
||||
UL8 --> UL1
|
||||
UL1 -- no --> UL9["Close exp_ch"]
|
||||
end
|
||||
subgraph U_SQL["SQLite Worker"]
|
||||
US1["BEGIN TRANSACTION"] --> US2{"exp_ch has items?"}
|
||||
US2 -- yes --> US3["Receive GeneratedUser"]
|
||||
US3 --> US4["ProcessUser(user)"]
|
||||
US4 --> US5["PipelineLogger.Log(Info, UserGeneration, city, user_id, sqlite)"]
|
||||
US5 --> US6["Append → user_pool_"]
|
||||
US6 --> US7{"Batch size reached?"}
|
||||
US7 -- yes --> US8["COMMIT and BEGIN"] --> US2
|
||||
US7 -- no --> US2
|
||||
US2 -- no --> US9["COMMIT (Final)"]
|
||||
end
|
||||
end
|
||||
|
||||
UCreate --> UP1
|
||||
UP2 -.->|loc_ch| UL1
|
||||
UL9 -.->|exp_ch| US2
|
||||
UL9 & US9 --> UserJoin["Orchestrator: Join LLM Worker, SQLite Worker"]
|
||||
|
||||
UserJoin --> BreweryPhase["Orchestrator: RunBreweryPhase(sampled_locations)"]
|
||||
BreweryPhase --> BCreate["Create BoundedChannels (loc_ch, exp_ch)"]
|
||||
|
||||
subgraph BREWGEN["Phase 1a — Brewery Generation (concurrent)"]
|
||||
direction LR
|
||||
subgraph B_PROD["Orchestrator (producer)"]
|
||||
BP1["Loop: sample User from user_pool_, pair with Location"]
|
||||
BP1 --> BP2["Send BreweryTask(Location, User) → loc_ch"]
|
||||
BP2 --> BP3["Close loc_ch"]
|
||||
end
|
||||
subgraph B_LLM["LLM Worker"]
|
||||
BL1{"loc_ch has items?"}
|
||||
BL1 -- yes --> BL2["Receive BreweryTask(Location, User)"]
|
||||
BL2 --> BL3["GetLocationContextFromCache(task.location) — guaranteed cache hit"]
|
||||
BL3 --> BL4["GenerateBrewery(enriched_city, context, task.user) via DataGenerator<br/>KV cache stays warm; brewery linked to sampled owner_user_id"]
|
||||
BL4 --> BL5["PipelineLogger.Log(Info, BreweryGeneration, city, brewery_id, llm)"]
|
||||
BL5 --> BL6["Send GeneratedBrewery → exp_ch"]
|
||||
BL6 --> BL1
|
||||
BL1 -- no --> BL7["Close exp_ch"]
|
||||
end
|
||||
subgraph B_SQL["SQLite Worker"]
|
||||
BS1["BEGIN TRANSACTION"] --> BS2{"exp_ch has items?"}
|
||||
BS2 -- yes --> BS3["Receive GeneratedBrewery"]
|
||||
BS3 --> BS4["ProcessBrewery(brewery)"]
|
||||
BS4 --> BS5["PipelineLogger.Log(Info, BreweryGeneration, city, brewery_id, sqlite)"]
|
||||
BS5 --> BS6["Append → brewery_pool_"]
|
||||
BS6 --> BS7{"Batch size reached?"}
|
||||
BS7 -- yes --> BS8["COMMIT and BEGIN"] --> BS2
|
||||
BS7 -- no --> BS2
|
||||
BS2 -- no --> BS9["COMMIT (Final)"]
|
||||
end
|
||||
end
|
||||
|
||||
BCreate --> BP1
|
||||
BP3 -.->|loc_ch| BL1
|
||||
BL7 -.->|exp_ch| BS2
|
||||
BL7 & BS9 --> BrewJoin["Orchestrator: Join LLM Worker, SQLite Worker<br/>brewery_pool_ is now fully populated"]
|
||||
|
||||
BrewJoin --> BeerPhase["Orchestrator: RunBeerPhase()"]
|
||||
BeerPhase --> BeCreate["Create BoundedChannels (brew_ch, exp_ch)"]
|
||||
|
||||
subgraph BEERGEN["Phase 1b — Beer Generation (concurrent)"]
|
||||
direction LR
|
||||
subgraph BE_PROD["Orchestrator (producer)"]
|
||||
BeP1["Loop: send Breweries → brew_ch"]
|
||||
BeP1 --> BeP2["Close brew_ch"]
|
||||
end
|
||||
subgraph BE_LLM["LLM Worker"]
|
||||
BeL1{"brew_ch has items?"}
|
||||
BeL1 -- yes --> BeL2["Receive GeneratedBrewery"]
|
||||
BeL2 --> BeL3["IBeerSelectionStrategy.SelectStyles(brewery, beer_style_palette_)"]
|
||||
BeL3 --> BeL4{"For each selected BeerStyle"}
|
||||
BeL4 -- remaining --> BeL5["GetStyleContextFromCache(style) — guaranteed cache hit; KV cache stays warm across all beer generations in this phase"]
|
||||
BeL5 --> BeL6["GenerateBeer(brewery, style_context) via DataGenerator"]
|
||||
BeL6 --> BeL7["Attach GeneratedBeer to bundle"] --> BeL4
|
||||
BeL4 -- done --> BeL8["PipelineLogger.Log(Info, BeerGeneration, city, brewery_id, llm)"]
|
||||
BeL8 --> BeL9["Send BeersBundle → exp_ch"]
|
||||
BeL9 --> BeL1
|
||||
BeL1 -- no --> BeL10["Close exp_ch"]
|
||||
end
|
||||
subgraph BE_SQL["SQLite Worker"]
|
||||
BeS1["BEGIN TRANSACTION"] --> BeS2{"exp_ch has items?"}
|
||||
BeS2 -- yes --> BeS3["Receive BeersBundle"]
|
||||
BeS3 --> BeS4{"For each beer in bundle"}
|
||||
BeS4 -- remaining --> BeS5["Set beer.brewery_id from bundle"]
|
||||
BeS5 --> BeS6["ProcessBeer(beer)"]
|
||||
BeS6 --> BeS7["Append → beer_pool_"] --> BeS4
|
||||
BeS4 -- done --> BeS8["PipelineLogger.Log(Info, BeerGeneration, city, brewery_id, sqlite)"]
|
||||
BeS8 --> BeS9{"Batch size reached?"}
|
||||
BeS9 -- yes --> BeS10["COMMIT and BEGIN"] --> BeS2
|
||||
BeS9 -- no --> BeS2
|
||||
BeS2 -- no --> BeS11["COMMIT (Final)"]
|
||||
end
|
||||
end
|
||||
|
||||
BeCreate --> BeP1
|
||||
BeP2 -.->|brew_ch| BeL1
|
||||
BeL10 -.->|exp_ch| BeS2
|
||||
BeL10 & BeS11 --> BeerJoin["Orchestrator: Join LLM Worker, SQLite Worker<br/>brewery_pool_ and beer_pool_ are now completely populated — Checkin and Follow phases may run in parallel"]
|
||||
|
||||
BeerJoin --> CheckinPhase["Orchestrator: RunCheckinPhase()"]
|
||||
BeerJoin --> FollowPhase["Orchestrator: RunFollowPhase()"]
|
||||
|
||||
subgraph CHECKIN["Checkin Generation"]
|
||||
CK1["ICheckinDistributionStrategy.AssignActivityWeights(user_pool_)<br/>weights seeded from persona.checkin_weight; J-curve profile emerges from persona distribution"]
|
||||
CK1 --> CK2["BEGIN TRANSACTION"]
|
||||
CK2 --> CK3{"For each GeneratedUser in user_pool_"}
|
||||
CK3 -- remaining --> CK4["CheckinsForUser(user, brewery_pool_.size())"]
|
||||
CK4 --> CK5{"For each checkin index"}
|
||||
CK5 -- remaining --> CK6["TimestampFor(user, index)"]
|
||||
CK6 --> CK7["Select brewery from brewery_pool_"]
|
||||
CK7 --> CK8["GenerateCheckin(user, brewery, timestamp) via DataGenerator"]
|
||||
CK8 --> CK9["ProcessCheckin(checkin)"]
|
||||
CK9 --> CK10["PipelineLogger.Log(Info, CheckinGeneration, nullopt, checkin_id, sqlite)"]
|
||||
CK10 --> CK11["Append → checkin_pool_"]
|
||||
CK11 --> CK12{"Batch size reached?"}
|
||||
CK12 -- yes --> CK13["COMMIT and BEGIN"] --> CK5
|
||||
CK12 -- no --> CK5
|
||||
CK5 -- done --> CK3
|
||||
CK3 -- done --> CK14["COMMIT (Final)"]
|
||||
end
|
||||
|
||||
subgraph FOLLOW["Follow Generation"]
|
||||
FL1["IFollowGenerationStrategy.AssignFollowWeights(user_pool_)<br/>RandomFollowStrategy: uniform weights; ActivityWeightedFollowStrategy: derived from user.activity_weight so high-activity users attract more followers"]
|
||||
FL1 --> FL2["BEGIN TRANSACTION"]
|
||||
FL2 --> FL3["IFollowGenerationStrategy.GenerateFollows(user_pool_)<br/>self-follow constraint (follower_id != followed_id) enforced here and at the DB schema level"]
|
||||
FL3 --> FL4{"For each GeneratedFollow"}
|
||||
FL4 -- remaining --> FL5["ProcessFollow(follow)"]
|
||||
FL5 --> FL6["PipelineLogger.Log(Info, FollowGeneration, nullopt, follower_id, sqlite)"]
|
||||
FL6 --> FL7["Append → follow_pool_"]
|
||||
FL7 --> FL8{"Batch size reached?"}
|
||||
FL8 -- yes --> FL9["COMMIT and BEGIN"] --> FL4
|
||||
FL8 -- no --> FL4
|
||||
FL4 -- done --> FL10["COMMIT (Final)"]
|
||||
end
|
||||
|
||||
CheckinPhase --> CK1
|
||||
FollowPhase --> FL1
|
||||
CK14 & FL10 --> PhaseJoin["Orchestrator: Join CheckinPhase, FollowPhase<br/>checkin_pool_ and follow_pool_ are now fully populated"]
|
||||
|
||||
PhaseJoin --> RatingPhase["Orchestrator: RunRatingPhase()<br/>beer selection biased by user.persona.style_affinities and abv_range; rating skew modulated per persona"]
|
||||
RatingPhase --> RT1["BEGIN TRANSACTION"]
|
||||
RT1 --> RT2{"For each GeneratedCheckin in checkin_pool_"}
|
||||
RT2 -- remaining --> RT3["Match brewery_id, select beer from beer_pool_ (same brewery_id, biased by persona affinities)"]
|
||||
RT3 --> RT4{"Beer exists for brewery?"}
|
||||
RT4 -- yes --> RT5["GenerateRating(user, beer, checkin_id) via DataGenerator"]
|
||||
RT5 --> RT6["ProcessRating(rating)"]
|
||||
RT6 --> RT7["PipelineLogger.Log(Info, RatingGeneration, nullopt, rating_id, sqlite)"]
|
||||
RT7 --> RT8{"Batch size reached?"}
|
||||
RT8 -- yes --> RT9["COMMIT and BEGIN"] --> RT2
|
||||
RT8 -- no --> RT2
|
||||
RT4 -- no --> RT10["Log warning; skip — brewery has no beers"] --> RT2
|
||||
RT2 -- done --> RT11["COMMIT (Final)"]
|
||||
|
||||
RT11 --> Teardown["Orchestrator: Finalize SqliteExportService — safely closes the DB connection"]
|
||||
Teardown --> CloseLogCh["Orchestrator: close log_ch"]
|
||||
CloseLogCh --> MainDone["Main: spdlog::info 'Pipeline complete in X ms'"]
|
||||
MainDone --> JoinLog["Main: Join Log Worker — drain guarantees no LogEntry is dropped at shutdown"]
|
||||
JoinLog --> Stop2([stop])
|
||||
@@ -1,7 +1,7 @@
|
||||
# Biergarten Data Pipeline — Class Diagram (Planned)
|
||||
|
||||
```mermaid
|
||||
--8<-- "diagrams/pipeline/planned/class.mmd"
|
||||
--8<-- "pipeline/diagrams/planned/class.mmd"
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
584
docs/pipeline/diagrams/planned/class.mmd
Normal file
584
docs/pipeline/diagrams/planned/class.mmd
Normal file
@@ -0,0 +1,584 @@
|
||||
classDiagram
|
||||
namespace DomainModels {
|
||||
class Location {
|
||||
+string city
|
||||
+string state_province
|
||||
+string iso3166_2
|
||||
+string country
|
||||
+string iso3166_1
|
||||
+vector~string~ local_languages
|
||||
+double latitude
|
||||
+double longitude
|
||||
}
|
||||
|
||||
class LocationContext {
|
||||
+string text
|
||||
+Completeness completeness
|
||||
+size_t char_count
|
||||
}
|
||||
|
||||
class Completeness {
|
||||
<<enumeration>>
|
||||
Full
|
||||
Partial
|
||||
Absent
|
||||
}
|
||||
|
||||
class EnrichedCity {
|
||||
+Location location
|
||||
+LocationContext context
|
||||
}
|
||||
|
||||
class BeerStyle {
|
||||
+string name
|
||||
+string description
|
||||
+float min_abv
|
||||
+float max_abv
|
||||
+int min_ibu
|
||||
+int max_ibu
|
||||
}
|
||||
|
||||
class BreweryResult {
|
||||
+string name_en
|
||||
+string description_en
|
||||
+string name_local
|
||||
+string description_local
|
||||
}
|
||||
|
||||
class BeerResult {
|
||||
+string name_en
|
||||
+string description_en
|
||||
+string name_local
|
||||
+string description_local
|
||||
+string style
|
||||
+float abv
|
||||
+int ibu
|
||||
}
|
||||
|
||||
class UserResult {
|
||||
+string first_name
|
||||
+string last_name
|
||||
+string gender
|
||||
+string username
|
||||
+string bio
|
||||
+float activity_weight
|
||||
}
|
||||
|
||||
class Name {
|
||||
+string first_name
|
||||
+string last_name
|
||||
+string gender
|
||||
}
|
||||
|
||||
class ForenameEntry {
|
||||
+string name
|
||||
+string gender
|
||||
}
|
||||
|
||||
class CheckinResult {
|
||||
+string checked_in_at
|
||||
+string note
|
||||
}
|
||||
|
||||
class RatingResult {
|
||||
+float score
|
||||
+string note
|
||||
}
|
||||
|
||||
class GenerationMetadata {
|
||||
+uint64_t generation_id
|
||||
+string generated_time
|
||||
+bool context_provided
|
||||
+string generated_with
|
||||
}
|
||||
|
||||
class GeneratedBrewery {
|
||||
+uint64_t brewery_id
|
||||
+Location location
|
||||
+BreweryResult brewery
|
||||
+Completeness context_completeness
|
||||
+GenerationMetadata metadata
|
||||
}
|
||||
|
||||
class GeneratedBeer {
|
||||
+uint64_t beer_id
|
||||
+uint64_t brewery_id
|
||||
+Location location
|
||||
+BeerStyle style
|
||||
+BeerResult beer
|
||||
+GenerationMetadata metadata
|
||||
}
|
||||
|
||||
class GeneratedUser {
|
||||
+uint64_t user_id
|
||||
+Location location
|
||||
+UserResult user
|
||||
+string email
|
||||
+string date_of_birth
|
||||
+string password
|
||||
+GenerationMetadata metadata
|
||||
}
|
||||
|
||||
class GeneratedCheckin {
|
||||
+uint64_t checkin_id
|
||||
+uint64_t user_id
|
||||
+uint64_t brewery_id
|
||||
+CheckinResult checkin
|
||||
+GenerationMetadata metadata
|
||||
}
|
||||
|
||||
class GeneratedRating {
|
||||
+uint64_t user_id
|
||||
+uint64_t beer_id
|
||||
+uint64_t checkin_id
|
||||
+RatingResult rating
|
||||
+GenerationMetadata metadata
|
||||
}
|
||||
|
||||
class GeneratedFollow {
|
||||
+uint64_t follower_id
|
||||
+uint64_t followed_id
|
||||
+GenerationMetadata metadata
|
||||
}
|
||||
|
||||
class UserPersona {
|
||||
+string name
|
||||
+string description
|
||||
+vector~string~ style_affinities
|
||||
}
|
||||
}
|
||||
note for ForenameEntry "forename_list/surname_list are unordered_set aliases, keyed by ISO 3166-1 country code via ICuratedDataService. SampleName() is a free function, not a method, sourced from popular-names-by-country-dataset (CC0)."
|
||||
note for GeneratedUser "email, date_of_birth, and password are generated programmatically by the orchestrator (not LLM-authored) so a downstream auth-account seeding consumer can register real accounts from this export."
|
||||
ForenameEntry ..> Name : SampleName() produces
|
||||
LocationContext *-- Completeness
|
||||
|
||||
namespace DomainApplicationConfiguration {
|
||||
class SamplingOptions {
|
||||
+float temperature
|
||||
+float top_p
|
||||
+uint32_t top_k
|
||||
+uint32_t n_ctx
|
||||
+int seed
|
||||
}
|
||||
|
||||
class GeneratorOptions {
|
||||
+path model_path
|
||||
+bool use_mocked
|
||||
+optional~SamplingOptions~ sampling
|
||||
}
|
||||
|
||||
class PipelineOptions {
|
||||
+path output_path
|
||||
+path log_path
|
||||
}
|
||||
|
||||
class ApplicationOptions {
|
||||
+GeneratorOptions generator
|
||||
+PipelineOptions pipeline
|
||||
}
|
||||
}
|
||||
ApplicationOptions *-- GeneratorOptions
|
||||
ApplicationOptions *-- PipelineOptions
|
||||
GeneratorOptions o-- SamplingOptions
|
||||
|
||||
namespace DomainPolicy {
|
||||
class ContextStrategy {
|
||||
<<interface>>
|
||||
+QueriesFor(loc) vector~string~
|
||||
+MaxContextChars() size_t
|
||||
}
|
||||
|
||||
class BreweryContextStrategy {
|
||||
+QueriesFor(loc) vector~string~
|
||||
+MaxContextChars() size_t
|
||||
}
|
||||
|
||||
class BeerContextStrategy {
|
||||
+QueriesFor(loc) vector~string~
|
||||
+MaxContextChars() size_t
|
||||
}
|
||||
|
||||
class SamplingStrategy {
|
||||
<<interface>>
|
||||
+Sample(locations) vector~Location~
|
||||
}
|
||||
|
||||
class UniformSamplingStrategy {
|
||||
-size_t sample_size_
|
||||
+Sample(locations) vector~Location~
|
||||
}
|
||||
|
||||
class BeerSelectionStrategy {
|
||||
<<interface>>
|
||||
+SelectStyles(brewery, palette) vector~BeerStyle~
|
||||
}
|
||||
|
||||
class RandomBeerSelectionStrategy {
|
||||
-mt19937 rng_
|
||||
-size_t min_beers_
|
||||
-size_t max_beers_
|
||||
+SelectStyles(brewery, palette) vector~BeerStyle~
|
||||
}
|
||||
|
||||
class CheckinDistributionStrategy {
|
||||
<<interface>>
|
||||
+AssignActivityWeights(users) void
|
||||
+CheckinsForUser(user, brewery_count) size_t
|
||||
+TimestampFor(user, index) string
|
||||
}
|
||||
|
||||
class JCurveCheckinStrategy {
|
||||
-mt19937 rng_
|
||||
+AssignActivityWeights(users) void
|
||||
+CheckinsForUser(user, brewery_count) size_t
|
||||
+TimestampFor(user, index) string
|
||||
}
|
||||
|
||||
class RandomCheckinStrategy {
|
||||
-mt19937 rng_
|
||||
-size_t min_checkins_
|
||||
-size_t max_checkins_
|
||||
+AssignActivityWeights(users) void
|
||||
+CheckinsForUser(user, brewery_count) size_t
|
||||
+TimestampFor(user, index) string
|
||||
}
|
||||
|
||||
class FollowGenerationStrategy {
|
||||
<<interface>>
|
||||
+GenerateFollows(users) vector~GeneratedFollow~
|
||||
}
|
||||
|
||||
class RandomFollowStrategy {
|
||||
-mt19937 rng_
|
||||
-size_t min_follows_
|
||||
-size_t max_follows_
|
||||
+GenerateFollows(users) vector~GeneratedFollow~
|
||||
}
|
||||
|
||||
class ActivityWeightedFollowStrategy {
|
||||
-mt19937 rng_
|
||||
-size_t min_follows_
|
||||
-size_t max_follows_
|
||||
+GenerateFollows(users) vector~GeneratedFollow~
|
||||
}
|
||||
}
|
||||
|
||||
namespace InfrastructureLogging {
|
||||
class LogLevel {
|
||||
<<enumeration>>
|
||||
Debug
|
||||
Info
|
||||
Warn
|
||||
Error
|
||||
}
|
||||
|
||||
class PipelinePhase {
|
||||
<<enumeration>>
|
||||
Startup
|
||||
UserGeneration
|
||||
BreweryAndBeerGeneration
|
||||
CheckinGeneration
|
||||
RatingGeneration
|
||||
FollowGeneration
|
||||
Teardown
|
||||
}
|
||||
|
||||
class LogEntry {
|
||||
+time_point timestamp
|
||||
+LogLevel level
|
||||
+PipelinePhase phase
|
||||
+string message
|
||||
+optional~string~ worker
|
||||
}
|
||||
|
||||
class ILogger {
|
||||
<<interface>>
|
||||
+Log(entry) void
|
||||
}
|
||||
|
||||
class LogProducer {
|
||||
-BoundedChannel~LogEntry~ channel_
|
||||
+Log(entry) void
|
||||
}
|
||||
|
||||
class LogDispatcher {
|
||||
-BoundedChannel~LogEntry~ channel_
|
||||
+Run() void
|
||||
-ToSpdlogLevel(level) level_enum
|
||||
}
|
||||
}
|
||||
LogEntry *-- LogLevel
|
||||
LogEntry *-- PipelinePhase
|
||||
ILogger <|.. LogProducer
|
||||
LogProducer ..> LogEntry : emits
|
||||
LogDispatcher ..> LogEntry : consumes
|
||||
|
||||
namespace InfrastructurePipelineChannel {
|
||||
class BoundedChannel~T~ {
|
||||
-queue~T~ queue_
|
||||
-mutex mutex_
|
||||
-condition_variable not_full_
|
||||
-condition_variable not_empty_
|
||||
-size_t capacity_
|
||||
-bool closed_
|
||||
+Send(item) void
|
||||
+Receive() optional~T~
|
||||
+Close() void
|
||||
}
|
||||
}
|
||||
|
||||
namespace InfrastructureDataPreloading {
|
||||
class ICuratedDataService {
|
||||
<<interface>>
|
||||
+LoadLocations(filepath) vector~Location~
|
||||
+LoadBeerStyles(filepath) vector~BeerStyle~
|
||||
+LoadPersonas(filepath) vector~UserPersona~
|
||||
+LoadForenamesByCountry(filepath) unordered_map~string, forename_list~
|
||||
+LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~
|
||||
}
|
||||
|
||||
class JsonLoader {
|
||||
+LoadLocations(filepath) vector~Location~
|
||||
+LoadBeerStyles(filepath) vector~BeerStyle~
|
||||
+LoadPersonas(filepath) vector~UserPersona~
|
||||
+LoadForenamesByCountry(filepath) unordered_map~string, forename_list~
|
||||
+LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~
|
||||
}
|
||||
|
||||
class MockCuratedDataService {
|
||||
+LoadLocations(filepath) vector~Location~
|
||||
+LoadBeerStyles(filepath) vector~BeerStyle~
|
||||
+LoadPersonas(filepath) vector~UserPersona~
|
||||
+LoadForenamesByCountry(filepath) unordered_map~string, forename_list~
|
||||
+LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~
|
||||
}
|
||||
}
|
||||
note for JsonLoader "Each Load* call is memoized after its first parse — implemented today, ahead of LoadBeerStyles which is still planned."
|
||||
note for MockCuratedDataService "Fixed in-memory dataset for --mocked mode, implemented today for Locations/Personas/Forenames/Surnames; would need a LoadBeerStyles fixture too once that method is implemented."
|
||||
|
||||
namespace InfrastructureEnrichment {
|
||||
class EnrichmentService {
|
||||
<<interface>>
|
||||
+GetLocationContext(loc, strategy) LocationContext
|
||||
}
|
||||
|
||||
class WikipediaService {
|
||||
-unique_ptr~WebClient~ client_
|
||||
-unordered_map~string, string~ extract_cache_
|
||||
+GetLocationContext(loc, strategy) LocationContext
|
||||
-FetchExtract(query) string
|
||||
}
|
||||
|
||||
class WebClient {
|
||||
<<interface>>
|
||||
+Get(url) string
|
||||
+UrlEncode(value) string
|
||||
}
|
||||
|
||||
class HttpWebClient {
|
||||
+Get(url) string
|
||||
+UrlEncode(value) string
|
||||
}
|
||||
}
|
||||
|
||||
namespace InfrastructurePrompting {
|
||||
class IPromptDirectory {
|
||||
<<interface>>
|
||||
+Load(key) string
|
||||
}
|
||||
|
||||
class PromptDirectory {
|
||||
-path prompt_dir_
|
||||
-unordered_map~string, string~ cache_
|
||||
+PromptDirectory(prompt_dir)
|
||||
+Load(key) string
|
||||
}
|
||||
}
|
||||
IPromptDirectory <|.. PromptDirectory
|
||||
|
||||
namespace InfrastructureDataGeneration {
|
||||
class DataGenerator {
|
||||
<<interface>>
|
||||
+GenerateBrewery(location, context) BreweryResult
|
||||
+GenerateBeer(brewery_id, location, context, style) BeerResult
|
||||
+GenerateUser(city, persona, name) UserResult
|
||||
+GenerateCheckin(user, brewery, timestamp) CheckinResult
|
||||
+GenerateRating(user, beer, checkin_id) RatingResult
|
||||
}
|
||||
|
||||
class MockGenerator {
|
||||
+GenerateBrewery(...) BreweryResult
|
||||
+GenerateBeer(...) BeerResult
|
||||
+GenerateUser(...) UserResult
|
||||
+GenerateCheckin(...) CheckinResult
|
||||
+GenerateRating(...) RatingResult
|
||||
-DeterministicHash(location) size_t
|
||||
}
|
||||
|
||||
class LlamaGenerator {
|
||||
-ModelHandle model_
|
||||
-ContextHandle context_
|
||||
-unique_ptr~PromptFormatter~ prompt_formatter_
|
||||
-unique_ptr~IPromptDirectory~ prompt_directory_
|
||||
-mt19937 rng_
|
||||
+GenerateBrewery(...) BreweryResult
|
||||
+GenerateBeer(...) BeerResult
|
||||
+GenerateUser(...) UserResult
|
||||
+GenerateCheckin(...) CheckinResult
|
||||
+GenerateRating(...) RatingResult
|
||||
-Load(opts) void
|
||||
-Infer(system_prompt, user_prompt, max_tokens, grammar) string
|
||||
-ValidateModelArchitecture() void
|
||||
}
|
||||
|
||||
class PromptFormatter {
|
||||
<<interface>>
|
||||
+Format(system_prompt, user_prompt) string
|
||||
+ExpectedArchitecture() string_view
|
||||
}
|
||||
|
||||
class Gemma4JinjaPromptFormatter {
|
||||
+Format(system_prompt, user_prompt) string
|
||||
+ExpectedArchitecture() string_view
|
||||
}
|
||||
}
|
||||
|
||||
namespace InfrastructureDataExport {
|
||||
class ExportService {
|
||||
<<interface>>
|
||||
+Initialize() void
|
||||
+ProcessBrewery(brewery) uint64_t
|
||||
+ProcessBeer(beer) uint64_t
|
||||
+ProcessUser(user) uint64_t
|
||||
+ProcessCheckin(checkin) uint64_t
|
||||
+ProcessRating(rating) void
|
||||
+ProcessFollow(follow) void
|
||||
+Finalize() void
|
||||
}
|
||||
|
||||
class SqliteExportService {
|
||||
-unique_ptr~DateTimeProvider~ date_time_provider_
|
||||
-SqliteDatabaseHandle db_handle_
|
||||
-SqliteStatementHandle insert_location_stmt_
|
||||
-SqliteStatementHandle insert_brewery_stmt_
|
||||
-SqliteStatementHandle insert_beer_stmt_
|
||||
-SqliteStatementHandle insert_user_stmt_
|
||||
-SqliteStatementHandle insert_checkin_stmt_
|
||||
-SqliteStatementHandle insert_rating_stmt_
|
||||
-SqliteStatementHandle insert_follow_stmt_
|
||||
-bool transaction_open_
|
||||
-unordered_map~string, uint64_t~ location_cache_
|
||||
-unordered_map~string, uint64_t~ brewery_cache_
|
||||
+Initialize() void
|
||||
+ProcessRecord(brewery) uint64_t
|
||||
+ProcessRecord(beer) uint64_t
|
||||
+ProcessRecord(user) uint64_t
|
||||
+ProcessRecord(checkin) uint64_t
|
||||
+ProcessRecord(rating) void
|
||||
+ProcessRecord(follow) void
|
||||
+Finalize() void
|
||||
-InitializeSchema() void
|
||||
-PrepareStatements() void
|
||||
-RollbackAndCloseNoThrow() void
|
||||
-FinalizeStatements() void
|
||||
}
|
||||
|
||||
class DateTimeProvider {
|
||||
<<interface>>
|
||||
+GetUtcTimestamp() string
|
||||
}
|
||||
|
||||
class SystemDateTimeProvider {
|
||||
+GetUtcTimestamp() string
|
||||
}
|
||||
}
|
||||
|
||||
class BiergartenPipelineOrchestrator {
|
||||
-unique_ptr~ICuratedDataService~ curated_data_service_
|
||||
-unique_ptr~EnrichmentService~ enrichment_service_
|
||||
-unique_ptr~DataGenerator~ generator_
|
||||
-unique_ptr~Logger~ logger_
|
||||
-unique_ptr~ExportService~ exporter_
|
||||
-unique_ptr~ContextStrategy~ brewery_context_strategy_
|
||||
-unique_ptr~SamplingStrategy~ sampling_strategy_
|
||||
-unique_ptr~BeerSelectionStrategy~ beer_selection_strategy_
|
||||
-unique_ptr~CheckinDistributionStrategy~ checkin_strategy_
|
||||
-unique_ptr~FollowGenerationStrategy~ follow_strategy_
|
||||
-vector~BeerStyle~ beer_style_palette_
|
||||
-ApplicationOptions options_
|
||||
-vector~GeneratedUser~ user_pool_
|
||||
-vector~GeneratedBrewery~ brewery_pool_
|
||||
-vector~GeneratedBeer~ beer_pool_
|
||||
-vector~GeneratedCheckin~ checkin_pool_
|
||||
-vector~GeneratedFollow~ follow_pool_
|
||||
+Run() bool
|
||||
-RunUserPhase(locations) void
|
||||
-RunBreweryAndBeerPhase(locations) void
|
||||
-RunCheckinPhase() void
|
||||
-RunRatingPhase() void
|
||||
-RunFollowPhase() void
|
||||
}
|
||||
|
||||
BiergartenPipelineOrchestrator *-- ICuratedDataService
|
||||
BiergartenPipelineOrchestrator *-- EnrichmentService
|
||||
BiergartenPipelineOrchestrator *-- DataGenerator
|
||||
BiergartenPipelineOrchestrator *-- ExportService
|
||||
BiergartenPipelineOrchestrator *-- CheckinDistributionStrategy
|
||||
BiergartenPipelineOrchestrator *-- FollowGenerationStrategy
|
||||
BiergartenPipelineOrchestrator *-- SamplingStrategy
|
||||
BiergartenPipelineOrchestrator *-- BeerSelectionStrategy
|
||||
BiergartenPipelineOrchestrator *-- ApplicationOptions
|
||||
BiergartenPipelineOrchestrator *-- Logger
|
||||
|
||||
BiergartenPipelineOrchestrator "1" *-- "0..*" GeneratedUser : user_pool_
|
||||
BiergartenPipelineOrchestrator "1" *-- "0..*" GeneratedBrewery : brewery_pool_
|
||||
BiergartenPipelineOrchestrator "1" *-- "0..*" GeneratedBeer : beer_pool_
|
||||
BiergartenPipelineOrchestrator "1" *-- "0..*" GeneratedCheckin : checkin_pool_
|
||||
BiergartenPipelineOrchestrator "1" *-- "0..*" GeneratedFollow : follow_pool_
|
||||
|
||||
ICuratedDataService <|.. JsonLoader
|
||||
ICuratedDataService <|.. MockCuratedDataService
|
||||
Logger <|.. PipelineLogger
|
||||
ContextStrategy <|.. BreweryContextStrategy
|
||||
ContextStrategy <|.. BeerContextStrategy
|
||||
SamplingStrategy <|.. UniformSamplingStrategy
|
||||
BeerSelectionStrategy <|.. RandomBeerSelectionStrategy
|
||||
CheckinDistributionStrategy <|.. JCurveCheckinStrategy
|
||||
CheckinDistributionStrategy <|.. RandomCheckinStrategy
|
||||
FollowGenerationStrategy <|.. RandomFollowStrategy
|
||||
FollowGenerationStrategy <|.. ActivityWeightedFollowStrategy
|
||||
EnrichmentService <|.. WikipediaService
|
||||
WebClient <|.. HttpWebClient
|
||||
DataGenerator <|.. MockGenerator
|
||||
DataGenerator <|.. LlamaGenerator
|
||||
PromptFormatter <|.. Gemma4JinjaPromptFormatter
|
||||
ExportService <|.. SqliteExportService
|
||||
DateTimeProvider <|.. SystemDateTimeProvider
|
||||
|
||||
WikipediaService *-- WebClient
|
||||
WikipediaService ..> ContextStrategy
|
||||
LlamaGenerator *-- PromptFormatter
|
||||
LlamaGenerator *-- IPromptDirectory
|
||||
LlamaGenerator ..> GeneratorOptions
|
||||
SqliteExportService *-- DateTimeProvider
|
||||
|
||||
PipelineLogger o-- BoundedChannel : logs to
|
||||
LogWorker o-- BoundedChannel : drains from
|
||||
|
||||
EnrichedCity *-- Location
|
||||
EnrichedCity *-- LocationContext
|
||||
GeneratedBrewery *-- Location
|
||||
GeneratedBrewery *-- BreweryResult
|
||||
GeneratedBrewery *-- GenerationMetadata
|
||||
GeneratedBeer *-- Location
|
||||
GeneratedBeer *-- BeerStyle
|
||||
GeneratedBeer *-- BeerResult
|
||||
GeneratedBeer *-- GenerationMetadata
|
||||
GeneratedUser *-- Location
|
||||
GeneratedUser *-- UserResult
|
||||
GeneratedUser *-- GenerationMetadata
|
||||
GeneratedCheckin *-- CheckinResult
|
||||
GeneratedCheckin *-- GenerationMetadata
|
||||
GeneratedRating *-- RatingResult
|
||||
GeneratedRating *-- GenerationMetadata
|
||||
GeneratedFollow *-- GenerationMetadata
|
||||
Reference in New Issue
Block a user