Pipeline: Add LLM and mocked user generation to the pipeline (#230)

docs/pipeline/diagrams/current/activity.puml

@@ -93,6 +93,77 @@ while (For each sampled Location?) is (Remaining cities)
   endif
 endwhile (Done)
 
+|#EAF0E8|BiergartenPipelineOrchestrator|
+:GenerateUsers(enriched_cities);
+
+|#E2EBDC|JsonLoader|
+:LoadPersonas("personas.json");
+:LoadForenamesByCountry("forenames-by-country.json");
+:LoadSurnamesByCountry("surnames-by-country.json");
+note right
+  Each call is cached after its first parse.
+  MockCuratedDataService (--mocked) returns a
+  fixed in-memory dataset instead and skips
+  these three JSON files entirely.
+end note
+
+|#EAF0E8|BiergartenPipelineOrchestrator|
+while (For each EnrichedCity?) is (Remaining cities)
+  :SampleName(forenames_by_country,\n  surnames_by_country, location.iso3166_1);
+  if (Names available for country?) then (no)
+    :Log warning, skip city;
+  else (yes)
+    :Sample UserPersona (uniform random);
+
+    |#E5EDE1|DataGenerator|
+    if (Generator Mode) then (MockGenerator)
+      :DeterministicHash & Format;
+    else (LlamaGenerator)
+      :prompt_directory_->Load("USER_GENERATION");
+      note right
+        Resolves to USER_GENERATION.md inside --prompt-dir.
+      end note
+      repeat
+        :Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
+        :ValidateUserJson(raw, user);
+        if (Is JSON Valid?) then (yes)
+          break
+        else (no)
+          :Attempt++;
+          :Append validation error to retry prompt;
+        endif
+      repeat while (Attempt < 3?) is (yes)
+    endif
+
+    |#EAF0E8|BiergartenPipelineOrchestrator|
+    if (Generation successful?) then (yes)
+      :BuildEmail(name, used_email_local_parts);
+      :GenerateDateOfBirth(rng);
+      note right
+        email and date_of_birth are generated
+        programmatically, never LLM-authored.
+      end note
+
+      |#E0EAE0|SqliteExportService|
+      :ProcessRecord(UserRecord);
+      if (Location in cache?) then (yes)
+        :Reuse location_id;
+      else (no)
+        :Insert Location & Cache ID;
+      endif
+      :Insert User (FK: location_id);
+
+      if (Exception caught during insert?) then (yes)
+        |#EAF0E8|BiergartenPipelineOrchestrator|
+        :Log warning "SQLite export failed";
+      else (no)
+      endif
+    else (no)
+      :Log warning "Generation failed, skipping...";
+    endif
+  endif
+endwhile (Done)
+
 |#EAF0E8|BiergartenPipelineOrchestrator|
 :GenerateBreweries(enriched_cities);
 

docs/pipeline/diagrams/current/class.puml

@@ -30,11 +30,14 @@ class BiergartenPipelineOrchestrator {
   - context_service_ : std::unique_ptr<IEnrichmentService>
   - generator_ : std::unique_ptr<DataGenerator>
   - exporter_ : std::unique_ptr<IExportService>
+  - curated_data_service_ : std::unique_ptr<ICuratedDataService>
   - application_options_ : ApplicationOptions
-  - generated_breweries_ : std::vector<GeneratedBrewery>
+  - generated_breweries_ : std::vector<BreweryRecord>
+  - generated_users_ : std::vector<UserRecord>
   + Run() : bool
   - QueryCitiesWithCountries() : std::vector<Location>
   - GenerateBreweries(cities : std::span<const EnrichedCity>) : void
+  - GenerateUsers(cities : std::span<const EnrichedCity>) : void
   - LogResults() : void
 }
 
@@ -113,13 +116,14 @@ class HttpWebClient {
 
 interface DataGenerator <<interface>> {
   + GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
-  + GenerateUser(locale : const std::string&) : UserResult
+  + GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
 }
 
 class MockGenerator {
-  + GenerateBrewery(...) : BreweryResult
-  + GenerateUser(...) : UserResult
+  + GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
+  + GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
   - DeterministicHash(location : const Location&) : size_t
+  - DeterministicHash(location : const Location&, persona : const UserPersona&, name : const Name&) : size_t
 }
 
 class LlamaGenerator {
@@ -153,13 +157,49 @@ class PromptDirectory {
   + Load(key : std::string_view) : std::string
 }
 
-class JsonLoader {
-  + {static} LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
+interface ICuratedDataService <<interface>> {
+  + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+  + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+  + LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+  + LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
 }
 
+class JsonLoader {
+  - cache_ : cache
+  + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+  + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+  + LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+  + LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
+}
+
+note right of JsonLoader
+  Each Load* method memoizes its result in
+  cache_ on first call; later calls on the
+  same instance return the cached value
+  without re-parsing.
+end note
+
+class MockCuratedDataService {
+  - locations_ : std::vector<Location>
+  - personas_ : std::vector<UserPersona>
+  - forenames_by_country_ : std::unordered_map<std::string, forename_list>
+  - surnames_by_country_ : std::unordered_map<std::string, surname_list>
+  + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+  + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+  + LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+  + LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
+}
+
+note right of MockCuratedDataService
+  Fixed 4-location / 3-persona dataset
+  (US, DE, FR, BE) used in --mocked runs;
+  filepath arguments are ignored.
+end note
+
 interface IExportService <<interface>> {
   + Initialize() : void
-  + ProcessRecord(brewery : const GeneratedBrewery&) : void
+  + ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+  + ProcessRecord(user : const UserRecord&) : uint64_t
   + Finalize() : void
 }
 
@@ -167,15 +207,18 @@ class SqliteExportService {
   - date_time_provider_ : std::unique_ptr<IDateTimeProvider>
   - run_timestamp_utc_ : std::string
   - database_path_ : std::filesystem::path
-  - db_handle_ : sqlite3*
-  - insert_location_stmt_ : sqlite3_stmt*
-  - insert_brewery_stmt_ : sqlite3_stmt*
+  - db_handle_ : SqliteDatabaseHandle
+  - insert_location_stmt_ : SqliteStatementHandle
+  - insert_brewery_stmt_ : SqliteStatementHandle
+  - insert_user_stmt_ : SqliteStatementHandle
   - transaction_open_ : bool
   - location_cache_ : std::unordered_map<std::string, sqlite3_int64>
   + Initialize() : void
-  + ProcessRecord(brewery : const GeneratedBrewery&) : void
+  + ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+  + ProcessRecord(user : const UserRecord&) : uint64_t
   + Finalize() : void
   - InitializeSchema() : void
+  - ResolveLocationId(location : const Location&) : sqlite3_int64
 }
 
 interface IDateTimeProvider <<interface>> {
@@ -191,6 +234,7 @@ BiergartenPipelineOrchestrator *-- ILogger : owns
 BiergartenPipelineOrchestrator *-- IEnrichmentService : owns
 BiergartenPipelineOrchestrator *-- DataGenerator : owns
 BiergartenPipelineOrchestrator *-- IExportService : owns
+BiergartenPipelineOrchestrator *-- ICuratedDataService : owns
 
 LogEntry *-- LogLevel
 LogEntry *-- PipelinePhase
@@ -215,7 +259,8 @@ LlamaGenerator *-- IPromptDirectory : uses
 IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements
 IPromptDirectory <|.. PromptDirectory : implements
 
-BiergartenPipelineOrchestrator ..> JsonLoader : uses
+ICuratedDataService <|.. JsonLoader : implements
+ICuratedDataService <|.. MockCuratedDataService : implements
 
 IExportService <|.. SqliteExportService : implements
 SqliteExportService *-- IDateTimeProvider : owns

docs/pipeline/diagrams/planned/activity.puml

@@ -43,10 +43,18 @@ fork again
   :EnrichmentService::PreWarmLocationCache(sampled_locations);
 end fork
 fork
-  :JsonLoader::LoadNamesByCountry("names-by-country.json");
+  :JsonLoader::LoadForenamesByCountry(\n  "forenames-by-country.json");
+fork again
+  :JsonLoader::LoadSurnamesByCountry(\n  "surnames-by-country.json");
 fork again
   :JsonLoader::LoadPersonas("personas.json");
 end fork
+note right
+  Each call is memoized after its first parse
+  (implemented today). MockCuratedDataService
+  returns a fixed in-memory dataset instead
+  under --mocked, skipping these JSON files.
+end note
 
 ' ═══════════════════════════════════════════
 ' PHASE 0 — USER GENERATION
@@ -82,11 +90,15 @@ fork again
       ibu_preference, checkin_weight.
     end note
 
-    :NamesByCountry::SampleName(\n  location.iso3166_1);
+    :SampleName(forenames_by_country,\n  surnames_by_country, location.iso3166_1, rng);
     note right
       Deterministic lookup -- no LLM involved.
-      Name selected from pre-keyed table
+      Free function (implemented today in
+      generate_users.cc), not a class method.
+      Name selected from pre-keyed maps
       and passed into the generation prompt.
+      Skips the city if either map has no
+      entry for iso3166_1.
     end note
 
     :GenerateUser(enriched_city, persona, sampled_name)\nvia DataGenerator;

docs/pipeline/diagrams/planned/class.puml

@@ -67,11 +67,50 @@ package "Domain: Models" {
   }
 
   class UserResult {
+    + first_name : std::string
+    + last_name : std::string
+    + gender : std::string
     + username : std::string
     + bio : std::string
     + activity_weight : float
   }
 
+  class Name {
+    + first_name : std::string
+    + last_name : std::string
+    + gender : std::string
+  }
+
+  class ForenameEntry {
+    + name : std::string
+    + gender : std::string
+  }
+
+  note right of ForenameEntry
+    forename_list = std::unordered_set<ForenameEntry>
+    surname_list = std::unordered_set<std::string>
+    (aliases declared in curated_data_service.h).
+    ICuratedDataService::LoadForenamesByCountry() /
+    LoadSurnamesByCountry() each return a flat
+    std::unordered_map<std::string, forename_list>
+    or <std::string, surname_list> keyed by ISO
+    3166-1 country code -- no NamesByCountry
+    wrapper class. SampleName(forenames_by_country,
+    surnames_by_country, iso3166_1, rng) is a free
+    function, not a method, so it can be called by
+    any per-city worker without owning the maps.
+    Sourced from popular-names-by-country-dataset
+    (CC0) -- see ETHICS-AND-KNOWN-ISSUES.md. Gender
+    is preserved per forename so it can be threaded
+    through to persona/bio generation later; surnames
+    carry no gender in the source data. Pairing
+    happens at sample time, not at fixture-authoring
+    time, so no information from the source dataset
+    is discarded up front.
+  end note
+
+  ForenameEntry ..> Name : SampleName() produces
+
   class CheckinResult {
     + checked_in_at : std::string
     + note : std::string
@@ -110,9 +149,20 @@ package "Domain: Models" {
     + user_id : uint64_t
     + location : Location
     + user : UserResult
+    + email : std::string
+    + date_of_birth : std::string
+    + password : std::string
     + metadata : GenerationMetadata
   }
 
+  note right of GeneratedUser
+    email, date_of_birth, and password are
+    programmatically generated by the orchestrator
+    (not LLM-authored) so a downstream auth-account
+    seeding consumer can register real accounts from
+    this export. See ROADMAP.md §1.
+  end note
+
   class GeneratedCheckin {
     + checkin_id : uint64_t
     + user_id : uint64_t
@@ -318,20 +368,43 @@ package "Infrastructure: Pipeline Channel" {
 
 package "Infrastructure: Data Preloading" {
 
-  interface DataPreloader <<interface>> {
-    + LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
-    + LoadBeerStyles(filepath : const std::filesystem::path&) : std::vector<BeerStyle>
-    + LoadPersonas(filepath : const std::filesystem::path&) : std::vector<Persona>
-    + LoadNamesByCountry(filepath : const std::filesystem::path&) : NamesByCountry
+  interface ICuratedDataService <<interface>> {
+    + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+    + LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+    + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+    + LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+    + LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
   }
 
   class JsonLoader {
-    + LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
-    + LoadBeerStyles(filepath : const std::filesystem::path&) : std::vector<BeerStyle>
-    + LoadPersonas(filepath : const std::filesystem::path&) : std::vector<Persona>
-    + LoadNamesByCountry(filepath : const std::filesystem::path&) : NamesByCountry
+    + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+    + LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+    + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+    + LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+    + LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
   }
 
+  note right of JsonLoader
+    Each Load* call is memoized after its first
+    parse -- implemented today, ahead of the rest
+    of this package (LoadBeerStyles is still planned).
+  end note
+
+  class MockCuratedDataService {
+    + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+    + LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+    + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+    + LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+    + LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
+  }
+
+  note right of MockCuratedDataService
+    Fixed in-memory dataset used in --mocked mode,
+    implemented today for Locations/Personas/
+    Forenames/Surnames; would need a LoadBeerStyles
+    fixture too once that method is implemented.
+  end note
+
 }
 
 package "Infrastructure: Enrichment" {
@@ -380,7 +453,7 @@ package "Infrastructure: Data Generation" {
   interface DataGenerator <<interface>> {
     + GenerateBrewery(location : const Location&,\n  context : const LocationContext&) : BreweryResult
     + GenerateBeer(brewery_id : uint64_t,\n  location : const Location&,\n  context : const LocationContext&,\n  style : const BeerStyle&) : BeerResult
-    + GenerateUser(location : const Location&) : UserResult
+    + GenerateUser(city : const EnrichedCity&,\n  persona : const UserPersona&,\n  name : const Name&) : UserResult
     + GenerateCheckin(user : const GeneratedUser&,\n  brewery : const GeneratedBrewery&,\n  timestamp : const std::string&) : CheckinResult
     + GenerateRating(user : const GeneratedUser&,\n  beer : const GeneratedBeer&,\n  checkin_id : uint64_t) : RatingResult
   }
@@ -473,7 +546,7 @@ package "Infrastructure: Data Export" {
 }
 
 class BiergartenPipelineOrchestrator {
-  - preloader_ : std::unique_ptr<DataPreloader>
+  - curated_data_service_ : std::unique_ptr<ICuratedDataService>
   - enrichment_service_ : std::unique_ptr<EnrichmentService>
   - generator_ : std::unique_ptr<DataGenerator>
   - logger_ : std::unique_ptr<Logger>
@@ -501,7 +574,7 @@ class BiergartenPipelineOrchestrator {
 }
 
 ' --- Orchestration Aggregations (Services & Strategies) ---
-BiergartenPipelineOrchestrator *--  DataPreloader
+BiergartenPipelineOrchestrator *--  ICuratedDataService
 BiergartenPipelineOrchestrator *--  EnrichmentService
 BiergartenPipelineOrchestrator *--  DataGenerator
 BiergartenPipelineOrchestrator *--  ExportService
@@ -520,7 +593,8 @@ BiergartenPipelineOrchestrator *-- "0..*" GeneratedCheckin : checkin_pool_
 BiergartenPipelineOrchestrator *-- "0..*" GeneratedFollow  : follow_pool_
 
 ' --- Interfaces & Implementations ---
-DataPreloader                  <|.. JsonLoader
+ICuratedDataService            <|.. JsonLoader
+ICuratedDataService            <|.. MockCuratedDataService
 Logger                         <|.. PipelineLogger
 ContextStrategy                <|.. BreweryContextStrategy
 ContextStrategy                <|.. BeerContextStrategy