diff --git a/docs/pipeline/diagrams/current/activity.mmd b/docs/pipeline/diagrams/current/activity.mmd index 0e011519..c4d72e72 100644 --- a/docs/pipeline/diagrams/current/activity.mmd +++ b/docs/pipeline/diagrams/current/activity.mmd @@ -10,7 +10,7 @@ flowchart TD Run --> SqliteInit["SqliteExportService.Initialize()
builds biergarten_seed_<UTC datetime>.sqlite (numeric suffix if it exists), opens DB connection, executes schema DDL, begins transaction"] SqliteInit --> QueryLoc[Orchestrator: QueryLocations] - QueryLoc --> LoadLoc["JsonLoader.LoadLocations(locations.json)"] + QueryLoc --> LoadLoc["CuratedJsonDataService.LoadLocations()"] LoadLoc --> SampleLoc["std::ranges::sample(all_locations, --location-count) — defaults to 10"] SampleLoc --> EnrichLoop{For each sampled Location} @@ -20,7 +20,7 @@ flowchart TD LookupFailed -- no --> StoreEnriched["Store EnrichedCity{Location, region_context}"] --> EnrichLoop EnrichLoop -- done --> GenUsers[Orchestrator: GenerateUsers enriched_cities] - GenUsers --> LoadUserData["JsonLoader: LoadPersonas / LoadForenamesByCountry / LoadSurnamesByCountry
each memoized after first parse; MockCuratedDataService (--mocked) returns a fixed in-memory dataset instead, skipping these JSON files"] + GenUsers --> LoadUserData["CuratedJsonDataService: LoadPersonas / LoadForenamesByCountry / LoadSurnamesByCountry
each memoized after first parse; MockCuratedDataService (--mocked) returns a fixed in-memory dataset instead, skipping these JSON files"] LoadUserData --> UserLoop{For each EnrichedCity} UserLoop --> SampleName["SampleName(forenames_by_country, surnames_by_country, location.iso3166_1)"] @@ -42,7 +42,7 @@ flowchart TD UserGenDone -- yes --> UserPostProc["GetRandomCoordsWithinRange (5km jitter), BuildEmail, GenerateDateOfBirth
email and date_of_birth are generated programmatically, never LLM-authored; generation + coordinate jitter run inside one try/catch per record"] UserGenDone -- no --> UserGenFail["Log warning: generation failed, skipping"] --> UserLoop - UserPostProc --> UserExport["SqliteExportService.ProcessRecord(UserRecord)
reuse cached location_id or insert Location and cache its id; insert User (FK location_id)"] + UserPostProc --> UserExport["SqliteExportService.ProcessRecord(UserRecord)
reuse cached city_id or insert City and cache its id; insert User + UserAddress (FK city_id)"] UserExport --> UserExportFail{Exception during insert?} UserExportFail -- yes --> UserExportWarn["Log warning: SQLite export failed"] --> UserLoop UserExportFail -- no --> UserLoop @@ -64,7 +64,10 @@ flowchart TD BrewGenDone -- yes --> BrewPostProc["GetRandomCoordsWithinRange (5km jitter)
generation + jitter run inside one try/catch per record"] BrewGenDone -- no --> BrewGenFail["Log warning: generation failed, skipping"] --> BrewLoop - BrewPostProc --> BrewExport["SqliteExportService.ProcessRecord(GeneratedBrewery)
reuse cached location_id or insert Location and cache its id; insert Brewery (FK location_id)"] + BrewPostProc --> BrewGeocode["IAddressService.ReverseGeocode(longitude, latitude)
NominatimAddressService hits the live Nominatim API (1 req/sec) and returns nullopt on failure; MockAddressService (--mocked) returns a fixed placeholder address instead"] + BrewGeocode --> BrewGeocodeFailed{Geocode resolved?} + BrewGeocodeFailed -- no --> BrewPlaceholder["Log warning: could not reverse geocode; use placeholder address_line1, empty postal_code"] --> BrewExport + BrewGeocodeFailed -- yes --> BrewExport["SqliteExportService.ProcessRecord(BreweryRecord)
reuse cached city_id or insert City and cache its id; insert Brewery + BreweryAddress (FK city_id)"] BrewExport --> BrewExportFail{Exception during insert?} BrewExportFail -- yes --> BrewExportWarn["Log warning: SQLite export failed (data loss prevented per-record; pipeline continues)"] --> BrewLoop BrewExportFail -- no --> BrewLoop diff --git a/docs/pipeline/diagrams/current/class.mmd b/docs/pipeline/diagrams/current/class.mmd index f970cb72..cee4ed4d 100644 --- a/docs/pipeline/diagrams/current/class.mmd +++ b/docs/pipeline/diagrams/current/class.mmd @@ -5,6 +5,7 @@ classDiagram -unique_ptr~DataGenerator~ generator_ -unique_ptr~IExportService~ exporter_ -unique_ptr~ICuratedDataService~ curated_data_service_ + -unique_ptr~IAddressService~ address_service_ -ApplicationOptions application_options_ -vector~BreweryRecord~ generated_breweries_ -vector~UserRecord~ generated_users_ @@ -14,7 +15,7 @@ classDiagram -GenerateUsers(cities) void -LogResults() void } - note for BiergartenPipelineOrchestrator "GenerateBreweries/GenerateUsers jitter each address via GetRandomCoordsWithinRange(), uniformly within 5km of the city centre" + note for BiergartenPipelineOrchestrator "GenerateBreweries/GenerateUsers jitter each address via GetRandomCoordsWithinRange(), uniformly within 5km of the city centre. GenerateBreweries additionally reverse-geocodes the jittered coordinates via address_service_." class LogLevel { <> @@ -157,32 +158,50 @@ classDiagram class ICuratedDataService { <> - +LoadLocations(filepath) vector~Location~ - +LoadPersonas(filepath) vector~UserPersona~ - +LoadForenamesByCountry(filepath) unordered_map~string, forename_list~ - +LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~ + +LoadLocations() vector~Location~ + +LoadPersonas() vector~UserPersona~ + +LoadForenamesByCountry() unordered_map~string, forename_list~ + +LoadSurnamesByCountry() unordered_map~string, surname_list~ } - class JsonLoader { + class CuratedJsonDataService { + -CuratedDataFilePaths filepaths_ -cache cache_ - +LoadLocations(filepath) vector~Location~ - +LoadPersonas(filepath) vector~UserPersona~ - +LoadForenamesByCountry(filepath) unordered_map~string, forename_list~ - +LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~ + +LoadLocations() vector~Location~ + +LoadPersonas() vector~UserPersona~ + +LoadForenamesByCountry() unordered_map~string, forename_list~ + +LoadSurnamesByCountry() unordered_map~string, surname_list~ } - note for JsonLoader "Each Load* method memoizes its result in cache_ after its first call" + note for CuratedJsonDataService "Takes a CuratedDataFilePaths DTO (locations/personas/forenames/surnames paths) once at construction, not per Load* call. Each Load* method memoizes its result in cache_ after its first call." class MockCuratedDataService { -vector~Location~ locations_ -vector~UserPersona~ personas_ -unordered_map~string, forename_list~ forenames_by_country_ -unordered_map~string, surname_list~ surnames_by_country_ - +LoadLocations(filepath) vector~Location~ - +LoadPersonas(filepath) vector~UserPersona~ - +LoadForenamesByCountry(filepath) unordered_map~string, forename_list~ - +LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~ + +LoadLocations() vector~Location~ + +LoadPersonas() vector~UserPersona~ + +LoadForenamesByCountry() unordered_map~string, forename_list~ + +LoadSurnamesByCountry() unordered_map~string, surname_list~ } - note for MockCuratedDataService "Fixed 4-location/3-persona dataset (US, DE, FR, BE) for --mocked runs; filepath arguments are ignored" + note for MockCuratedDataService "Fixed 4-location/3-persona dataset (US, DE, FR, BE) for --mocked runs" + + class IAddressService { + <> + +ReverseGeocode(longitude, latitude) optional~AddressLookupResult~ + } + + class NominatimAddressService { + -unique_ptr~WebClient~ client_ + -shared_ptr~ILogger~ logger_ + +ReverseGeocode(longitude, latitude) optional~AddressLookupResult~ + } + note for NominatimAddressService "Queries the public Nominatim API; sleeps 1s per request per its usage policy; returns nullopt on failure or when no usable street address is found" + + class MockAddressService { + +ReverseGeocode(longitude, latitude) optional~AddressLookupResult~ + } + note for MockAddressService "Returns a fixed placeholder address ('123 Mock Street', '00000') for --mocked runs, no network call" class IExportService { <> @@ -197,17 +216,19 @@ classDiagram -string run_timestamp_utc_ -path database_path_ -SqliteDatabaseHandle db_handle_ - -SqliteStatementHandle insert_location_stmt_ + -SqliteStatementHandle insert_city_stmt_ -SqliteStatementHandle insert_brewery_stmt_ + -SqliteStatementHandle insert_brewery_address_stmt_ -SqliteStatementHandle insert_user_stmt_ + -SqliteStatementHandle insert_user_address_stmt_ -bool transaction_open_ - -unordered_map~string, sqlite3_int64~ location_cache_ + -unordered_map~string, sqlite3_int64~ city_cache_ +Initialize() void +ProcessRecord(brewery) uint64_t +ProcessRecord(user) uint64_t +Finalize() void -InitializeSchema() void - -ResolveLocationId(location) sqlite3_int64 + -ResolveCityId(city) sqlite3_int64 } class IDateTimeProvider { @@ -224,6 +245,7 @@ classDiagram BiergartenPipelineOrchestrator *-- DataGenerator : owns BiergartenPipelineOrchestrator *-- IExportService : owns BiergartenPipelineOrchestrator *-- ICuratedDataService : owns + BiergartenPipelineOrchestrator *-- IAddressService : owns LogEntry *-- LogLevel LogEntry *-- PipelinePhase @@ -252,9 +274,12 @@ classDiagram IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements IPromptDirectory <|.. PromptDirectory : implements - ICuratedDataService <|.. JsonLoader : implements + ICuratedDataService <|.. CuratedJsonDataService : implements ICuratedDataService <|.. MockCuratedDataService : implements + IAddressService <|.. NominatimAddressService : implements + IAddressService <|.. MockAddressService : implements + IExportService <|.. SqliteExportService : implements SqliteExportService *-- IDateTimeProvider : owns IDateTimeProvider <|.. SystemDateTimeProvider : implements diff --git a/docs/pipeline/diagrams/planned/class.mmd b/docs/pipeline/diagrams/planned/class.mmd index c0fcc5c1..c5fccea5 100644 --- a/docs/pipeline/diagrams/planned/class.mmd +++ b/docs/pipeline/diagrams/planned/class.mmd @@ -161,10 +161,19 @@ classDiagram +int seed } + class GeneratorMode { + <> + kMock + kLlama + kOpenAI + } + class GeneratorOptions { +path model_path - +bool use_mocked + +GeneratorMode mode +optional~SamplingOptions~ sampling + +string openai_api_key + +string openai_model } class PipelineOptions { @@ -177,8 +186,10 @@ classDiagram +PipelineOptions pipeline } } + note for GeneratorOptions "Already implemented (models.h), not planned: the three-way GeneratorMode split (kMock/kLlama/kOpenAI) and the openai_api_key/openai_model fields ship today, ahead of the rest of this diagram." ApplicationOptions *-- GeneratorOptions ApplicationOptions *-- PipelineOptions + GeneratorOptions *-- GeneratorMode GeneratorOptions o-- SamplingOptions namespace DomainPolicy {