Rename JsonLoader to CuratedJsonDataService and add the address-lookup classes to the pipeline diagrams

This commit is contained in:
Aaron Po
2026-08-31 22:03:24 -04:00
parent 399ae86c1f
commit a6d6724251
3 changed files with 64 additions and 25 deletions

View File

@@ -10,7 +10,7 @@ flowchart TD
Run --> SqliteInit["SqliteExportService.Initialize()<br/>builds biergarten_seed_&lt;UTC datetime&gt;.sqlite (numeric suffix if it exists), opens DB connection, executes schema DDL, begins transaction"] Run --> SqliteInit["SqliteExportService.Initialize()<br/>builds biergarten_seed_&lt;UTC datetime&gt;.sqlite (numeric suffix if it exists), opens DB connection, executes schema DDL, begins transaction"]
SqliteInit --> QueryLoc[Orchestrator: QueryLocations] SqliteInit --> QueryLoc[Orchestrator: QueryLocations]
QueryLoc --> LoadLoc["JsonLoader.LoadLocations(locations.json)"] QueryLoc --> LoadLoc["CuratedJsonDataService.LoadLocations()"]
LoadLoc --> SampleLoc["std::ranges::sample(all_locations, --location-count) — defaults to 10"] LoadLoc --> SampleLoc["std::ranges::sample(all_locations, --location-count) — defaults to 10"]
SampleLoc --> EnrichLoop{For each sampled Location} SampleLoc --> EnrichLoop{For each sampled Location}
@@ -20,7 +20,7 @@ flowchart TD
LookupFailed -- no --> StoreEnriched["Store EnrichedCity{Location, region_context}"] --> EnrichLoop LookupFailed -- no --> StoreEnriched["Store EnrichedCity{Location, region_context}"] --> EnrichLoop
EnrichLoop -- done --> GenUsers[Orchestrator: GenerateUsers enriched_cities] EnrichLoop -- done --> GenUsers[Orchestrator: GenerateUsers enriched_cities]
GenUsers --> LoadUserData["JsonLoader: LoadPersonas / LoadForenamesByCountry / LoadSurnamesByCountry<br/>each memoized after first parse; MockCuratedDataService (--mocked) returns a fixed in-memory dataset instead, skipping these JSON files"] GenUsers --> LoadUserData["CuratedJsonDataService: LoadPersonas / LoadForenamesByCountry / LoadSurnamesByCountry<br/>each memoized after first parse; MockCuratedDataService (--mocked) returns a fixed in-memory dataset instead, skipping these JSON files"]
LoadUserData --> UserLoop{For each EnrichedCity} LoadUserData --> UserLoop{For each EnrichedCity}
UserLoop --> SampleName["SampleName(forenames_by_country, surnames_by_country, location.iso3166_1)"] UserLoop --> SampleName["SampleName(forenames_by_country, surnames_by_country, location.iso3166_1)"]
@@ -42,7 +42,7 @@ flowchart TD
UserGenDone -- yes --> UserPostProc["GetRandomCoordsWithinRange (5km jitter), BuildEmail, GenerateDateOfBirth<br/>email and date_of_birth are generated programmatically, never LLM-authored; generation + coordinate jitter run inside one try/catch per record"] UserGenDone -- yes --> UserPostProc["GetRandomCoordsWithinRange (5km jitter), BuildEmail, GenerateDateOfBirth<br/>email and date_of_birth are generated programmatically, never LLM-authored; generation + coordinate jitter run inside one try/catch per record"]
UserGenDone -- no --> UserGenFail["Log warning: generation failed, skipping"] --> UserLoop UserGenDone -- no --> UserGenFail["Log warning: generation failed, skipping"] --> UserLoop
UserPostProc --> UserExport["SqliteExportService.ProcessRecord(UserRecord)<br/>reuse cached location_id or insert Location and cache its id; insert User (FK location_id)"] UserPostProc --> UserExport["SqliteExportService.ProcessRecord(UserRecord)<br/>reuse cached city_id or insert City and cache its id; insert User + UserAddress (FK city_id)"]
UserExport --> UserExportFail{Exception during insert?} UserExport --> UserExportFail{Exception during insert?}
UserExportFail -- yes --> UserExportWarn["Log warning: SQLite export failed"] --> UserLoop UserExportFail -- yes --> UserExportWarn["Log warning: SQLite export failed"] --> UserLoop
UserExportFail -- no --> UserLoop UserExportFail -- no --> UserLoop
@@ -64,7 +64,10 @@ flowchart TD
BrewGenDone -- yes --> BrewPostProc["GetRandomCoordsWithinRange (5km jitter)<br/>generation + jitter run inside one try/catch per record"] BrewGenDone -- yes --> BrewPostProc["GetRandomCoordsWithinRange (5km jitter)<br/>generation + jitter run inside one try/catch per record"]
BrewGenDone -- no --> BrewGenFail["Log warning: generation failed, skipping"] --> BrewLoop BrewGenDone -- no --> BrewGenFail["Log warning: generation failed, skipping"] --> BrewLoop
BrewPostProc --> BrewExport["SqliteExportService.ProcessRecord(GeneratedBrewery)<br/>reuse cached location_id or insert Location and cache its id; insert Brewery (FK location_id)"] BrewPostProc --> BrewGeocode["IAddressService.ReverseGeocode(longitude, latitude)<br/>NominatimAddressService hits the live Nominatim API (1 req/sec) and returns nullopt on failure; MockAddressService (--mocked) returns a fixed placeholder address instead"]
BrewGeocode --> BrewGeocodeFailed{Geocode resolved?}
BrewGeocodeFailed -- no --> BrewPlaceholder["Log warning: could not reverse geocode; use placeholder address_line1, empty postal_code"] --> BrewExport
BrewGeocodeFailed -- yes --> BrewExport["SqliteExportService.ProcessRecord(BreweryRecord)<br/>reuse cached city_id or insert City and cache its id; insert Brewery + BreweryAddress (FK city_id)"]
BrewExport --> BrewExportFail{Exception during insert?} BrewExport --> BrewExportFail{Exception during insert?}
BrewExportFail -- yes --> BrewExportWarn["Log warning: SQLite export failed (data loss prevented per-record; pipeline continues)"] --> BrewLoop BrewExportFail -- yes --> BrewExportWarn["Log warning: SQLite export failed (data loss prevented per-record; pipeline continues)"] --> BrewLoop
BrewExportFail -- no --> BrewLoop BrewExportFail -- no --> BrewLoop

View File

@@ -5,6 +5,7 @@ classDiagram
-unique_ptr~DataGenerator~ generator_ -unique_ptr~DataGenerator~ generator_
-unique_ptr~IExportService~ exporter_ -unique_ptr~IExportService~ exporter_
-unique_ptr~ICuratedDataService~ curated_data_service_ -unique_ptr~ICuratedDataService~ curated_data_service_
-unique_ptr~IAddressService~ address_service_
-ApplicationOptions application_options_ -ApplicationOptions application_options_
-vector~BreweryRecord~ generated_breweries_ -vector~BreweryRecord~ generated_breweries_
-vector~UserRecord~ generated_users_ -vector~UserRecord~ generated_users_
@@ -14,7 +15,7 @@ classDiagram
-GenerateUsers(cities) void -GenerateUsers(cities) void
-LogResults() void -LogResults() void
} }
note for BiergartenPipelineOrchestrator "GenerateBreweries/GenerateUsers jitter each address via GetRandomCoordsWithinRange(), uniformly within 5km of the city centre" note for BiergartenPipelineOrchestrator "GenerateBreweries/GenerateUsers jitter each address via GetRandomCoordsWithinRange(), uniformly within 5km of the city centre. GenerateBreweries additionally reverse-geocodes the jittered coordinates via address_service_."
class LogLevel { class LogLevel {
<<enumeration>> <<enumeration>>
@@ -157,32 +158,50 @@ classDiagram
class ICuratedDataService { class ICuratedDataService {
<<interface>> <<interface>>
+LoadLocations(filepath) vector~Location~ +LoadLocations() vector~Location~
+LoadPersonas(filepath) vector~UserPersona~ +LoadPersonas() vector~UserPersona~
+LoadForenamesByCountry(filepath) unordered_map~string, forename_list~ +LoadForenamesByCountry() unordered_map~string, forename_list~
+LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~ +LoadSurnamesByCountry() unordered_map~string, surname_list~
} }
class JsonLoader { class CuratedJsonDataService {
-CuratedDataFilePaths filepaths_
-cache cache_ -cache cache_
+LoadLocations(filepath) vector~Location~ +LoadLocations() vector~Location~
+LoadPersonas(filepath) vector~UserPersona~ +LoadPersonas() vector~UserPersona~
+LoadForenamesByCountry(filepath) unordered_map~string, forename_list~ +LoadForenamesByCountry() unordered_map~string, forename_list~
+LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~ +LoadSurnamesByCountry() unordered_map~string, surname_list~
} }
note for JsonLoader "Each Load* method memoizes its result in cache_ after its first call" note for CuratedJsonDataService "Takes a CuratedDataFilePaths DTO (locations/personas/forenames/surnames paths) once at construction, not per Load* call. Each Load* method memoizes its result in cache_ after its first call."
class MockCuratedDataService { class MockCuratedDataService {
-vector~Location~ locations_ -vector~Location~ locations_
-vector~UserPersona~ personas_ -vector~UserPersona~ personas_
-unordered_map~string, forename_list~ forenames_by_country_ -unordered_map~string, forename_list~ forenames_by_country_
-unordered_map~string, surname_list~ surnames_by_country_ -unordered_map~string, surname_list~ surnames_by_country_
+LoadLocations(filepath) vector~Location~ +LoadLocations() vector~Location~
+LoadPersonas(filepath) vector~UserPersona~ +LoadPersonas() vector~UserPersona~
+LoadForenamesByCountry(filepath) unordered_map~string, forename_list~ +LoadForenamesByCountry() unordered_map~string, forename_list~
+LoadSurnamesByCountry(filepath) unordered_map~string, surname_list~ +LoadSurnamesByCountry() unordered_map~string, surname_list~
} }
note for MockCuratedDataService "Fixed 4-location/3-persona dataset (US, DE, FR, BE) for --mocked runs; filepath arguments are ignored" note for MockCuratedDataService "Fixed 4-location/3-persona dataset (US, DE, FR, BE) for --mocked runs"
class IAddressService {
<<interface>>
+ReverseGeocode(longitude, latitude) optional~AddressLookupResult~
}
class NominatimAddressService {
-unique_ptr~WebClient~ client_
-shared_ptr~ILogger~ logger_
+ReverseGeocode(longitude, latitude) optional~AddressLookupResult~
}
note for NominatimAddressService "Queries the public Nominatim API; sleeps 1s per request per its usage policy; returns nullopt on failure or when no usable street address is found"
class MockAddressService {
+ReverseGeocode(longitude, latitude) optional~AddressLookupResult~
}
note for MockAddressService "Returns a fixed placeholder address ('123 Mock Street', '00000') for --mocked runs, no network call"
class IExportService { class IExportService {
<<interface>> <<interface>>
@@ -197,17 +216,19 @@ classDiagram
-string run_timestamp_utc_ -string run_timestamp_utc_
-path database_path_ -path database_path_
-SqliteDatabaseHandle db_handle_ -SqliteDatabaseHandle db_handle_
-SqliteStatementHandle insert_location_stmt_ -SqliteStatementHandle insert_city_stmt_
-SqliteStatementHandle insert_brewery_stmt_ -SqliteStatementHandle insert_brewery_stmt_
-SqliteStatementHandle insert_brewery_address_stmt_
-SqliteStatementHandle insert_user_stmt_ -SqliteStatementHandle insert_user_stmt_
-SqliteStatementHandle insert_user_address_stmt_
-bool transaction_open_ -bool transaction_open_
-unordered_map~string, sqlite3_int64~ location_cache_ -unordered_map~string, sqlite3_int64~ city_cache_
+Initialize() void +Initialize() void
+ProcessRecord(brewery) uint64_t +ProcessRecord(brewery) uint64_t
+ProcessRecord(user) uint64_t +ProcessRecord(user) uint64_t
+Finalize() void +Finalize() void
-InitializeSchema() void -InitializeSchema() void
-ResolveLocationId(location) sqlite3_int64 -ResolveCityId(city) sqlite3_int64
} }
class IDateTimeProvider { class IDateTimeProvider {
@@ -224,6 +245,7 @@ classDiagram
BiergartenPipelineOrchestrator *-- DataGenerator : owns BiergartenPipelineOrchestrator *-- DataGenerator : owns
BiergartenPipelineOrchestrator *-- IExportService : owns BiergartenPipelineOrchestrator *-- IExportService : owns
BiergartenPipelineOrchestrator *-- ICuratedDataService : owns BiergartenPipelineOrchestrator *-- ICuratedDataService : owns
BiergartenPipelineOrchestrator *-- IAddressService : owns
LogEntry *-- LogLevel LogEntry *-- LogLevel
LogEntry *-- PipelinePhase LogEntry *-- PipelinePhase
@@ -252,9 +274,12 @@ classDiagram
IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements
IPromptDirectory <|.. PromptDirectory : implements IPromptDirectory <|.. PromptDirectory : implements
ICuratedDataService <|.. JsonLoader : implements ICuratedDataService <|.. CuratedJsonDataService : implements
ICuratedDataService <|.. MockCuratedDataService : implements ICuratedDataService <|.. MockCuratedDataService : implements
IAddressService <|.. NominatimAddressService : implements
IAddressService <|.. MockAddressService : implements
IExportService <|.. SqliteExportService : implements IExportService <|.. SqliteExportService : implements
SqliteExportService *-- IDateTimeProvider : owns SqliteExportService *-- IDateTimeProvider : owns
IDateTimeProvider <|.. SystemDateTimeProvider : implements IDateTimeProvider <|.. SystemDateTimeProvider : implements

View File

@@ -161,10 +161,19 @@ classDiagram
+int seed +int seed
} }
class GeneratorMode {
<<enumeration>>
kMock
kLlama
kOpenAI
}
class GeneratorOptions { class GeneratorOptions {
+path model_path +path model_path
+bool use_mocked +GeneratorMode mode
+optional~SamplingOptions~ sampling +optional~SamplingOptions~ sampling
+string openai_api_key
+string openai_model
} }
class PipelineOptions { class PipelineOptions {
@@ -177,8 +186,10 @@ classDiagram
+PipelineOptions pipeline +PipelineOptions pipeline
} }
} }
note for GeneratorOptions "Already implemented (models.h), not planned: the three-way GeneratorMode split (kMock/kLlama/kOpenAI) and the openai_api_key/openai_model fields ship today, ahead of the rest of this diagram."
ApplicationOptions *-- GeneratorOptions ApplicationOptions *-- GeneratorOptions
ApplicationOptions *-- PipelineOptions ApplicationOptions *-- PipelineOptions
GeneratorOptions *-- GeneratorMode
GeneratorOptions o-- SamplingOptions GeneratorOptions o-- SamplingOptions
namespace DomainPolicy { namespace DomainPolicy {