Pipeline: rename Location to City, wire postal code into breweries

This commit is contained in:
Aaron Po
2026-07-13 02:26:44 -04:00
parent fbcf438381
commit d52dba904c
48 changed files with 2261 additions and 1227 deletions

View File

@@ -69,14 +69,14 @@ end note
:QueryCitiesWithCountries();
|#E2EBDC|JsonLoader|
:JsonLoader::LoadLocations("locations.json");
:JsonLoader::LoadCities("cities.json");
:std::ranges::sample(all_locations, --location-count);
note right
--location-count defaults to 10.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
while (For each sampled Location?) is (Remaining cities)
while (For each sampled City?) is (Remaining cities)
|#DCE8D8|IEnrichmentService|
:GetLocationContext(loc);
note right
@@ -89,7 +89,7 @@ while (For each sampled Location?) is (Remaining cities)
if (Lookup failed?) then (yes)
:Log warning, skip city;
else (no)
:Store EnrichedCity{Location, region_context};
:Store EnrichedCity{City, region_context};
endif
endwhile (Done)
@@ -146,12 +146,12 @@ while (For each EnrichedCity?) is (Remaining cities)
|#E0EAE0|SqliteExportService|
:ProcessRecord(UserRecord);
if (Location in cache?) then (yes)
:Reuse location_id;
if (City in cache?) then (yes)
:Reuse city_id;
else (no)
:Insert Location & Cache ID;
:Insert City & Cache ID;
endif
:Insert User (FK: location_id);
:Insert User (FK: city_id);
if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator|
@@ -197,12 +197,12 @@ while (For each EnrichedCity?) is (Remaining cities)
if (Generation successful?) then (yes)
|#E0EAE0|SqliteExportService|
:ProcessRecord(GeneratedBrewery);
if (Location in cache?) then (yes)
:Reuse location_id;
if (City in cache?) then (yes)
:Reuse city_id;
else (no)
:Insert Location & Cache ID;
:Insert City & Cache ID;
endif
:Insert Brewery (FK: location_id);
:Insert Brewery (FK: city_id);
if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator|

View File

@@ -31,11 +31,12 @@ class BiergartenPipelineOrchestrator {
- generator_ : std::unique_ptr<DataGenerator>
- exporter_ : std::unique_ptr<IExportService>
- curated_data_service_ : std::unique_ptr<ICuratedDataService>
- postal_code_service_ : std::unique_ptr<IPostalCodeService>
- application_options_ : ApplicationOptions
- generated_breweries_ : std::vector<BreweryRecord>
- generated_users_ : std::vector<UserRecord>
+ Run() : bool
- QueryCitiesWithCountries() : std::vector<Location>
- QueryCitiesWithCountries() : std::vector<City>
- GenerateBreweries(cities : std::span<const EnrichedCity>) : void
- GenerateUsers(cities : std::span<const EnrichedCity>) : void
- LogResults() : void
@@ -90,17 +91,17 @@ class LogDispatcher {
}
interface IEnrichmentService <<interface>> {
+ GetLocationContext(loc : const Location&) : std::string
+ GetLocationContext(loc : const City&) : std::string
}
class MockEnrichmentService {
+ GetLocationContext(loc : const Location&) : std::string
+ GetLocationContext(loc : const City&) : std::string
}
class WikipediaEnrichmentService {
- client_ : std::unique_ptr<WebClient>
- extract_cache_ : std::unordered_map<std::string, std::string>
+ GetLocationContext(loc : const Location&) : std::string
+ GetLocationContext(loc : const City&) : std::string
- FetchExtract(query : std::string_view) : std::string
}
@@ -115,15 +116,15 @@ class HttpWebClient {
}
interface DataGenerator <<interface>> {
+ GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
+ GenerateBrewery(location : const City&, region_context : const std::string&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
}
class MockGenerator {
+ GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
+ GenerateBrewery(location : const City&, region_context : const std::string&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
- DeterministicHash(location : const Location&) : size_t
- DeterministicHash(location : const Location&, persona : const UserPersona&, name : const Name&) : size_t
- DeterministicHash(location : const City&) : size_t
- DeterministicHash(location : const City&, persona : const UserPersona&, name : const Name&) : size_t
}
class LlamaGenerator {
@@ -158,7 +159,7 @@ class PromptDirectory {
}
interface ICuratedDataService <<interface>> {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
@@ -166,7 +167,7 @@ interface ICuratedDataService <<interface>> {
class JsonLoader {
- cache_ : cache
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
@@ -180,11 +181,11 @@ note right of JsonLoader
end note
class MockCuratedDataService {
- locations_ : std::vector<Location>
- locations_ : std::vector<City>
- personas_ : std::vector<UserPersona>
- forenames_by_country_ : std::unordered_map<std::string, forename_list>
- surnames_by_country_ : std::unordered_map<std::string, surname_list>
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
@@ -196,6 +197,27 @@ note right of MockCuratedDataService
filepath arguments are ignored.
end note
interface IPostalCodeService <<interface>> {
+ GeneratePostalCode(location : const City&) : std::string
}
class MockPostalCodeService {
+ GeneratePostalCode(location : const City&) : std::string
}
note right of MockPostalCodeService
Ignores location.postal_regex and returns
location.postal_code_examples.front(). Bound
unconditionally in main.cc's DI graph (no
--mocked branch yet -- no real implementation
exists to switch to) and called from
GenerateBreweries() per brewery, stored on
BreweryRecord::address. A real implementation
needs a xeger-style generator to turn
postal_regex into a synthesized matching
string. See ROADMAP.md §9.
end note
interface IExportService <<interface>> {
+ Initialize() : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
@@ -208,17 +230,17 @@ class SqliteExportService {
- run_timestamp_utc_ : std::string
- database_path_ : std::filesystem::path
- db_handle_ : SqliteDatabaseHandle
- insert_location_stmt_ : SqliteStatementHandle
- insert_city_stmt_ : SqliteStatementHandle
- insert_brewery_stmt_ : SqliteStatementHandle
- insert_user_stmt_ : SqliteStatementHandle
- transaction_open_ : bool
- location_cache_ : std::unordered_map<std::string, sqlite3_int64>
- city_cache_ : std::unordered_map<std::string, sqlite3_int64>
+ Initialize() : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+ ProcessRecord(user : const UserRecord&) : uint64_t
+ Finalize() : void
- InitializeSchema() : void
- ResolveLocationId(location : const Location&) : sqlite3_int64
- ResolveCityId(location : const City&) : sqlite3_int64
}
interface IDateTimeProvider <<interface>> {
@@ -235,6 +257,7 @@ BiergartenPipelineOrchestrator *-- IEnrichmentService : owns
BiergartenPipelineOrchestrator *-- DataGenerator : owns
BiergartenPipelineOrchestrator *-- IExportService : owns
BiergartenPipelineOrchestrator *-- ICuratedDataService : owns
BiergartenPipelineOrchestrator *-- IPostalCodeService : owns
LogEntry *-- LogLevel
LogEntry *-- PipelinePhase
@@ -262,6 +285,8 @@ IPromptDirectory <|.. PromptDirectory : implements
ICuratedDataService <|.. JsonLoader : implements
ICuratedDataService <|.. MockCuratedDataService : implements
IPostalCodeService <|.. MockPostalCodeService : implements
IExportService <|.. SqliteExportService : implements
SqliteExportService *-- IDateTimeProvider : owns
IDateTimeProvider <|.. SystemDateTimeProvider : implements

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 86 KiB

After

Width:  |  Height:  |  Size: 86 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 87 KiB

After

Width:  |  Height:  |  Size: 94 KiB

View File

@@ -39,7 +39,7 @@ fork
:JsonLoader::LoadBeerStyles("beer-styles.json");
:EnrichmentService::PreWarmBeerStyleCache(beer_styles);
fork again
:JsonLoader::LoadLocations("locations.json");
:JsonLoader::LoadCities("cities.json");
:EnrichmentService::PreWarmLocationCache(sampled_locations);
end fork
fork
@@ -75,7 +75,7 @@ fork
fork again
|LLM Worker|
while (loc_ch has items?) is (yes)
:Receive Location;
:Receive City;
:GetLocationContextFromCache(location);
note right
@@ -146,13 +146,13 @@ end fork
fork
|Orchestrator|
:Loop: Sample User from user_pool_
and pair with Location;
:Send BreweryTask(Location, User) -> loc_ch;
and pair with City;
:Send BreweryTask(City, User) -> loc_ch;
:Close loc_ch;
fork again
|LLM Worker|
while (loc_ch has items?) is (yes)
:Receive BreweryTask(Location, User);
:Receive BreweryTask(City, User);
:GetLocationContextFromCache(task.location);
note right

View File

@@ -12,17 +12,40 @@ title Biergarten Data Pipeline — Class Diagram
package "Domain: Models" {
class Location {
class City {
+ city : std::string
+ state_province : std::string
+ iso3166_2 : std::string
+ country : std::string
+ iso3166_1 : std::string
+ postal_regex : std::vector<std::string>
+ postal_code_examples : std::vector<std::string>
+ local_languages : std::vector<std::string>
+ latitude : double
+ longitude : double
}
note right of City
postal_regex / postal_code_examples are
sourced from each cities.json entry's
postal_code.city_regex / postal_code.examples
(implemented today, ahead of the rest of this
diagram -- see ROADMAP.md §1/§2). Renamed from
Location to City to match the web backend's
City/CityId vocabulary -- see ROADMAP.md §1.
end note
class Address {
+ postal_code : std::string
}
note right of Address
Mirrors the web backend's BreweryPostLocation.
Only postal_code is populated today --
address_line1/address_line2 have no fixture
data or generator yet. See ROADMAP.md §9.
end note
class LocationContext {
+ text : std::string
+ completeness : Completeness
@@ -36,7 +59,7 @@ package "Domain: Models" {
}
class EnrichedCity {
+ location : Location
+ location : City
+ context : LocationContext
}
@@ -130,7 +153,8 @@ package "Domain: Models" {
class GeneratedBrewery {
+ brewery_id : uint64_t
+ location : Location
+ location : City
+ address : Address
+ brewery : BreweryResult
+ context_completeness : LocationContext::Completeness
+ metadata : GenerationMetadata
@@ -139,7 +163,7 @@ package "Domain: Models" {
class GeneratedBeer {
+ beer_id : uint64_t
+ brewery_id : uint64_t
+ location : Location
+ location : City
+ style : BeerStyle
+ beer : BeerResult
+ metadata : GenerationMetadata
@@ -147,7 +171,7 @@ package "Domain: Models" {
class GeneratedUser {
+ user_id : uint64_t
+ location : Location
+ location : City
+ user : UserResult
+ email : std::string
+ date_of_birth : std::string
@@ -227,27 +251,27 @@ package "Domain: Application Configuration" {
package "Domain: Policy" {
interface ContextStrategy <<interface>> {
+ QueriesFor(loc : const Location&) : std::vector<std::string>
+ QueriesFor(loc : const City&) : std::vector<std::string>
+ MaxContextChars() : size_t
}
class BreweryContextStrategy {
+ QueriesFor(loc : const Location&) : std::vector<std::string>
+ QueriesFor(loc : const City&) : std::vector<std::string>
+ MaxContextChars() : size_t
}
class BeerContextStrategy {
+ QueriesFor(loc : const Location&) : std::vector<std::string>
+ QueriesFor(loc : const City&) : std::vector<std::string>
+ MaxContextChars() : size_t
}
interface SamplingStrategy <<interface>> {
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
+ Sample(locations : const std::vector<City>&) : std::vector<City>
}
class UniformSamplingStrategy {
- sample_size_ : size_t
+ Sample(locations : const std::vector<Location>&) : std::vector<Location>
+ Sample(locations : const std::vector<City>&) : std::vector<City>
}
interface BeerSelectionStrategy <<interface>> {
@@ -369,7 +393,7 @@ package "Infrastructure: Pipeline Channel" {
package "Infrastructure: Data Preloading" {
interface ICuratedDataService <<interface>> {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
@@ -377,7 +401,7 @@ package "Infrastructure: Data Preloading" {
}
class JsonLoader {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
@@ -391,7 +415,7 @@ package "Infrastructure: Data Preloading" {
end note
class MockCuratedDataService {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
@@ -410,13 +434,13 @@ package "Infrastructure: Data Preloading" {
package "Infrastructure: Enrichment" {
interface EnrichmentService <<interface>> {
+ GetLocationContext(loc : const Location&,\n strategy : const ContextStrategy&) : LocationContext
+ GetLocationContext(loc : const City&,\n strategy : const ContextStrategy&) : LocationContext
}
class WikipediaService {
- client_ : std::unique_ptr<WebClient>
- extract_cache_ : std::unordered_map<std::string, std::string>
+ GetLocationContext(loc : const Location&,\n strategy : const ContextStrategy&) : LocationContext
+ GetLocationContext(loc : const City&,\n strategy : const ContextStrategy&) : LocationContext
- FetchExtract(query : std::string_view) : std::string
}
@@ -432,6 +456,39 @@ package "Infrastructure: Enrichment" {
}
package "Infrastructure: Postal Code Generation" {
interface IPostalCodeService <<interface>> {
+ GeneratePostalCode(location : const City&) : std::string
}
class MockPostalCodeService {
+ GeneratePostalCode(location : const City&) : std::string
}
class XegerPostalCodeService {
+ GeneratePostalCode(location : const City&) : std::string
}
note right of MockPostalCodeService
Implemented today: ignores postal_regex and
returns postal_code_examples.front(). Not yet
bound in main.cc's DI graph or used by
BiergartenPipelineOrchestrator.
end note
note right of XegerPostalCodeService
Not implemented yet. Picks one of
location.postal_regex and generates a random
string matching it (a "xeger" -- the inverse of
regex matching) instead of always replaying a
fixed example. See ROADMAP.md §9.
end note
IPostalCodeService <|.. MockPostalCodeService
IPostalCodeService <|.. XegerPostalCodeService
}
package "Infrastructure: Prompting" {
interface IPromptDirectory <<interface>> {
@@ -451,8 +508,8 @@ package "Infrastructure: Prompting" {
package "Infrastructure: Data Generation" {
interface DataGenerator <<interface>> {
+ GenerateBrewery(location : const Location&,\n context : const LocationContext&) : BreweryResult
+ GenerateBeer(brewery_id : uint64_t,\n location : const Location&,\n context : const LocationContext&,\n style : const BeerStyle&) : BeerResult
+ GenerateBrewery(location : const City&,\n context : const LocationContext&) : BreweryResult
+ GenerateBeer(brewery_id : uint64_t,\n location : const City&,\n context : const LocationContext&,\n style : const BeerStyle&) : BeerResult
+ GenerateUser(city : const EnrichedCity&,\n persona : const UserPersona&,\n name : const Name&) : UserResult
+ GenerateCheckin(user : const GeneratedUser&,\n brewery : const GeneratedBrewery&,\n timestamp : const std::string&) : CheckinResult
+ GenerateRating(user : const GeneratedUser&,\n beer : const GeneratedBeer&,\n checkin_id : uint64_t) : RatingResult
@@ -464,7 +521,7 @@ package "Infrastructure: Data Generation" {
+ GenerateUser(...) : UserResult
+ GenerateCheckin(...) : CheckinResult
+ GenerateRating(...) : RatingResult
- DeterministicHash(location : const Location&) : size_t
- DeterministicHash(location : const City&) : size_t
}
class LlamaGenerator {
@@ -566,8 +623,8 @@ class BiergartenPipelineOrchestrator {
- follow_pool_ : std::vector<GeneratedFollow>
--
+ Run() : bool
- RunUserPhase(locations : const std::vector<Location>&) : void
- RunBreweryAndBeerPhase(locations : const std::vector<Location>&) : void
- RunUserPhase(locations : const std::vector<City>&) : void
- RunBreweryAndBeerPhase(locations : const std::vector<City>&) : void
- RunCheckinPhase() : void
- RunRatingPhase() : void
- RunFollowPhase() : void
@@ -625,16 +682,17 @@ PipelineLogger o-- BoundedChannel : logs to
LogWorker o-- BoundedChannel : drains from
' --- Domain Containment ---
EnrichedCity *-- Location
EnrichedCity *-- City
EnrichedCity *-- LocationContext
GeneratedBrewery *-- Location
GeneratedBrewery *-- City
GeneratedBrewery *-- Address
GeneratedBrewery *-- BreweryResult
GeneratedBrewery *-- GenerationMetadata
GeneratedBeer *-- Location
GeneratedBeer *-- City
GeneratedBeer *-- BeerStyle
GeneratedBeer *-- BeerResult
GeneratedBeer *-- GenerationMetadata
GeneratedUser *-- Location
GeneratedUser *-- City
GeneratedUser *-- UserResult
GeneratedUser *-- GenerationMetadata
GeneratedCheckin *-- CheckinResult

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 142 KiB

After

Width:  |  Height:  |  Size: 142 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 240 KiB

After

Width:  |  Height:  |  Size: 257 KiB