1 Commits

Author SHA1 Message Date
Aaron Po
0473f48fb7 fix mock-only builds 2026-07-03 02:27:00 -04:00
49 changed files with 1311 additions and 2431 deletions

View File

@@ -109,10 +109,10 @@ gender from the source data is preserved through to the sampled `Name`
persona/bio generation later. persona/bio generation later.
The full multinational dataset is kept as-is (106 countries for forenames, The full multinational dataset is kept as-is (106 countries for forenames,
75 for surnames) rather than trimmed to `cities.json`'s current country 75 for surnames) rather than trimmed to `locations.json`'s current country
list, so it doesn't need re-sourcing if more countries are added later. list, so it doesn't need re-sourcing if more countries are added later.
`SampleName()` (a free helper in `generate_users.cc`) returns no result for `SampleName()` (a free helper in `generate_users.cc`) returns no result for
a country present in neither file; of the countries in `cities.json`, a country present in neither file; of the countries in `locations.json`,
that's currently `KE`, `SE`, `SG`, `TH`, `VN`, and `ZA``GenerateUsers` that's currently `KE`, `SE`, `SG`, `TH`, `VN`, and `ZA``GenerateUsers`
skips cities in those countries the same way brewery generation skips skips cities in those countries the same way brewery generation skips
cities whose enrichment lookup fails. cities whose enrichment lookup fails.
@@ -281,7 +281,7 @@ Output sample:
a location's code is unlisted, skip `description_local` generation and fall a location's code is unlisted, skip `description_local` generation and fall
back to English. back to English.
- **Upstream sanitization:** strip known low-resource language codes from the - **Upstream sanitization:** strip known low-resource language codes from the
`cities.json` payload before generation. `locations.json` payload before generation.
- **Downstream flagging:** add a `description_local_confidence` column to the - **Downstream flagging:** add a `description_local_confidence` column to the
SQLite schema so downstream applications can filter or flag potentially SQLite schema so downstream applications can filter or flag potentially
hallucinated text by language tier. hallucinated text by language tier.

View File

@@ -86,7 +86,7 @@ curl -L \
### Run ### Run
Run from `build/` so the copied `cities.json` and `prompts/` are available. Run from `build/` so the copied `locations.json` and `prompts/` are available.
Each run writes a fresh dated SQLite file such as Each run writes a fresh dated SQLite file such as
`biergarten_seed_2026-04-19T15-30-45.123456Z.sqlite` into the working directory. `biergarten_seed_2026-04-19T15-30-45.123456Z.sqlite` into the working directory.
@@ -109,7 +109,7 @@ Each run writes a fresh dated SQLite file such as
| `--prompt-dir` | Directory containing prompt files (e.g. `BREWERY_GENERATION.md`). Required unless `--mocked` is set. | | `--prompt-dir` | Directory containing prompt files (e.g. `BREWERY_GENERATION.md`). Required unless `--mocked` is set. |
| `--output, -o` | Directory for generated SQLite artifacts. Default: `output`. | | `--output, -o` | Directory for generated SQLite artifacts. Default: `output`. |
| `--log-path` | Path for application logs. Default: `pipeline.log`. | | `--log-path` | Path for application logs. Default: `pipeline.log`. |
| `--location-count` | Number of cities to sample from `cities.json` per run. Default: `10`. | | `--location-count` | Number of cities to sample from `locations.json` per run. Default: `10`. |
| `--temperature` | Sampling temperature. Default: `1.0`. | | `--temperature` | Sampling temperature. Default: `1.0`. |
| `--top-p` | Nucleus sampling. Default: `0.95`. | | `--top-p` | Nucleus sampling. Default: `0.95`. |
| `--top-k` | Top-k sampling. Default: `64`. | | `--top-k` | Top-k sampling. Default: `64`. |
@@ -222,12 +222,12 @@ environment variables listed above to match your run.
| Stage | Implementation | | Stage | Implementation |
| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
| Load | `ICuratedDataService` (`CuratedJsonDataService`) reads `cities.json`, `personas.json`, `forenames-by-country.json`, and `surnames-by-country.json` (paths supplied via a `CuratedDataFilePaths` DTO at construction) into typed records, caching each after its first load. `--mocked` runs use `MockCuratedDataService`'s fixed in-memory dataset instead. | | Load | `ICuratedDataService` (`CuratedJsonDataService`) reads `locations.json`, `personas.json`, `forenames-by-country.json`, and `surnames-by-country.json` (paths supplied via a `CuratedDataFilePaths` DTO at construction) into typed records, caching each after its first load. `--mocked` runs use `MockCuratedDataService`'s fixed in-memory dataset instead. |
| Sample | `BiergartenPipelineOrchestrator::QueryCitiesWithCountries()` samples `--location-count` locations per run (default `10`). | | Sample | `BiergartenPipelineOrchestrator::QueryCitiesWithCountries()` samples `--location-count` locations per run (default `10`). |
| Enrich | `WikipediaEnrichmentService` fetches brewing and beer-related context. Keeps going when a lookup fails. `--mocked` runs use `MockEnrichmentService` instead and skip Wikipedia entirely. | | Enrich | `WikipediaEnrichmentService` fetches brewing and beer-related context. Keeps going when a lookup fails. `--mocked` runs use `MockEnrichmentService` instead and skip Wikipedia entirely. |
| Generate Users | `GenerateUsers()` samples a persona and a forename/surname pair per enriched city (skipping countries with no name data), then `MockGenerator` or `LlamaGenerator` produces a username, bio, and activity weight around the sampled name. | | Generate Users | `GenerateUsers()` samples a persona and a forename/surname pair per enriched city (skipping countries with no name data), then `MockGenerator` or `LlamaGenerator` produces a username, bio, and activity weight around the sampled name. |
| Generate Breweries | `MockGenerator` or `LlamaGenerator` produces brewery names and descriptions in English and the local language. | | Generate Breweries | `MockGenerator` or `LlamaGenerator` produces brewery names and descriptions in English and the local language. |
| Store | `SqliteExportService` writes each successful user and brewery into a fresh dated `.sqlite` database with normalized `cities`, `users`, and `breweries` tables. | | Store | `SqliteExportService` writes each successful user and brewery into a fresh dated `.sqlite` database with normalized `locations`, `users`, and `breweries` tables. |
| Log | `spdlog` writes results and warnings to the console. | | Log | `spdlog` writes results and warnings to the console. |
If name sampling, enrichment, or generation fails for a city, that city is If name sampling, enrichment, or generation fails for a city, that city is
@@ -241,27 +241,12 @@ skipped and the pipeline continues. `GenerateUsers()` runs before
`CuratedDataFilePaths` DTO (locations/personas/forenames/surnames paths) in `CuratedDataFilePaths` DTO (locations/personas/forenames/surnames paths) in
its constructor, then parses and validates curated location, persona, and its constructor, then parses and validates curated location, persona, and
forename/surname JSON, memoizing each result after its first load on a forename/surname JSON, memoizing each result after its first load on a
given instance. Each `cities.json` entry's `postal_code.city_regex` and given instance. `MockCuratedDataService` is the in-memory substitute (4
`postal_code.examples` are parsed into `City::postal_regex` and fixed locations, 3 personas, and name data for `US`/`DE`/`FR`/`BE`) used in
`City::postal_code_examples`. `MockCuratedDataService` is the in-memory `--mocked` runs.
substitute (4 fixed locations, 3 personas, and name data for
`US`/`DE`/`FR`/`BE`) used in `--mocked` runs, and carries matching
`postal_regex`/`postal_code_examples` values for its 4 locations.
- `WikipediaEnrichmentService` — queries Wikipedia extracts, caches results, - `WikipediaEnrichmentService` — queries Wikipedia extracts, caches results,
returns empty context on failure. `MockEnrichmentService` is the no-op returns empty context on failure. `MockEnrichmentService` is the no-op
substitute used in `--mocked` runs. substitute used in `--mocked` runs.
- `IPostalCodeService` — generates a postal code for a `City`, consumed by
`GenerateBreweries()` and stored on `BreweryRecord::address` (an `Address`
struct, currently just `postal_code`, mirroring the web backend's
`BreweryPostLocation`). Only `MockPostalCodeService` exists today, which
ignores `postal_regex` and returns `postal_code_examples.front()` — it's
wired into the Boost.DI graph unconditionally (no `--mocked` branch yet,
since there's no real implementation to switch to). A real implementation
still needs a **xeger**-style generator — turning a `postal_regex` pattern
into a random matching string — instead of always replaying a fixed
example; see [ROADMAP.md §9](./ROADMAP.md#9-postal-code-generation).
Street-address generation (`Address::address_line1`) has no fixture data or
service yet and remains future work.
- `LlamaGenerator` — formats prompts for Gemma 4, validates JSON output for - `LlamaGenerator` — formats prompts for Gemma 4, validates JSON output for
both `GenerateBrewery` and `GenerateUser`, retries malformed responses up both `GenerateBrewery` and `GenerateUser`, retries malformed responses up
to three times with corrective feedback in the retry prompt. The token to three times with corrective feedback in the retry prompt. The token
@@ -294,14 +279,14 @@ valid output on the first pass; the retry path is kept for resilience.
`MockGenerator` uses stable hashes for repeatable output in demos and Storybook `MockGenerator` uses stable hashes for repeatable output in demos and Storybook
runs. runs.
`CuratedJsonDataService` memoizes each of `LoadCities()`, `LoadPersonas()`, `CuratedJsonDataService` memoizes each of `LoadLocations()`, `LoadPersonas()`,
`LoadForenamesByCountry()`, and `LoadSurnamesByCountry()` independently the `LoadForenamesByCountry()`, and `LoadSurnamesByCountry()` independently the
first time each is called, since `BiergartenPipelineOrchestrator` owns a first time each is called, since `BiergartenPipelineOrchestrator` owns a
single `ICuratedDataService` instance for the whole run — later calls return single `ICuratedDataService` instance for the whole run — later calls return
the cached result instead of re-parsing. the cached result instead of re-parsing.
`GenerateUsers()` samples a forename/surname pair per city via `SampleName()`, `GenerateUsers()` samples a forename/surname pair per city via `SampleName()`,
keyed by the city's ISO 3166-1 code. Countries present in `cities.json` keyed by the city's ISO 3166-1 code. Countries present in `locations.json`
but absent from either name fixture (currently `KE`, `SE`, `SG`, `TH`, `VN`, but absent from either name fixture (currently `KE`, `SE`, `SG`, `TH`, `VN`,
`ZA`) are skipped, the same way a failed enrichment or generation call skips `ZA`) are skipped, the same way a failed enrichment or generation call skips
a city — see ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section. a city — see ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section.
@@ -318,9 +303,9 @@ a city — see ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section.
## Generated Output ## Generated Output
Each successful run stores a `BreweryRecord` (source `City`, an `Address`, Each successful run stores a `BreweryRecord` pair with the source location
and a `BreweryResult` payload), and a `UserRecord` pair with the source and a `BreweryResult` payload, and a `UserRecord` pair with the source
`City` and a `UserResult` payload. The same generated records are also location and a `UserResult` payload. The same generated records are also
written to a fresh SQLite export file named with the current UTC timestamp. written to a fresh SQLite export file named with the current UTC timestamp.
| Field | Meaning | | Field | Meaning |
@@ -329,7 +314,6 @@ written to a fresh SQLite export file named with the current UTC timestamp.
| `description_en` | Brewery description in English. | | `description_en` | Brewery description in English. |
| `name_local` | Brewery name in the local language. | | `name_local` | Brewery name in the local language. |
| `description_local` | Brewery description in the local language. | | `description_local` | Brewery description in the local language. |
| `postal_code` | Postal code generated for the brewery's city (see `IPostalCodeService`, above). |
| Field | Meaning | | Field | Meaning |
| ----------------- | ----------------------------------------------------------------- | | ----------------- | ----------------------------------------------------------------- |
@@ -355,7 +339,6 @@ code, latitude, and longitude for each entry.
| `local_languages` | Locale-aware copy selection | | `local_languages` | Locale-aware copy selection |
| `name_en`, `description_en` | Default English display content | | `name_en`, `description_en` | Default English display content |
| `name_local`, `description_local` | Local-language display content | | `name_local`, `description_local` | Local-language display content |
| `postal_code` | Brewery address matching, mirrors the web backend's `BreweryPostLocation.PostalCode` |
--- ---
@@ -424,7 +407,7 @@ Silicon; CUDA or HIP/ROCm is detected on Linux when the toolkit is present.
`MockCuratedDataService` swaps in for `CuratedJsonDataService`, so no `MockCuratedDataService` swaps in for `CuratedJsonDataService`, so no
fixture files need to be present on disk. fixture files need to be present on disk.
- `--model` when geographically grounded content matters for demos. - `--model` when geographically grounded content matters for demos.
- Keep `cities.json` structured enough to support discovery and future - Keep `locations.json` structured enough to support discovery and future
filtering. filtering.
- `personas.json`, `forenames-by-country.json`, and - `personas.json`, `forenames-by-country.json`, and
`surnames-by-country.json` are curated/vendored fixture data, not `surnames-by-country.json` are curated/vendored fixture data, not
@@ -441,7 +424,7 @@ Silicon; CUDA or HIP/ROCm is detected on Linux when the toolkit is present.
| -------------------------------------- | -------------------------------------------------- | | -------------------------------------- | -------------------------------------------------- |
| `tooling/pipeline/includes/` | Public headers and shared models. | | `tooling/pipeline/includes/` | Public headers and shared models. |
| `tooling/pipeline/src/` | Implementation files. | | `tooling/pipeline/src/` | Implementation files. |
| `tooling/pipeline/cities.json` | Curated city input copied into the build tree. | | `tooling/pipeline/locations.json` | Curated city input copied into the build tree. |
| `tooling/pipeline/personas.json` | Curated user persona archetypes copied into the build tree. | | `tooling/pipeline/personas.json` | Curated user persona archetypes copied into the build tree. |
| `tooling/pipeline/forenames-by-country.json` | Vendored (CC0) forename data by ISO 3166-1 country code. | | `tooling/pipeline/forenames-by-country.json` | Vendored (CC0) forename data by ISO 3166-1 country code. |
| `tooling/pipeline/surnames-by-country.json` | Vendored (CC0) surname data by ISO 3166-1 country code. | | `tooling/pipeline/surnames-by-country.json` | Vendored (CC0) surname data by ISO 3166-1 country code. |
@@ -463,10 +446,6 @@ Paths below are relative to `tooling/pipeline/`.
`ICuratedDataService`, and `MockCuratedDataService`, the in-memory `ICuratedDataService`, and `MockCuratedDataService`, the in-memory
`ICuratedDataService` used in `--mocked` runs. `ICuratedDataService` used in `--mocked` runs.
- `src/services/enrichment/wikipedia/` — enrichment service and cache. - `src/services/enrichment/wikipedia/` — enrichment service and cache.
- `includes/services/postal_code/``IPostalCodeService` and
`MockPostalCodeService` (header-only), consumed by `GenerateBreweries()`.
The real xeger-based implementation and its `--mocked`-aware DI wiring are
still to come.
- `src/services/sqlite/` — SQLite export implementation. - `src/services/sqlite/` — SQLite export implementation.
- `src/data_generation/llama/` — local inference, prompt loading, output - `src/data_generation/llama/` — local inference, prompt loading, output
validation. validation.

View File

@@ -54,9 +54,7 @@ architecture needs.
(present per-forename in the source data) is never discarded the way a (present per-forename in the source data) is never discarded the way a
pre-flattened `{first_name, last_name}` list would lose it; see §2. pre-flattened `{first_name, last_name}` list would lose it; see §2.
- [ ] Extend `GeneratedBrewery` with `brewery_id`, `context_completeness`, - [ ] Extend `GeneratedBrewery` with `brewery_id`, `context_completeness`,
`metadata` (currently `{location, address, brewery}` — see the `metadata` (currently just `{location, brewery}`).
`Location``City` rename and `Address` struct added in the
`postal_regex`/§9 entries below).
- [ ] Add `GeneratedBeer`, `GeneratedCheckin`, `GeneratedRating`, - [ ] Add `GeneratedBeer`, `GeneratedCheckin`, `GeneratedRating`,
`GeneratedFollow` aggregate structs. `GeneratedFollow` aggregate structs.
- [x] Add `UserRecord` (`location`, `user : UserResult`, `email`, - [x] Add `UserRecord` (`location`, `user : UserResult`, `email`,
@@ -72,30 +70,6 @@ architecture needs.
holding `generated_users_` in memory. Still missing vs. the planned holding `generated_users_` in memory. Still missing vs. the planned
`GeneratedUser`: `user_id`, `password`, and `metadata`. `GeneratedUser`: `user_id`, `password`, and `metadata`.
- [x] Add `UserPersona` (`name`, `description`, `style_affinities`). - [x] Add `UserPersona` (`name`, `description`, `style_affinities`).
- [x] Add `postal_regex` (`std::vector<std::string>`) and
`postal_code_examples` (`std::vector<std::string>`) to `Location`
(now `City` — see below), sourced from each `cities.json` entry's
`postal_code.city_regex` / `postal_code.examples`. Not part of the
originally planned `Location` shape in `diagrams/planned/class.puml`
added ahead of postal code generation work; see §9.
- [x] Renamed `Location``City` throughout the pipeline (struct, all
parameter/return types, `LocationsList``CityList`,
`LoadLocations()``LoadCities()`) so the domain vocabulary matches the
web backend's `City`/`CityId`. Field/variable names that merely hold a
`City` instance (e.g. `EnrichedCity::location`, `BreweryRecord::location`)
were left as-is — `City` itself has a `city` field (the city name
string), so renaming the container field too would read as
`enriched_city.city.city`. The SQLite export layer's own vocabulary
(`locations` table, `location_id` column, `location_cache_`, etc.) was
renamed to `cities`/`city_id` to match, since it's this pipeline's own
throwaway export, not the backend's SQL Server schema (see §5).
- [x] Added a new `Address` struct (`postal_code` only, for now) and a
`BreweryRecord::address` member, mirroring the web backend's
`BreweryPostLocation` (`AddressLine1`, `AddressLine2`, `PostalCode`,
`CityId`). Only `postal_code` is populated — `address_line1`/
`address_line2` generation has no fixture data or service yet. Users
do not get an `Address`: the backend's `UserAccount` has no address
fields, only `BreweryPostLocation` does.
## 2. Data Preloading ## 2. Data Preloading
@@ -104,7 +78,7 @@ architecture needs.
- [x] Extract an interface; have `CuratedJsonDataService` implement it. - [x] Extract an interface; have `CuratedJsonDataService` implement it.
Landed as `ICuratedDataService` Landed as `ICuratedDataService`
(`services/curated_data/curated_data_service.h`), not `DataPreloader` (`services/curated_data/curated_data_service.h`), not `DataPreloader`
`LoadCities()`, `LoadPersonas()`, `LoadForenamesByCountry()`, and `LoadLocations()`, `LoadPersonas()`, `LoadForenamesByCountry()`, and
`LoadSurnamesByCountry()` are all virtual methods returning `const&` `LoadSurnamesByCountry()` are all virtual methods returning `const&`
and take no arguments. `CuratedJsonDataService` and take no arguments. `CuratedJsonDataService`
(`services/curated_data/curated_json_data_service.h`, originally (`services/curated_data/curated_json_data_service.h`, originally
@@ -127,7 +101,7 @@ architecture needs.
and is already copied into the Docker image and is already copied into the Docker image
(`runpod/Dockerfile`), but no loader reads it yet, and the native (`runpod/Dockerfile`), but no loader reads it yet, and the native
CMake build doesn't copy it into `build/` at all — `CMakeLists.txt`'s CMake build doesn't copy it into `build/` at all — `CMakeLists.txt`'s
"Runtime Assets" step only copies `cities.json` and `prompts/`. "Runtime Assets" step only copies `locations.json` and `prompts/`.
- [x] Add `LoadPersonas()` and author `personas.json` (doesn't exist yet). - [x] Add `LoadPersonas()` and author `personas.json` (doesn't exist yet).
- [x] Add name-by-country loading, parsing - [x] Add name-by-country loading, parsing
`tooling/pipeline/forenames-by-country.json` and `tooling/pipeline/forenames-by-country.json` and
@@ -142,16 +116,10 @@ architecture needs.
`sigpwned/popular-names-by-country-dataset` (CC0) — see `sigpwned/popular-names-by-country-dataset` (CC0) — see
ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section for ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section for
provenance. Deliberately not pre-paired or filtered down to just the provenance. Deliberately not pre-paired or filtered down to just the
countries in `cities.json`: keeping the source shape (including countries in `locations.json`: keeping the source shape (including
per-forename gender) intact means the loader can support more per-forename gender) intact means the loader can support more
countries later, or gender-aware persona/bio generation, without countries later, or gender-aware persona/bio generation, without
re-sourcing data. re-sourcing data.
- [x] Parse `postal_code.city_regex` and `postal_code.examples` out of each
`cities.json` entry's nested `postal_code` object in
`CuratedJsonDataService::LoadCities()`, into `City::postal_regex`
/ `City::postal_code_examples` (see §1). `MockCuratedDataService`'s
4 fixed locations carry matching values pulled from the corresponding
`cities.json` entries.
## 3. Policy / Strategy Layer ## 3. Policy / Strategy Layer
@@ -206,20 +174,17 @@ Entirely new — no `includes/policy/` (or equivalent) directory exists today.
`kCreateUsersTableSql` / `kInsertUserSql` in `kCreateUsersTableSql` / `kInsertUserSql` in
`includes/services/database/sqlite_statement_helpers.h`) are `includes/services/database/sqlite_statement_helpers.h`) are
implemented — `GenerateUsers()` exports every successful user implemented — `GenerateUsers()` exports every successful user
alongside its resolved `city_id`. alongside its resolved `location_id`.
- [x] `ProcessRecord(const BreweryRecord&)` now also binds
`BreweryRecord::address.postal_code` into a `postal_code` column on
`breweries` (see §9).
- [ ] Extend `IExportService` with `ProcessBeer`, `ProcessCheckin`, - [ ] Extend `IExportService` with `ProcessBeer`, `ProcessCheckin`,
`ProcessRating`, `ProcessFollow` (today `ProcessRecord(const `ProcessRating`, `ProcessFollow` (today `ProcessRecord(const
BreweryRecord&)` and `ProcessRecord(const UserRecord&)` exist). BreweryRecord&)` and `ProcessRecord(const UserRecord&)` exist).
- [ ] Add `beers`, `checkins`, `ratings`, `follows` tables to - [ ] Add `beers`, `checkins`, `ratings`, `follows` tables to
`SqliteExportService::InitializeSchema()` (`cities`, `breweries`, `SqliteExportService::InitializeSchema()` (`locations`, `breweries`,
and `users` already exist) — see `kCreateCitiesTableSql` / and `users` already exist) — see `kCreateLocationsTableSql` /
`kCreateBreweriesTableSql` / `kCreateUsersTableSql` in `kCreateBreweriesTableSql` / `kCreateUsersTableSql` in
`includes/services/database/sqlite_statement_helpers.h` for the `includes/services/database/sqlite_statement_helpers.h` for the
existing pattern to follow. existing pattern to follow.
- [ ] Add a `brewery_cache_` alongside the existing `city_cache_`, and - [ ] Add a `brewery_cache_` alongside the existing `location_cache_`, and
move from one open transaction for the whole run to per-phase batched move from one open transaction for the whole run to per-phase batched
commits (`BEGIN` / `COMMIT & BEGIN` on a batch-size threshold), as commits (`BEGIN` / `COMMIT & BEGIN` on a batch-size threshold), as
shown in the planned activity diagram. shown in the planned activity diagram.
@@ -268,36 +233,6 @@ Entirely new — no `includes/policy/` (or equivalent) directory exists today.
Assets" `configure_file` step in `CMakeLists.txt` so native builds and Assets" `configure_file` step in `CMakeLists.txt` so native builds and
the Docker image agree on what's available at runtime. the Docker image agree on what's available at runtime.
## 9. Postal Code Generation
`tooling/pipeline/includes/services/postal_code/`. Not part of the original
`diagrams/planned/class.puml` sketch — added once `Location` (now `City`)
gained `postal_regex`/`postal_code_examples` (§1, §2).
- [x] Add the `IPostalCodeService` interface —
`GeneratePostalCode(location : const City&) : std::string`.
- [x] Add `MockPostalCodeService`: ignores `location.postal_regex` entirely
and returns `location.postal_code_examples.front()`.
- [ ] **Implement a real, xeger-based `IPostalCodeService`.** A xeger
generator turns a regex pattern (here, one of
`City::postal_regex`) into a random string that matches it — the
inverse of matching a string against a regex. Without it, every
generated postal code is either a hardcoded mock value or absent
entirely; a real implementation is required before postal codes can
vary per generated brewery instead of always reusing the same
curated example.
- [x] Wire `IPostalCodeService` into `main.cc`'s Boost.DI injector — bound
unconditionally to `MockPostalCodeService` (no `--mocked` branch yet,
since a real implementation doesn't exist to switch to; add one once
the xeger-based generator above lands, mirroring `IEnrichmentService`)
— and into `BiergartenPipelineOrchestrator::GenerateBreweries()`,
which calls `GeneratePostalCode(location)` per brewery and stores the
result on `BreweryRecord::address` (a new `Address` struct — see §1),
exported to the `breweries.postal_code` SQLite column (§5).
Postal-code generation is per-brewery, not per-city or per-user: users
don't have an address concept in the web backend, only breweries do
(`BreweryPostLocation`), so `UserRecord` was deliberately not extended.
--- ---
## Suggested build order ## Suggested build order
@@ -315,7 +250,3 @@ populated. Phase 1b may begin.", etc.) imply a dependency order. Roughly:
planned diagrams. planned diagrams.
5. Enrichment cache pre-warming and the city/region query decision (§7) — 5. Enrichment cache pre-warming and the city/region query decision (§7) —
useful at any point, but most valuable once phases run concurrently. useful at any point, but most valuable once phases run concurrently.
6. Postal code generation (§9) is independent of the phases above — the
xeger-based generator just needs `City::postal_regex`, already in
place. Wiring the mock into brewery output has already landed (§9); the
real generator can be swapped in once built.

View File

@@ -69,14 +69,14 @@ end note
:QueryCitiesWithCountries(); :QueryCitiesWithCountries();
|#E2EBDC|JsonLoader| |#E2EBDC|JsonLoader|
:JsonLoader::LoadCities("cities.json"); :JsonLoader::LoadLocations("locations.json");
:std::ranges::sample(all_locations, --location-count); :std::ranges::sample(all_locations, --location-count);
note right note right
--location-count defaults to 10. --location-count defaults to 10.
end note end note
|#EAF0E8|BiergartenPipelineOrchestrator| |#EAF0E8|BiergartenPipelineOrchestrator|
while (For each sampled City?) is (Remaining cities) while (For each sampled Location?) is (Remaining cities)
|#DCE8D8|IEnrichmentService| |#DCE8D8|IEnrichmentService|
:GetLocationContext(loc); :GetLocationContext(loc);
note right note right
@@ -89,7 +89,7 @@ while (For each sampled City?) is (Remaining cities)
if (Lookup failed?) then (yes) if (Lookup failed?) then (yes)
:Log warning, skip city; :Log warning, skip city;
else (no) else (no)
:Store EnrichedCity{City, region_context}; :Store EnrichedCity{Location, region_context};
endif endif
endwhile (Done) endwhile (Done)
@@ -146,12 +146,12 @@ while (For each EnrichedCity?) is (Remaining cities)
|#E0EAE0|SqliteExportService| |#E0EAE0|SqliteExportService|
:ProcessRecord(UserRecord); :ProcessRecord(UserRecord);
if (City in cache?) then (yes) if (Location in cache?) then (yes)
:Reuse city_id; :Reuse location_id;
else (no) else (no)
:Insert City & Cache ID; :Insert Location & Cache ID;
endif endif
:Insert User (FK: city_id); :Insert User (FK: location_id);
if (Exception caught during insert?) then (yes) if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator| |#EAF0E8|BiergartenPipelineOrchestrator|
@@ -197,12 +197,12 @@ while (For each EnrichedCity?) is (Remaining cities)
if (Generation successful?) then (yes) if (Generation successful?) then (yes)
|#E0EAE0|SqliteExportService| |#E0EAE0|SqliteExportService|
:ProcessRecord(GeneratedBrewery); :ProcessRecord(GeneratedBrewery);
if (City in cache?) then (yes) if (Location in cache?) then (yes)
:Reuse city_id; :Reuse location_id;
else (no) else (no)
:Insert City & Cache ID; :Insert Location & Cache ID;
endif endif
:Insert Brewery (FK: city_id); :Insert Brewery (FK: location_id);
if (Exception caught during insert?) then (yes) if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator| |#EAF0E8|BiergartenPipelineOrchestrator|

View File

@@ -31,12 +31,11 @@ class BiergartenPipelineOrchestrator {
- generator_ : std::unique_ptr<DataGenerator> - generator_ : std::unique_ptr<DataGenerator>
- exporter_ : std::unique_ptr<IExportService> - exporter_ : std::unique_ptr<IExportService>
- curated_data_service_ : std::unique_ptr<ICuratedDataService> - curated_data_service_ : std::unique_ptr<ICuratedDataService>
- postal_code_service_ : std::unique_ptr<IPostalCodeService>
- application_options_ : ApplicationOptions - application_options_ : ApplicationOptions
- generated_breweries_ : std::vector<BreweryRecord> - generated_breweries_ : std::vector<BreweryRecord>
- generated_users_ : std::vector<UserRecord> - generated_users_ : std::vector<UserRecord>
+ Run() : bool + Run() : bool
- QueryCitiesWithCountries() : std::vector<City> - QueryCitiesWithCountries() : std::vector<Location>
- GenerateBreweries(cities : std::span<const EnrichedCity>) : void - GenerateBreweries(cities : std::span<const EnrichedCity>) : void
- GenerateUsers(cities : std::span<const EnrichedCity>) : void - GenerateUsers(cities : std::span<const EnrichedCity>) : void
- LogResults() : void - LogResults() : void
@@ -91,17 +90,17 @@ class LogDispatcher {
} }
interface IEnrichmentService <<interface>> { interface IEnrichmentService <<interface>> {
+ GetLocationContext(loc : const City&) : std::string + GetLocationContext(loc : const Location&) : std::string
} }
class MockEnrichmentService { class MockEnrichmentService {
+ GetLocationContext(loc : const City&) : std::string + GetLocationContext(loc : const Location&) : std::string
} }
class WikipediaEnrichmentService { class WikipediaEnrichmentService {
- client_ : std::unique_ptr<WebClient> - client_ : std::unique_ptr<WebClient>
- extract_cache_ : std::unordered_map<std::string, std::string> - extract_cache_ : std::unordered_map<std::string, std::string>
+ GetLocationContext(loc : const City&) : std::string + GetLocationContext(loc : const Location&) : std::string
- FetchExtract(query : std::string_view) : std::string - FetchExtract(query : std::string_view) : std::string
} }
@@ -116,15 +115,15 @@ class HttpWebClient {
} }
interface DataGenerator <<interface>> { interface DataGenerator <<interface>> {
+ GenerateBrewery(location : const City&, region_context : const std::string&) : BreweryResult + GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult + GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
} }
class MockGenerator { class MockGenerator {
+ GenerateBrewery(location : const City&, region_context : const std::string&) : BreweryResult + GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult + GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
- DeterministicHash(location : const City&) : size_t - DeterministicHash(location : const Location&) : size_t
- DeterministicHash(location : const City&, persona : const UserPersona&, name : const Name&) : size_t - DeterministicHash(location : const Location&, persona : const UserPersona&, name : const Name&) : size_t
} }
class LlamaGenerator { class LlamaGenerator {
@@ -159,7 +158,7 @@ class PromptDirectory {
} }
interface ICuratedDataService <<interface>> { interface ICuratedDataService <<interface>> {
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>& + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>& + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>& + LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>& + LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
@@ -167,7 +166,7 @@ interface ICuratedDataService <<interface>> {
class JsonLoader { class JsonLoader {
- cache_ : cache - cache_ : cache
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>& + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>& + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>& + LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>& + LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
@@ -181,11 +180,11 @@ note right of JsonLoader
end note end note
class MockCuratedDataService { class MockCuratedDataService {
- locations_ : std::vector<City> - locations_ : std::vector<Location>
- personas_ : std::vector<UserPersona> - personas_ : std::vector<UserPersona>
- forenames_by_country_ : std::unordered_map<std::string, forename_list> - forenames_by_country_ : std::unordered_map<std::string, forename_list>
- surnames_by_country_ : std::unordered_map<std::string, surname_list> - surnames_by_country_ : std::unordered_map<std::string, surname_list>
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>& + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>& + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>& + LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>& + LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
@@ -197,27 +196,6 @@ note right of MockCuratedDataService
filepath arguments are ignored. filepath arguments are ignored.
end note end note
interface IPostalCodeService <<interface>> {
+ GeneratePostalCode(location : const City&) : std::string
}
class MockPostalCodeService {
+ GeneratePostalCode(location : const City&) : std::string
}
note right of MockPostalCodeService
Ignores location.postal_regex and returns
location.postal_code_examples.front(). Bound
unconditionally in main.cc's DI graph (no
--mocked branch yet -- no real implementation
exists to switch to) and called from
GenerateBreweries() per brewery, stored on
BreweryRecord::address. A real implementation
needs a xeger-style generator to turn
postal_regex into a synthesized matching
string. See ROADMAP.md §9.
end note
interface IExportService <<interface>> { interface IExportService <<interface>> {
+ Initialize() : void + Initialize() : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t + ProcessRecord(brewery : const BreweryRecord&) : uint64_t
@@ -230,17 +208,17 @@ class SqliteExportService {
- run_timestamp_utc_ : std::string - run_timestamp_utc_ : std::string
- database_path_ : std::filesystem::path - database_path_ : std::filesystem::path
- db_handle_ : SqliteDatabaseHandle - db_handle_ : SqliteDatabaseHandle
- insert_city_stmt_ : SqliteStatementHandle - insert_location_stmt_ : SqliteStatementHandle
- insert_brewery_stmt_ : SqliteStatementHandle - insert_brewery_stmt_ : SqliteStatementHandle
- insert_user_stmt_ : SqliteStatementHandle - insert_user_stmt_ : SqliteStatementHandle
- transaction_open_ : bool - transaction_open_ : bool
- city_cache_ : std::unordered_map<std::string, sqlite3_int64> - location_cache_ : std::unordered_map<std::string, sqlite3_int64>
+ Initialize() : void + Initialize() : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t + ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+ ProcessRecord(user : const UserRecord&) : uint64_t + ProcessRecord(user : const UserRecord&) : uint64_t
+ Finalize() : void + Finalize() : void
- InitializeSchema() : void - InitializeSchema() : void
- ResolveCityId(location : const City&) : sqlite3_int64 - ResolveLocationId(location : const Location&) : sqlite3_int64
} }
interface IDateTimeProvider <<interface>> { interface IDateTimeProvider <<interface>> {
@@ -257,7 +235,6 @@ BiergartenPipelineOrchestrator *-- IEnrichmentService : owns
BiergartenPipelineOrchestrator *-- DataGenerator : owns BiergartenPipelineOrchestrator *-- DataGenerator : owns
BiergartenPipelineOrchestrator *-- IExportService : owns BiergartenPipelineOrchestrator *-- IExportService : owns
BiergartenPipelineOrchestrator *-- ICuratedDataService : owns BiergartenPipelineOrchestrator *-- ICuratedDataService : owns
BiergartenPipelineOrchestrator *-- IPostalCodeService : owns
LogEntry *-- LogLevel LogEntry *-- LogLevel
LogEntry *-- PipelinePhase LogEntry *-- PipelinePhase
@@ -285,8 +262,6 @@ IPromptDirectory <|.. PromptDirectory : implements
ICuratedDataService <|.. JsonLoader : implements ICuratedDataService <|.. JsonLoader : implements
ICuratedDataService <|.. MockCuratedDataService : implements ICuratedDataService <|.. MockCuratedDataService : implements
IPostalCodeService <|.. MockPostalCodeService : implements
IExportService <|.. SqliteExportService : implements IExportService <|.. SqliteExportService : implements
SqliteExportService *-- IDateTimeProvider : owns SqliteExportService *-- IDateTimeProvider : owns
IDateTimeProvider <|.. SystemDateTimeProvider : implements IDateTimeProvider <|.. SystemDateTimeProvider : implements

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 86 KiB

After

Width:  |  Height:  |  Size: 86 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 94 KiB

After

Width:  |  Height:  |  Size: 87 KiB

View File

@@ -39,7 +39,7 @@ fork
:JsonLoader::LoadBeerStyles("beer-styles.json"); :JsonLoader::LoadBeerStyles("beer-styles.json");
:EnrichmentService::PreWarmBeerStyleCache(beer_styles); :EnrichmentService::PreWarmBeerStyleCache(beer_styles);
fork again fork again
:JsonLoader::LoadCities("cities.json"); :JsonLoader::LoadLocations("locations.json");
:EnrichmentService::PreWarmLocationCache(sampled_locations); :EnrichmentService::PreWarmLocationCache(sampled_locations);
end fork end fork
fork fork
@@ -75,7 +75,7 @@ fork
fork again fork again
|LLM Worker| |LLM Worker|
while (loc_ch has items?) is (yes) while (loc_ch has items?) is (yes)
:Receive City; :Receive Location;
:GetLocationContextFromCache(location); :GetLocationContextFromCache(location);
note right note right
@@ -146,13 +146,13 @@ end fork
fork fork
|Orchestrator| |Orchestrator|
:Loop: Sample User from user_pool_ :Loop: Sample User from user_pool_
and pair with City; and pair with Location;
:Send BreweryTask(City, User) -> loc_ch; :Send BreweryTask(Location, User) -> loc_ch;
:Close loc_ch; :Close loc_ch;
fork again fork again
|LLM Worker| |LLM Worker|
while (loc_ch has items?) is (yes) while (loc_ch has items?) is (yes)
:Receive BreweryTask(City, User); :Receive BreweryTask(Location, User);
:GetLocationContextFromCache(task.location); :GetLocationContextFromCache(task.location);
note right note right

View File

@@ -12,40 +12,17 @@ title Biergarten Data Pipeline — Class Diagram
package "Domain: Models" { package "Domain: Models" {
class City { class Location {
+ city : std::string + city : std::string
+ state_province : std::string + state_province : std::string
+ iso3166_2 : std::string + iso3166_2 : std::string
+ country : std::string + country : std::string
+ iso3166_1 : std::string + iso3166_1 : std::string
+ postal_regex : std::vector<std::string>
+ postal_code_examples : std::vector<std::string>
+ local_languages : std::vector<std::string> + local_languages : std::vector<std::string>
+ latitude : double + latitude : double
+ longitude : double + longitude : double
} }
note right of City
postal_regex / postal_code_examples are
sourced from each cities.json entry's
postal_code.city_regex / postal_code.examples
(implemented today, ahead of the rest of this
diagram -- see ROADMAP.md §1/§2). Renamed from
Location to City to match the web backend's
City/CityId vocabulary -- see ROADMAP.md §1.
end note
class Address {
+ postal_code : std::string
}
note right of Address
Mirrors the web backend's BreweryPostLocation.
Only postal_code is populated today --
address_line1/address_line2 have no fixture
data or generator yet. See ROADMAP.md §9.
end note
class LocationContext { class LocationContext {
+ text : std::string + text : std::string
+ completeness : Completeness + completeness : Completeness
@@ -59,7 +36,7 @@ package "Domain: Models" {
} }
class EnrichedCity { class EnrichedCity {
+ location : City + location : Location
+ context : LocationContext + context : LocationContext
} }
@@ -153,8 +130,7 @@ package "Domain: Models" {
class GeneratedBrewery { class GeneratedBrewery {
+ brewery_id : uint64_t + brewery_id : uint64_t
+ location : City + location : Location
+ address : Address
+ brewery : BreweryResult + brewery : BreweryResult
+ context_completeness : LocationContext::Completeness + context_completeness : LocationContext::Completeness
+ metadata : GenerationMetadata + metadata : GenerationMetadata
@@ -163,7 +139,7 @@ package "Domain: Models" {
class GeneratedBeer { class GeneratedBeer {
+ beer_id : uint64_t + beer_id : uint64_t
+ brewery_id : uint64_t + brewery_id : uint64_t
+ location : City + location : Location
+ style : BeerStyle + style : BeerStyle
+ beer : BeerResult + beer : BeerResult
+ metadata : GenerationMetadata + metadata : GenerationMetadata
@@ -171,7 +147,7 @@ package "Domain: Models" {
class GeneratedUser { class GeneratedUser {
+ user_id : uint64_t + user_id : uint64_t
+ location : City + location : Location
+ user : UserResult + user : UserResult
+ email : std::string + email : std::string
+ date_of_birth : std::string + date_of_birth : std::string
@@ -251,27 +227,27 @@ package "Domain: Application Configuration" {
package "Domain: Policy" { package "Domain: Policy" {
interface ContextStrategy <<interface>> { interface ContextStrategy <<interface>> {
+ QueriesFor(loc : const City&) : std::vector<std::string> + QueriesFor(loc : const Location&) : std::vector<std::string>
+ MaxContextChars() : size_t + MaxContextChars() : size_t
} }
class BreweryContextStrategy { class BreweryContextStrategy {
+ QueriesFor(loc : const City&) : std::vector<std::string> + QueriesFor(loc : const Location&) : std::vector<std::string>
+ MaxContextChars() : size_t + MaxContextChars() : size_t
} }
class BeerContextStrategy { class BeerContextStrategy {
+ QueriesFor(loc : const City&) : std::vector<std::string> + QueriesFor(loc : const Location&) : std::vector<std::string>
+ MaxContextChars() : size_t + MaxContextChars() : size_t
} }
interface SamplingStrategy <<interface>> { interface SamplingStrategy <<interface>> {
+ Sample(locations : const std::vector<City>&) : std::vector<City> + Sample(locations : const std::vector<Location>&) : std::vector<Location>
} }
class UniformSamplingStrategy { class UniformSamplingStrategy {
- sample_size_ : size_t - sample_size_ : size_t
+ Sample(locations : const std::vector<City>&) : std::vector<City> + Sample(locations : const std::vector<Location>&) : std::vector<Location>
} }
interface BeerSelectionStrategy <<interface>> { interface BeerSelectionStrategy <<interface>> {
@@ -393,7 +369,7 @@ package "Infrastructure: Pipeline Channel" {
package "Infrastructure: Data Preloading" { package "Infrastructure: Data Preloading" {
interface ICuratedDataService <<interface>> { interface ICuratedDataService <<interface>> {
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>& + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>& + LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>& + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>& + LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
@@ -401,7 +377,7 @@ package "Infrastructure: Data Preloading" {
} }
class JsonLoader { class JsonLoader {
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>& + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>& + LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>& + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>& + LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
@@ -415,7 +391,7 @@ package "Infrastructure: Data Preloading" {
end note end note
class MockCuratedDataService { class MockCuratedDataService {
+ LoadCities(filepath : const std::filesystem::path&) : const std::vector<City>& + LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>& + LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>& + LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>& + LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
@@ -434,13 +410,13 @@ package "Infrastructure: Data Preloading" {
package "Infrastructure: Enrichment" { package "Infrastructure: Enrichment" {
interface EnrichmentService <<interface>> { interface EnrichmentService <<interface>> {
+ GetLocationContext(loc : const City&,\n strategy : const ContextStrategy&) : LocationContext + GetLocationContext(loc : const Location&,\n strategy : const ContextStrategy&) : LocationContext
} }
class WikipediaService { class WikipediaService {
- client_ : std::unique_ptr<WebClient> - client_ : std::unique_ptr<WebClient>
- extract_cache_ : std::unordered_map<std::string, std::string> - extract_cache_ : std::unordered_map<std::string, std::string>
+ GetLocationContext(loc : const City&,\n strategy : const ContextStrategy&) : LocationContext + GetLocationContext(loc : const Location&,\n strategy : const ContextStrategy&) : LocationContext
- FetchExtract(query : std::string_view) : std::string - FetchExtract(query : std::string_view) : std::string
} }
@@ -456,39 +432,6 @@ package "Infrastructure: Enrichment" {
} }
package "Infrastructure: Postal Code Generation" {
interface IPostalCodeService <<interface>> {
+ GeneratePostalCode(location : const City&) : std::string
}
class MockPostalCodeService {
+ GeneratePostalCode(location : const City&) : std::string
}
class XegerPostalCodeService {
+ GeneratePostalCode(location : const City&) : std::string
}
note right of MockPostalCodeService
Implemented today: ignores postal_regex and
returns postal_code_examples.front(). Not yet
bound in main.cc's DI graph or used by
BiergartenPipelineOrchestrator.
end note
note right of XegerPostalCodeService
Not implemented yet. Picks one of
location.postal_regex and generates a random
string matching it (a "xeger" -- the inverse of
regex matching) instead of always replaying a
fixed example. See ROADMAP.md §9.
end note
IPostalCodeService <|.. MockPostalCodeService
IPostalCodeService <|.. XegerPostalCodeService
}
package "Infrastructure: Prompting" { package "Infrastructure: Prompting" {
interface IPromptDirectory <<interface>> { interface IPromptDirectory <<interface>> {
@@ -508,8 +451,8 @@ package "Infrastructure: Prompting" {
package "Infrastructure: Data Generation" { package "Infrastructure: Data Generation" {
interface DataGenerator <<interface>> { interface DataGenerator <<interface>> {
+ GenerateBrewery(location : const City&,\n context : const LocationContext&) : BreweryResult + GenerateBrewery(location : const Location&,\n context : const LocationContext&) : BreweryResult
+ GenerateBeer(brewery_id : uint64_t,\n location : const City&,\n context : const LocationContext&,\n style : const BeerStyle&) : BeerResult + GenerateBeer(brewery_id : uint64_t,\n location : const Location&,\n context : const LocationContext&,\n style : const BeerStyle&) : BeerResult
+ GenerateUser(city : const EnrichedCity&,\n persona : const UserPersona&,\n name : const Name&) : UserResult + GenerateUser(city : const EnrichedCity&,\n persona : const UserPersona&,\n name : const Name&) : UserResult
+ GenerateCheckin(user : const GeneratedUser&,\n brewery : const GeneratedBrewery&,\n timestamp : const std::string&) : CheckinResult + GenerateCheckin(user : const GeneratedUser&,\n brewery : const GeneratedBrewery&,\n timestamp : const std::string&) : CheckinResult
+ GenerateRating(user : const GeneratedUser&,\n beer : const GeneratedBeer&,\n checkin_id : uint64_t) : RatingResult + GenerateRating(user : const GeneratedUser&,\n beer : const GeneratedBeer&,\n checkin_id : uint64_t) : RatingResult
@@ -521,7 +464,7 @@ package "Infrastructure: Data Generation" {
+ GenerateUser(...) : UserResult + GenerateUser(...) : UserResult
+ GenerateCheckin(...) : CheckinResult + GenerateCheckin(...) : CheckinResult
+ GenerateRating(...) : RatingResult + GenerateRating(...) : RatingResult
- DeterministicHash(location : const City&) : size_t - DeterministicHash(location : const Location&) : size_t
} }
class LlamaGenerator { class LlamaGenerator {
@@ -623,8 +566,8 @@ class BiergartenPipelineOrchestrator {
- follow_pool_ : std::vector<GeneratedFollow> - follow_pool_ : std::vector<GeneratedFollow>
-- --
+ Run() : bool + Run() : bool
- RunUserPhase(locations : const std::vector<City>&) : void - RunUserPhase(locations : const std::vector<Location>&) : void
- RunBreweryAndBeerPhase(locations : const std::vector<City>&) : void - RunBreweryAndBeerPhase(locations : const std::vector<Location>&) : void
- RunCheckinPhase() : void - RunCheckinPhase() : void
- RunRatingPhase() : void - RunRatingPhase() : void
- RunFollowPhase() : void - RunFollowPhase() : void
@@ -682,17 +625,16 @@ PipelineLogger o-- BoundedChannel : logs to
LogWorker o-- BoundedChannel : drains from LogWorker o-- BoundedChannel : drains from
' --- Domain Containment --- ' --- Domain Containment ---
EnrichedCity *-- City EnrichedCity *-- Location
EnrichedCity *-- LocationContext EnrichedCity *-- LocationContext
GeneratedBrewery *-- City GeneratedBrewery *-- Location
GeneratedBrewery *-- Address
GeneratedBrewery *-- BreweryResult GeneratedBrewery *-- BreweryResult
GeneratedBrewery *-- GenerationMetadata GeneratedBrewery *-- GenerationMetadata
GeneratedBeer *-- City GeneratedBeer *-- Location
GeneratedBeer *-- BeerStyle GeneratedBeer *-- BeerStyle
GeneratedBeer *-- BeerResult GeneratedBeer *-- BeerResult
GeneratedBeer *-- GenerationMetadata GeneratedBeer *-- GenerationMetadata
GeneratedUser *-- City GeneratedUser *-- Location
GeneratedUser *-- UserResult GeneratedUser *-- UserResult
GeneratedUser *-- GenerationMetadata GeneratedUser *-- GenerationMetadata
GeneratedCheckin *-- CheckinResult GeneratedCheckin *-- CheckinResult

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 142 KiB

After

Width:  |  Height:  |  Size: 142 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 257 KiB

After

Width:  |  Height:  |  Size: 240 KiB

View File

@@ -140,8 +140,6 @@ FetchContent_MakeAvailable(cpp-httplib)
add_executable(${PROJECT_NAME} add_executable(${PROJECT_NAME}
includes/services/enrichment/mock_enrichment.h includes/services/enrichment/mock_enrichment.h
includes/services/curated_data/mock_curated_data_service.h includes/services/curated_data/mock_curated_data_service.h
includes/services/postal_code/postal_code_service.h
includes/services/postal_code/mock_postal_code_service.h
includes/json_handling/pretty_print.h) includes/json_handling/pretty_print.h)
# --- Entry point --- # --- Entry point ---
@@ -266,8 +264,8 @@ target_compile_options(biergarten-pipeline PRIVATE
# 7. Runtime Assets # 7. Runtime Assets
configure_file( configure_file(
${CMAKE_SOURCE_DIR}/cities.json ${CMAKE_SOURCE_DIR}/locations.json
${CMAKE_BINARY_DIR}/cities.json ${CMAKE_BINARY_DIR}/locations.json
COPYONLY COPYONLY
) )
configure_file( configure_file(
@@ -290,3 +288,4 @@ add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
${CMAKE_SOURCE_DIR}/prompts ${CMAKE_SOURCE_DIR}/prompts
${CMAKE_BINARY_DIR}/prompts ${CMAKE_BINARY_DIR}/prompts
) )

File diff suppressed because it is too large Load Diff

View File

@@ -64,10 +64,6 @@
#include "services/logging/log_producer.h" #include "services/logging/log_producer.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
// --- services: postal_code ---
#include "services/postal_code/mock_postal_code_service.h"
#include "services/postal_code/postal_code_service.h"
// --- services: prompting --- // --- services: prompting ---
#include "services/prompting/prompt_directory.h" #include "services/prompting/prompt_directory.h"

View File

@@ -16,7 +16,7 @@
#include "services/database/export_service.h" #include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h" #include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
#include "services/postal_code/postal_code_service.h"
/** /**
* @brief Main orchestrator for the Biergarten data generation pipeline. * @brief Main orchestrator for the Biergarten data generation pipeline.
* *
@@ -34,7 +34,6 @@ class BiergartenPipelineOrchestrator {
* @param exporter Database backend for persisting generated records. * @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name * @param curated_data_service Loads curated location, persona, and name
* data used to seed generation. * data used to seed generation.
* @param postal_code_service
* @param application_options CLI configuration and paths. * @param application_options CLI configuration and paths.
*/ */
BiergartenPipelineOrchestrator( BiergartenPipelineOrchestrator(
@@ -43,7 +42,6 @@ class BiergartenPipelineOrchestrator {
std::unique_ptr<DataGenerator> generator, std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options); const ApplicationOptions& application_options);
/** /**
@@ -80,7 +78,6 @@ class BiergartenPipelineOrchestrator {
*/ */
std::unique_ptr<IExportService> exporter_; std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_; std::unique_ptr<ICuratedDataService> curated_data_service_;
std::unique_ptr<IPostalCodeService> postal_code_service_;
ApplicationOptions application_options_; ApplicationOptions application_options_;
@@ -90,7 +87,7 @@ class BiergartenPipelineOrchestrator {
* @return Vector of locations randomly sampled per * @return Vector of locations randomly sampled per
* PipelineOptions::location_count. * PipelineOptions::location_count.
*/ */
std::vector<City> QueryCitiesWithCountries(); std::vector<Location> QueryCitiesWithCountries();
/** /**
* @brief Generate breweries for enriched cities. * @brief Generate breweries for enriched cities.

View File

@@ -24,7 +24,7 @@ class DataGenerator {
* @param region_context Additional regional context text. * @param region_context Additional regional context text.
* @return Brewery generation result. * @return Brewery generation result.
*/ */
virtual BreweryResult GenerateBrewery(const City& location, virtual BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) = 0; const std::string& region_context) = 0;
/** /**

View File

@@ -55,7 +55,7 @@ class LlamaGenerator final : public DataGenerator {
* @param region_context Additional regional context. * @param region_context Additional regional context.
* @return Generated brewery result. * @return Generated brewery result.
*/ */
BreweryResult GenerateBrewery(const City& location, BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) override; const std::string& region_context) override;
/** /**

View File

@@ -24,7 +24,7 @@ class MockGenerator final : public DataGenerator {
* @param region_context Unused for mock generation. * @param region_context Unused for mock generation.
* @return Generated brewery result. * @return Generated brewery result.
*/ */
BreweryResult GenerateBrewery(const City& location, BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) override; const std::string& region_context) override;
/** /**
@@ -46,7 +46,7 @@ class MockGenerator final : public DataGenerator {
* @param location City and country names. * @param location City and country names.
* @return Deterministic hash value. * @return Deterministic hash value.
*/ */
static size_t DeterministicHash(const City& location); static size_t DeterministicHash(const Location& location);
/** /**
* @brief Combines city, persona, and name into a stable hash value. * @brief Combines city, persona, and name into a stable hash value.
@@ -56,7 +56,7 @@ class MockGenerator final : public DataGenerator {
* @param name Sampled first/last name. * @param name Sampled first/last name.
* @return Deterministic hash value. * @return Deterministic hash value.
*/ */
static size_t DeterministicHash(const City& location, static size_t DeterministicHash(const Location& location,
const UserPersona& persona, const Name& name); const UserPersona& persona, const Name& name);
// Hash stride constants for deterministic distribution across fixed-size // Hash stride constants for deterministic distribution across fixed-size

View File

@@ -7,7 +7,6 @@
* enriched data, and complete generation results. * enriched data, and complete generation results.
*/ */
#include <optional>
#include <string> #include <string>
#include "data_model/models.h" #include "data_model/models.h"
@@ -88,45 +87,15 @@ struct UserResult {
* @brief Enriched city data with Wikipedia context. * @brief Enriched city data with Wikipedia context.
*/ */
struct EnrichedCity { struct EnrichedCity {
City location; Location location;
std::string region_context{}; std::string region_context{};
}; };
/**
* @brief A brewery's address. Owns the `City` it belongs to alongside its
* street-level fields. Mirrors the `brewery_addresses` table and the
* backend's `BreweryPostLocation` shape.
*
* Only `city` and `postal_code` are populated today -- street-address
* generation has no fixture data or service yet, so the address lines stay
* empty.
*/
struct BreweryAddress {
City city{};
std::optional<std::string> address_line_1{};
std::optional<std::string> address_line_2{};
std::string postal_code{};
};
/**
* @brief A user's address. Owns the `City` it belongs to alongside its
* street-level fields. Mirrors the `user_addresses` table.
*
* Only `city` is populated today; the street-level columns exist for future
* enrichment.
*/
struct UserAddress {
City city{};
std::optional<std::string> address_line_1{};
std::optional<std::string> address_line_2{};
std::optional<std::string> postal_code{};
};
/** /**
* @brief Helper struct to store generated brewery data. * @brief Helper struct to store generated brewery data.
*/ */
struct BreweryRecord { struct BreweryRecord {
BreweryAddress address; Location location;
BreweryResult brewery; BreweryResult brewery;
}; };
@@ -138,7 +107,7 @@ struct BreweryRecord {
* consumer can register real accounts from the pipeline's SQLite export. * consumer can register real accounts from the pipeline's SQLite export.
*/ */
struct UserRecord { struct UserRecord {
UserAddress address{}; Location location;
UserResult user; UserResult user;
std::string email{}; std::string email{};
std::string date_of_birth{}; std::string date_of_birth{};

View File

@@ -22,13 +22,13 @@ class ILogger;
namespace prog_opts = boost::program_options; namespace prog_opts = boost::program_options;
// ============================================================================ // ============================================================================
// City Models // Location Models
// ============================================================================ // ============================================================================
/** /**
* @brief Canonical location record for city-level generation. * @brief Canonical location record for city-level generation.
*/ */
struct City { struct Location {
std::string city{}; std::string city{};
std::string state_province{}; std::string state_province{};
@@ -44,22 +44,20 @@ struct City {
*/ */
std::string iso3166_1{}; std::string iso3166_1{};
/**
* @brief A list of regular expressions used for valid postal codes in the
* region.
*
*/
std::vector<std::string> postal_regex{};
/**
* @brief Example postal codes for the region.
*/
std::vector<std::string> postal_code_examples{};
/** /**
* @brief Local language codes in priority order. * @brief Local language codes in priority order.
*/ */
std::vector<std::string> local_languages{}; std::vector<std::string> local_languages{};
/**
* @brief Latitude in decimal degrees.
*/
double latitude{};
/**
* @brief Longitude in decimal degrees.
*/
double longitude{};
}; };
// ============================================================================ // ============================================================================

View File

@@ -15,7 +15,7 @@
using ForenameList = std::vector<ForenameEntry>; using ForenameList = std::vector<ForenameEntry>;
using SurnameList = std::vector<std::string>; using SurnameList = std::vector<std::string>;
using CityList = std::vector<City>; using LocationsList = std::vector<Location>;
using PersonasList = std::vector<UserPersona>; using PersonasList = std::vector<UserPersona>;
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>; using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>; using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;
@@ -37,7 +37,7 @@ class ICuratedDataService {
/** /**
* @brief Loads all curated location records. * @brief Loads all curated location records.
*/ */
virtual const CityList& LoadCities() = 0; virtual const LocationsList& LoadLocations() = 0;
/** /**
* @brief Loads all curated persona records. * @brief Loads all curated persona records.

View File

@@ -16,7 +16,7 @@
* CuratedJsonDataService. * CuratedJsonDataService.
*/ */
struct CuratedDataFilePaths { struct CuratedDataFilePaths {
std::filesystem::path cities_path; std::filesystem::path locations_path;
std::filesystem::path personas_path; std::filesystem::path personas_path;
std::filesystem::path forenames_path; std::filesystem::path forenames_path;
std::filesystem::path surnames_path; std::filesystem::path surnames_path;
@@ -27,7 +27,7 @@ struct CuratedDataFilePaths {
*/ */
class CuratedJsonDataService final : public ICuratedDataService { class CuratedJsonDataService final : public ICuratedDataService {
struct cache { struct cache {
CityList locations; LocationsList locations;
PersonasList personas; PersonasList personas;
ForenamesByCountryMap forenames_by_country; ForenamesByCountryMap forenames_by_country;
SurnamesByCountryMap surnames_by_country; SurnamesByCountryMap surnames_by_country;
@@ -43,9 +43,9 @@ class CuratedJsonDataService final : public ICuratedDataService {
explicit CuratedJsonDataService(CuratedDataFilePaths filepaths); explicit CuratedJsonDataService(CuratedDataFilePaths filepaths);
/** /**
* @brief Parses a JSON array file and returns all city records. * @brief Parses a JSON array file and returns all location records.
*/ */
const CityList& LoadCities() override; const LocationsList& LoadLocations() override;
/** /**
* @brief Parses a JSON array file and returns all persona records. * @brief Parses a JSON array file and returns all persona records.

View File

@@ -20,7 +20,7 @@ class MockCuratedDataService final : public ICuratedDataService {
public: public:
MockCuratedDataService(); MockCuratedDataService();
const CityList& LoadCities() override; const LocationsList& LoadLocations() override;
const PersonasList& LoadPersonas() override; const PersonasList& LoadPersonas() override;
@@ -29,7 +29,7 @@ class MockCuratedDataService final : public ICuratedDataService {
const SurnamesByCountryMap& LoadSurnamesByCountry() override; const SurnamesByCountryMap& LoadSurnamesByCountry() override;
private: private:
CityList locations_; LocationsList locations_;
PersonasList personas_; PersonasList personas_;
ForenamesByCountryMap forenames_by_country_; ForenamesByCountryMap forenames_by_country_;
SurnamesByCountryMap surnames_by_country_; SurnamesByCountryMap surnames_by_country_;

View File

@@ -45,7 +45,7 @@ class SqliteExportService final : public IExportService {
void RollbackAndCloseNoThrow() noexcept; void RollbackAndCloseNoThrow() noexcept;
[[nodiscard]] std::filesystem::path BuildDatabasePath() const; [[nodiscard]] std::filesystem::path BuildDatabasePath() const;
[[nodiscard]] static std::string BuildCityKey(const City& location); [[nodiscard]] static std::string BuildLocationKey(const Location& location);
/** /**
* @brief Returns the row id for @p location, inserting it first if it has * @brief Returns the row id for @p location, inserting it first if it has
@@ -54,20 +54,18 @@ class SqliteExportService final : public IExportService {
* Shared by both ProcessRecord() overloads so breweries and users * Shared by both ProcessRecord() overloads so breweries and users
* referencing the same location resolve to the same row. * referencing the same location resolve to the same row.
*/ */
[[nodiscard]] sqlite3_int64 ResolveCityId(const City& location); [[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location);
std::unique_ptr<IDateTimeProvider> date_time_provider_; std::unique_ptr<IDateTimeProvider> date_time_provider_;
std::filesystem::path output_path_; std::filesystem::path output_path_;
std::string run_timestamp_utc_; std::string run_timestamp_utc_;
std::filesystem::path database_path_; std::filesystem::path database_path_;
SqliteDatabaseHandle db_handle_; SqliteDatabaseHandle db_handle_;
SqliteStatementHandle insert_city_stmt_; SqliteStatementHandle insert_location_stmt_;
SqliteStatementHandle insert_brewery_stmt_; SqliteStatementHandle insert_brewery_stmt_;
SqliteStatementHandle insert_brewery_address_stmt_;
SqliteStatementHandle insert_user_stmt_; SqliteStatementHandle insert_user_stmt_;
SqliteStatementHandle insert_user_address_stmt_;
bool transaction_open_ = false; bool transaction_open_ = false;
std::unordered_map<std::string, sqlite3_int64> city_cache_; std::unordered_map<std::string, sqlite3_int64> location_cache_;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_

View File

@@ -17,9 +17,9 @@
namespace sqlite_export_service_internal { namespace sqlite_export_service_internal {
inline constexpr std::string_view kCreateCitiesTableSql = R"sql( inline constexpr std::string_view kCreateLocationsTableSql = R"sql(
CREATE TABLE IF NOT EXISTS cities ( CREATE TABLE IF NOT EXISTS locations (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
city TEXT NOT NULL, city TEXT NOT NULL,
state_province TEXT NOT NULL, state_province TEXT NOT NULL,
@@ -27,7 +27,9 @@ CREATE TABLE IF NOT EXISTS cities (
country TEXT NOT NULL, country TEXT NOT NULL,
iso3166_1 TEXT NOT NULL, iso3166_1 TEXT NOT NULL,
local_languages_json TEXT NOT NULL, local_languages_json TEXT NOT NULL,
UNIQUE(city, state_province, iso3166_2, country) latitude REAL NOT NULL,
longitude REAL NOT NULL,
UNIQUE(city, state_province, iso3166_2, country, latitude, longitude)
); );
)sql"; )sql";
@@ -36,21 +38,23 @@ inline constexpr std::string_view kCreateBreweriesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS breweries ( CREATE TABLE IF NOT EXISTS breweries (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
city_id INTEGER NOT NULL, location_id INTEGER NOT NULL,
name_en TEXT NOT NULL, name_en TEXT NOT NULL,
description_en TEXT NOT NULL, description_en TEXT NOT NULL,
name_local TEXT NOT NULL, name_local TEXT NOT NULL,
description_local TEXT NOT NULL, description_local TEXT NOT NULL,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
); );
CREATE INDEX IF NOT EXISTS idx_breweries_city_id ON breweries(city_id); CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id);
)sql"; )sql";
inline constexpr std::string_view kCreateUsersTableSql = R"sql( inline constexpr std::string_view kCreateUsersTableSql = R"sql(
CREATE TABLE IF NOT EXISTS users ( CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
location_id INTEGER NOT NULL,
first_name TEXT NOT NULL, first_name TEXT NOT NULL,
last_name TEXT NOT NULL, last_name TEXT NOT NULL,
gender TEXT NOT NULL, gender TEXT NOT NULL,
@@ -58,61 +62,30 @@ CREATE TABLE IF NOT EXISTS users (
bio TEXT NOT NULL, bio TEXT NOT NULL,
activity_weight REAL NOT NULL, activity_weight REAL NOT NULL,
email TEXT NOT NULL UNIQUE, email TEXT NOT NULL UNIQUE,
date_of_birth TEXT NOT NULL date_of_birth TEXT NOT NULL,
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
); );
CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id);
)sql"; )sql";
inline constexpr std::string_view kCreateBreweryAddressTableSql = R"sql( inline constexpr std::string_view kInsertLocationSql = R"sql(
CREATE TABLE IF NOT EXISTS brewery_addresses ( INSERT INTO locations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
brewery_id INTEGER NOT NULL,
address_line_1 TEXT,
address_line_2 TEXT,
postal_code TEXT NOT NULL,
city_id INTEGER NOT NULL,
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_brewery_id
ON brewery_addresses(brewery_id);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_city_id
ON brewery_addresses(city_id);
)sql";
inline constexpr std::string_view kCreateUserAddressTableSql = R"sql(
CREATE TABLE IF NOT EXISTS user_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
address_line_1 TEXT,
address_line_2 TEXT,
postal_code TEXT,
city_id INTEGER NOT NULL,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_user_addresses_user_id
ON user_addresses(user_id);
CREATE INDEX IF NOT EXISTS idx_user_addresses_city_id
ON user_addresses(city_id);
)sql";
inline constexpr std::string_view kInsertCitySql = R"sql(
INSERT INTO cities (
city, city,
state_province, state_province,
iso3166_2, iso3166_2,
country, country,
iso3166_1, iso3166_1,
local_languages_json local_languages_json,
) VALUES (?, ?, ?, ?, ?, ?); latitude,
longitude
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
)sql"; )sql";
inline constexpr std::string_view kInsertBrewerySql = R"sql( inline constexpr std::string_view kInsertBrewerySql = R"sql(
INSERT INTO breweries ( INSERT INTO breweries (
city_id, location_id,
name_en, name_en,
description_en, description_en,
name_local, name_local,
@@ -122,6 +95,7 @@ INSERT INTO breweries (
inline constexpr std::string_view kInsertUserSql = R"sql( inline constexpr std::string_view kInsertUserSql = R"sql(
INSERT INTO users ( INSERT INTO users (
location_id,
first_name, first_name,
last_name, last_name,
gender, gender,
@@ -130,37 +104,24 @@ INSERT INTO users (
activity_weight, activity_weight,
email, email,
date_of_birth date_of_birth
) VALUES (?, ?, ?, ?, ?, ?, ?, ?); ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
INSERT INTO brewery_addresses (
brewery_id,
postal_code,
city_id
) VALUES (?, ?, ?);
)sql";
inline constexpr std::string_view kInsertUserAddressSql = R"sql(
INSERT INTO user_addresses (
user_id,
city_id
) VALUES (?, ?);
)sql"; )sql";
// sqlite3_bind_*() parameter indices are 1-based, matching the "?" // sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above. // placeholder order in the SQL above.
enum CityBindIndex { enum LocationBindIndex {
kCityNameBindIndex = 1, kLocationCityBindIndex = 1,
kCityStateProvinceBindIndex, kLocationStateProvinceBindIndex,
kCityIso31662BindIndex, kLocationIso31662BindIndex,
kCityCountryBindIndex, kLocationCountryBindIndex,
kCityIso31661BindIndex, kLocationIso31661BindIndex,
kCityLanguagesBindIndex, kLocationLanguagesBindIndex,
kLocationLatitudeBindIndex,
kLocationLongitudeBindIndex,
}; };
enum BreweryBindIndex { enum BreweryBindIndex {
kBreweryCityIdBindIndex = 1, kBreweryLocationIdBindIndex = 1,
kBreweryEnglishNameBindIndex, kBreweryEnglishNameBindIndex,
kBreweryEnglishDescriptionBindIndex, kBreweryEnglishDescriptionBindIndex,
kBreweryLocalNameBindIndex, kBreweryLocalNameBindIndex,
@@ -168,7 +129,8 @@ enum BreweryBindIndex {
}; };
enum UserBindIndex { enum UserBindIndex {
kUserFirstNameBindIndex = 1, kUserLocationIdBindIndex = 1,
kUserFirstNameBindIndex,
kUserLastNameBindIndex, kUserLastNameBindIndex,
kUserGenderBindIndex, kUserGenderBindIndex,
kUserUsernameBindIndex, kUserUsernameBindIndex,
@@ -178,17 +140,6 @@ enum UserBindIndex {
kUserDateOfBirthBindIndex, kUserDateOfBirthBindIndex,
}; };
enum BreweryAddressBindIndex {
kBreweryAddressBreweryIdBindIndex = 1,
kBreweryAddressPostalCodeBindIndex,
kBreweryAddressCityIdBindIndex,
};
enum UserAddressBindIndex {
kUserAddressUserIdBindIndex = 1,
kUserAddressCityIdBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql, std::string_view sql,
const char* action); const char* action);

View File

@@ -23,7 +23,7 @@ class IEnrichmentService {
* @param loc Location to enrich. * @param loc Location to enrich.
* @return Context text, or an empty string if unavailable. * @return Context text, or an empty string if unavailable.
*/ */
virtual std::string GetLocationContext(const City& loc) = 0; virtual std::string GetLocationContext(const Location& loc) = 0;
}; };
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_ #endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_

View File

@@ -15,7 +15,7 @@
*/ */
class MockEnrichmentService final : public IEnrichmentService { class MockEnrichmentService final : public IEnrichmentService {
public: public:
std::string GetLocationContext(const City& /*loc*/) override { std::string GetLocationContext(const Location& /*loc*/) override {
return {}; return {};
} }
}; };

View File

@@ -29,7 +29,7 @@ class WikipediaEnrichmentService final : public IEnrichmentService {
/** /**
* @brief Returns the Wikipedia-derived context for a location. * @brief Returns the Wikipedia-derived context for a location.
*/ */
[[nodiscard]] std::string GetLocationContext(const City& loc) override; [[nodiscard]] std::string GetLocationContext(const Location& loc) override;
private: private:
std::string FetchExtract(std::string_view query); std::string FetchExtract(std::string_view query);

View File

@@ -1,32 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/mock_postal_code_service.h
* @brief Placeholder IPostalCodeService used until xeger-based generation is
* implemented.
*/
#include <stdexcept>
#include <string>
#include "data_model/models.h"
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service that returns the location's first known
* example postal code rather than generating one from its regex.
*/
class MockPostalCodeService final : public IPostalCodeService {
public:
std::string GeneratePostalCode(const City& location) override {
if (location.postal_code_examples.empty()) {
throw std::runtime_error(
"MockPostalCodeService: location has no postal_code_examples: " +
location.city);
}
return location.postal_code_examples.front();
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_

View File

@@ -1,30 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/postal_code_service.h
* @brief Abstraction for generating a postal code for a location.
*/
#include <string>
#include "data_model/models.h"
/**
* @brief Interface for services that generate a postal code string matching
* a location's postal code format (`City::postal_regex`).
*/
class IPostalCodeService {
public:
virtual ~IPostalCodeService() = default;
/**
* @brief Generates a postal code for the given location.
*
* @param location Location whose postal code format to generate for.
* @return A postal code string matching `location.postal_regex`.
*/
virtual std::string GeneratePostalCode(const City& location) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_

File diff suppressed because it is too large Load Diff

View File

@@ -52,7 +52,7 @@ WORKDIR /app/build
COPY --from=builder /app/build/biergarten-pipeline ./ COPY --from=builder /app/build/biergarten-pipeline ./
# Copy required config files # Copy required config files
COPY cities.json /app/build/ COPY locations.json /app/build/
COPY beer-styles.json /app/build/ COPY beer-styles.json /app/build/
# Copy prompt templates # Copy prompt templates

View File

@@ -7,20 +7,16 @@
#include <utility> #include <utility>
#include "services/postal_code/postal_code_service.h"
BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator( BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::shared_ptr<ILogger> logger, std::shared_ptr<ILogger> logger,
std::unique_ptr<IEnrichmentService> context_service, std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator, std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options) const ApplicationOptions& application_options)
: logger_(std::move(logger)), : logger_(std::move(logger)),
context_service_(std::move(context_service)), context_service_(std::move(context_service)),
generator_(std::move(generator)), generator_(std::move(generator)),
exporter_(std::move(exporter)), exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)), curated_data_service_(std::move(curated_data_service)),
postal_code_service_(std::move(postal_code_service)),
application_options_(application_options) {} application_options_(application_options) {}

View File

@@ -22,17 +22,12 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
const auto generate_record = const auto generate_record =
[this, &skipped_count]( [this, &skipped_count](
const City& location, const Location& location,
const std::string& region_context) -> std::optional<BreweryRecord> { const std::string& region_context) -> std::optional<BreweryRecord> {
try { try {
const BreweryResult brewery = const BreweryResult brewery =
generator_->GenerateBrewery(location, region_context); generator_->GenerateBrewery(location, region_context);
const std::string postal_code = return BreweryRecord{.location = location, .brewery = brewery};
postal_code_service_->GeneratePostalCode(location);
return BreweryRecord{
.address =
BreweryAddress{.city = location, .postal_code = postal_code},
.brewery = brewery};
} catch (const std::exception& e) { } catch (const std::exception& e) {
++skipped_count; ++skipped_count;
@@ -52,13 +47,13 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
exporter_->ProcessRecord(record); exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) { } catch (const std::exception& export_exception) {
++export_failed_count; ++export_failed_count;
logger_->Log({.level = LogLevel::Warn, logger_->Log(
.phase = PipelinePhase::BreweryAndBeerGeneration, {.level = LogLevel::Warn,
.message = std::format( .phase = PipelinePhase::BreweryAndBeerGeneration,
"[Pipeline] Generated brewery for '{}' ({}) " .message = std::format("[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}", "but SQLite export failed: {}",
record.address.city.city, record.address.city.country, record.location.city, record.location.country,
export_exception.what())}); export_exception.what())});
} }
}; };

View File

@@ -137,7 +137,7 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
generator_->GenerateUser(city, persona, sampled_name); generator_->GenerateUser(city, persona, sampled_name);
return UserRecord{ return UserRecord{
.address = UserAddress{.city = city.location}, .location = city.location,
.user = user, .user = user,
.email = BuildEmail(sampled_name, used_email_local_parts), .email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng), .date_of_birth = GenerateDateOfBirth(rng),
@@ -161,13 +161,13 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
} catch (const std::exception& export_exception) { } catch (const std::exception& export_exception) {
++export_failed_count; ++export_failed_count;
logger_->Log({.level = LogLevel::Warn, logger_->Log(
.phase = PipelinePhase::UserGeneration, {.level = LogLevel::Warn,
.message = std::format( .phase = PipelinePhase::UserGeneration,
"[Pipeline] Generated user for '{}' ({}) but " .message = std::format("[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}", "SQLite export failed: {}",
record.address.city.city, record.address.city.country, record.location.city, record.location.country,
export_exception.what())}); export_exception.what())});
} }
}; };

View File

@@ -14,19 +14,19 @@
void BiergartenPipelineOrchestrator::LogResults() const { void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array brewery_output; boost::json::array brewery_output;
for (const auto& [address, brewery] : generated_breweries_) { for (const auto& [location, brewery] : generated_breweries_) {
const City& location = address.city;
brewery_output.push_back(boost::json::object{ brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en}, {"name_en", brewery.name_en},
{"description_en", brewery.description_en}, {"description_en", brewery.description_en},
{"name_local", brewery.name_local}, {"name_local", brewery.name_local},
{"description_local", brewery.description_local}, {"description_local", brewery.description_local},
{"postal_code", address.postal_code},
{"location", boost::json::object{ {"location", boost::json::object{
{"city", location.city}, {"city", location.city},
{"country", location.country}, {"country", location.country},
{"state_province", location.state_province}, {"state_province", location.state_province},
{"iso3166_2", location.iso3166_2}, {"iso3166_2", location.iso3166_2},
{"latitude", location.latitude},
{"longitude", location.longitude},
}}}); }}});
} }

View File

@@ -14,14 +14,14 @@
#include "services/curated_data/curated_json_data_service.h" #include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
std::vector<City> std::vector<Location>
BiergartenPipelineOrchestrator::QueryCitiesWithCountries() { BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
logger_->Log({.level = LogLevel::Info, logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="}); .message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
const std::vector<City>& all_locations = const std::vector<Location>& all_locations =
curated_data_service_->LoadCities(); curated_data_service_->LoadLocations();
const size_t sample_count = std::min( const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count), static_cast<size_t>(application_options_.pipeline.location_count),
@@ -31,7 +31,7 @@ BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>( static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count); sample_count);
std::vector<City> sampled_locations; std::vector<Location> sampled_locations;
sampled_locations.reserve(sample_count); sampled_locations.reserve(sample_count);
std::random_device random_generator; std::random_device random_generator;

View File

@@ -14,7 +14,7 @@ bool BiergartenPipelineOrchestrator::Run() {
try { try {
exporter_->Initialize(); exporter_->Initialize();
std::vector<City> cities = QueryCitiesWithCountries(); std::vector<Location> cities = QueryCitiesWithCountries();
std::vector<EnrichedCity> enriched; std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size()); enriched.reserve(cities.size());

View File

@@ -36,7 +36,7 @@ static std::string FormatLocalLanguageCodes(
static constexpr int kBreweryInitialMaxTokens = 2800; static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery( BreweryResult LlamaGenerator::GenerateBrewery(
const City& location, const std::string& region_context) { const Location& location, const std::string& region_context) {
/** /**
* Preprocess and truncate region context to manageable size * Preprocess and truncate region context to manageable size
*/ */

View File

@@ -8,14 +8,14 @@
#include "data_generation/mock_generator.h" #include "data_generation/mock_generator.h"
size_t MockGenerator::DeterministicHash(const City& location) { size_t MockGenerator::DeterministicHash(const Location& location) {
size_t seed = 0; size_t seed = 0;
boost::hash_combine(seed, location.city); boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country); boost::hash_combine(seed, location.country);
return seed; return seed;
} }
size_t MockGenerator::DeterministicHash(const City& location, size_t MockGenerator::DeterministicHash(const Location& location,
const UserPersona& persona, const UserPersona& persona,
const Name& name) { const Name& name) {
size_t seed = DeterministicHash(location); size_t seed = DeterministicHash(location);

View File

@@ -11,7 +11,7 @@
#include "data_generation/mock_generator.h" #include "data_generation/mock_generator.h"
BreweryResult MockGenerator::GenerateBrewery( BreweryResult MockGenerator::GenerateBrewery(
const City& location, const std::string& /*region_context*/) { const Location& location, const std::string& /*region_context*/) {
const size_t hash = DeterministicHash(location); const size_t hash = DeterministicHash(location);
const std::string_view adjective = const std::string_view adjective =

View File

@@ -102,14 +102,12 @@ int main(const int argc, char** argv) {
return std::make_unique<CuratedJsonDataService>( return std::make_unique<CuratedJsonDataService>(
CuratedDataFilePaths{ CuratedDataFilePaths{
.cities_path = "cities.json", .locations_path = "locations.json",
.personas_path = "personas.json", .personas_path = "personas.json",
.forenames_path = "forenames-by-country.json", .forenames_path = "forenames-by-country.json",
.surnames_path = "surnames-by-country.json", .surnames_path = "surnames-by-country.json",
}); });
}), }),
di::bind<IPostalCodeService>().to<MockPostalCodeService>(),
di::bind<IPromptFormatter>().to([options, log_producer] { di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) { if (options.generator.use_mocked) {
{ {

View File

@@ -1,6 +1,6 @@
/** /**
* @file json_handling/json_loader.cc * @file json_handling/json_loader.cc
* @brief Parses curated location JSON input into strongly typed City * @brief Parses curated location JSON input into strongly typed Location
* records with strict field validation and descriptive error reporting. * records with strict field validation and descriptive error reporting.
*/ */
@@ -109,68 +109,41 @@ std::string ReadFirstOfStringArray(const boost::json::object& object,
CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths) CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths)
: filepaths_(std::move(filepaths)) {} : filepaths_(std::move(filepaths)) {}
const CityList& CuratedJsonDataService::LoadCities() { const LocationsList& CuratedJsonDataService::LoadLocations() {
if (!cache_.locations.empty()) { if (!cache_.locations.empty()) {
return cache_.locations; return cache_.locations;
} }
const boost::json::value root = const boost::json::value root =
ParseJsonFile(filepaths_.cities_path, "cities"); ParseJsonFile(filepaths_.locations_path, "locations");
if (!root.is_array()) { if (!root.is_array()) {
throw std::runtime_error( throw std::runtime_error(
"Invalid cities JSON: root element must be an array"); "Invalid locations JSON: root element must be an array");
} }
CityList locations; LocationsList locations;
const auto& items = root.as_array(); const auto& items = root.as_array();
locations.reserve(items.size()); locations.reserve(items.size());
for (const auto& item : items) { for (const auto& item : items) {
if (!item.is_object()) { if (!item.is_object()) {
throw std::runtime_error( throw std::runtime_error(
"Invalid cities JSON: each entry must be an object"); "Invalid locations JSON: each entry must be an object");
} }
const auto& object = item.as_object(); const auto& object = item.as_object();
locations.push_back(Location{
const boost::json::value* postal_code = object.if_contains("postal_code");
if (postal_code == nullptr || !postal_code->is_object()) {
throw std::runtime_error("Missing or invalid object field: postal_code");
}
locations.push_back(City{
.city = ReadRequiredString(object, "city"), .city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"), .state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"), .iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"), .country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"), .iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"), .local_languages = ReadRequiredStringArray(object, "local_languages"),
.postal_regex = .latitude = ReadRequiredNumber(object, "latitude"),
ReadRequiredStringArray(postal_code->as_object(), "city_regex"), .longitude = ReadRequiredNumber(object, "longitude"),
.postal_code_examples =
ReadRequiredStringArray(postal_code->as_object(), "examples"),
}); });
} }
for (auto location : locations) {
std::cout << "Location: " << location.city << ", "
<< location.state_province << ", " << location.iso3166_2 << ", "
<< location.country << ", " << location.iso3166_1 << "\n";
for (const auto& regex : location.postal_regex) {
std::cout << " Postal regex: " << regex << "\n";
}
for (const auto& example : location.postal_code_examples) {
std::cout << " Postal example: " << example << "\n";
}
for (const auto& lang : location.local_languages) {
std::cout << " Local language: " << lang << "\n";
}
}
cache_.locations = std::move(locations); cache_.locations = std::move(locations);
return cache_.locations; return cache_.locations;
} }

View File

@@ -7,38 +7,38 @@
MockCuratedDataService::MockCuratedDataService() MockCuratedDataService::MockCuratedDataService()
: locations_{ : locations_{
{.city = "Portland", Location{.city = "Portland",
.state_province = "Oregon", .state_province = "Oregon",
.iso3166_2 = "US-OR", .iso3166_2 = "US-OR",
.country = "United States", .country = "United States",
.iso3166_1 = "US", .iso3166_1 = "US",
.local_languages = {"en"}, .local_languages = {"en"},
.postal_regex = {"^972[0-9]{2}(?:-[0-9]{4})?$"}, .latitude = 45.5152,
.postal_code_examples = {"97201", "97294"}}, .longitude = -122.6784},
{.city = "Munich", Location{.city = "Munich",
.state_province = "Bavaria", .state_province = "Bavaria",
.iso3166_2 = "DE-BY", .iso3166_2 = "DE-BY",
.country = "Germany", .country = "Germany",
.iso3166_1 = "DE", .iso3166_1 = "DE",
.local_languages = {"de"}, .local_languages = {"de"},
.postal_regex = {"^8[01][0-9]{3}$"}, .latitude = 48.1351,
.postal_code_examples = {"80331", "81929"}}, .longitude = 11.5820},
{.city = "Lyon", Location{.city = "Lyon",
.state_province = "Auvergne-Rhone-Alpes", .state_province = "Auvergne-Rhone-Alpes",
.iso3166_2 = "FR-ARA", .iso3166_2 = "FR-ARA",
.country = "France", .country = "France",
.iso3166_1 = "FR", .iso3166_1 = "FR",
.local_languages = {"fr"}, .local_languages = {"fr"},
.postal_regex = {"^6900[1-9]$"}, .latitude = 45.7640,
.postal_code_examples = {"69001", "69009"}}, .longitude = 4.8357},
{.city = "Brussels", Location{.city = "Brussels",
.state_province = "Brussels-Capital", .state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU", .iso3166_2 = "BE-BRU",
.country = "Belgium", .country = "Belgium",
.iso3166_1 = "BE", .iso3166_1 = "BE",
.local_languages = {"nl", "fr"}, .local_languages = {"nl", "fr"},
.postal_regex = {"^1(?:0[0-9]{2}|1[0-9]{2}|20[0-9]|210)$"}, .latitude = 50.8503,
.postal_code_examples = {"1000", "1210"}}, .longitude = 4.3517},
}, },
personas_{ personas_{
UserPersona{.name = "Hophead Explorer", UserPersona{.name = "Hophead Explorer",
@@ -83,7 +83,9 @@ MockCuratedDataService::MockCuratedDataService()
{"BE", SurnameList{"Peeters", "Janssens"}}, {"BE", SurnameList{"Peeters", "Janssens"}},
} {} } {}
const CityList& MockCuratedDataService::LoadCities() { return locations_; } const LocationsList& MockCuratedDataService::LoadLocations() {
return locations_;
}
const PersonasList& MockCuratedDataService::LoadPersonas() { return personas_; } const PersonasList& MockCuratedDataService::LoadPersonas() { return personas_; }

View File

@@ -11,7 +11,7 @@
#include "services/enrichment/wikipedia_service.h" #include "services/enrichment/wikipedia_service.h"
std::string WikipediaEnrichmentService::GetLocationContext( std::string WikipediaEnrichmentService::GetLocationContext(
const City& loc) { const Location& loc) {
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
if (!this->client_) { if (!this->client_) {
if (logger_) { if (logger_) {

View File

@@ -14,11 +14,9 @@ void SqliteExportService::Finalize() {
} }
try { try {
insert_user_address_stmt_.reset();
insert_user_stmt_.reset(); insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset(); insert_brewery_stmt_.reset();
insert_city_stmt_.reset(); insert_location_stmt_.reset();
if (transaction_open_) { if (transaction_open_) {
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction"); db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
@@ -26,7 +24,7 @@ void SqliteExportService::Finalize() {
} }
db_handle_.reset(); db_handle_.reset();
city_cache_.clear(); location_cache_.clear();
} catch (...) { } catch (...) {
RollbackAndCloseNoThrow(); RollbackAndCloseNoThrow();
throw; throw;

View File

@@ -28,39 +28,26 @@ std::filesystem::path SqliteExportService::BuildDatabasePath() const {
void SqliteExportService::InitializeSchema() const { void SqliteExportService::InitializeSchema() const {
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateCitiesTableSql, db_handle_, sqlite_export_service_internal::kCreateLocationsTableSql,
"Failed to create SQLite cities table"); "Failed to create SQLite locations table");
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql, db_handle_, sqlite_export_service_internal::kCreateBreweriesTableSql,
"Failed to create SQLite breweries table"); "Failed to create SQLite breweries table");
sqlite_export_service_internal::ExecSql( sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUsersTableSql, db_handle_, sqlite_export_service_internal::kCreateUsersTableSql,
"Failed to create SQLite users table"); "Failed to create SQLite users table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateBreweryAddressTableSql,
"Failed to create SQLite brewery addresses table");
sqlite_export_service_internal::ExecSql(
db_handle_, sqlite_export_service_internal::kCreateUserAddressTableSql,
"Failed to create SQLite user addresses table");
} }
void SqliteExportService::PrepareStatements() { void SqliteExportService::PrepareStatements() {
insert_city_stmt_ = sqlite_export_service_internal::PrepareStatement( insert_location_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertCitySql, db_handle_, sqlite_export_service_internal::kInsertLocationSql,
"Failed to prepare SQLite city insert statement"); "Failed to prepare SQLite location insert statement");
insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement( insert_brewery_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBrewerySql, db_handle_, sqlite_export_service_internal::kInsertBrewerySql,
"Failed to prepare SQLite brewery insert statement"); "Failed to prepare SQLite brewery insert statement");
insert_brewery_address_stmt_ =
sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertBreweryAddressSql,
"Failed to prepare SQLite brewery address insert statement");
insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement( insert_user_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserSql, db_handle_, sqlite_export_service_internal::kInsertUserSql,
"Failed to prepare SQLite user insert statement"); "Failed to prepare SQLite user insert statement");
insert_user_address_stmt_ = sqlite_export_service_internal::PrepareStatement(
db_handle_, sqlite_export_service_internal::kInsertUserAddressSql,
"Failed to prepare SQLite user address insert statement");
} }
void SqliteExportService::RollbackAndCloseNoThrow() noexcept { void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
@@ -73,13 +60,11 @@ void SqliteExportService::RollbackAndCloseNoThrow() noexcept {
transaction_open_ = false; transaction_open_ = false;
} }
insert_user_address_stmt_.reset();
insert_user_stmt_.reset(); insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset(); insert_brewery_stmt_.reset();
insert_city_stmt_.reset(); insert_location_stmt_.reset();
db_handle_.reset(); db_handle_.reset();
city_cache_.clear(); location_cache_.clear();
} }
void SqliteExportService::Initialize() { void SqliteExportService::Initialize() {

View File

@@ -1,9 +1,10 @@
/** /**
* @file services/sqlite/process_record.cc * @file services/sqlite/process_record.cc
* @brief SqliteExportService::ProcessRecord() implementation * @brief SqliteExportService::ProcessRecord() implementation
* and the shared city-resolution helper. * and the shared location-resolution helper.
*/ */
#include <iomanip>
#include <sstream> #include <sstream>
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
@@ -11,74 +12,94 @@
#include "services/database/sqlite_export_service.h" #include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h" #include "services/database/sqlite_export_service_helpers.h"
std::string SqliteExportService::BuildCityKey(const City& location) { constexpr int kLocationPrecision = 17;
std::string SqliteExportService::BuildLocationKey(const Location& location) {
std::ostringstream key_stream; std::ostringstream key_stream;
key_stream << location.city << '\n' key_stream << location.city << '\n'
<< location.state_province << '\n' << location.state_province << '\n'
<< location.iso3166_2 << '\n' << location.iso3166_2 << '\n'
<< location.country << '\n' << location.country << '\n'
<< location.iso3166_1 << '\n' << location.iso3166_1 << '\n'
<< std::setprecision(kLocationPrecision) << location.latitude
<< '\n'
<< std::setprecision(kLocationPrecision) << location.longitude
<< '\n'
<< sqlite_export_service_internal::SerializeVector( << sqlite_export_service_internal::SerializeVector(
location.local_languages); location.local_languages);
return key_stream.str(); return key_stream.str();
} }
sqlite3_int64 SqliteExportService::ResolveCityId(const City& location) { sqlite3_int64 SqliteExportService::ResolveLocationId(const Location& location) {
const std::string city_key = BuildCityKey(location); const std::string location_key = BuildLocationKey(location);
const auto cached_city = city_cache_.find(city_key); const auto cached_location = location_cache_.find(location_key);
if (cached_city != city_cache_.end()) { if (cached_location != location_cache_.end()) {
return cached_city->second; return cached_location->second;
} }
const std::string local_languages_json = const std::string local_languages_json =
sqlite_export_service_internal::SerializeVector(location.local_languages); sqlite_export_service_internal::SerializeVector(location.local_languages);
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_city_stmt_, insert_location_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityNameBindIndex, .index = sqlite_export_service_internal::kLocationCityBindIndex,
.value = location.city, .value = location.city,
.action = "Failed to bind SQLite city name"}); .action = "Failed to bind SQLite location city"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_city_stmt_, insert_location_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityStateProvinceBindIndex, .index =
sqlite_export_service_internal::kLocationStateProvinceBindIndex,
.value = location.state_province, .value = location.state_province,
.action = "Failed to bind SQLite city state/province"}); .action = "Failed to bind SQLite location state/province"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_city_stmt_, insert_location_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityIso31662BindIndex, .index = sqlite_export_service_internal::kLocationIso31662BindIndex,
.value = location.iso3166_2, .value = location.iso3166_2,
.action = "Failed to bind SQLite city ISO 3166-2 code"}); .action = "Failed to bind SQLite location ISO 3166-2 code"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_city_stmt_, insert_location_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityCountryBindIndex, .index = sqlite_export_service_internal::kLocationCountryBindIndex,
.value = location.country, .value = location.country,
.action = "Failed to bind SQLite city country"}); .action = "Failed to bind SQLite location country"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_city_stmt_, insert_location_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityIso31661BindIndex, .index = sqlite_export_service_internal::kLocationIso31661BindIndex,
.value = location.iso3166_1, .value = location.iso3166_1,
.action = "Failed to bind SQLite city ISO 3166-1 code"}); .action = "Failed to bind SQLite location ISO 3166-1 code"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_city_stmt_, insert_location_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::kCityLanguagesBindIndex, .index = sqlite_export_service_internal::kLocationLanguagesBindIndex,
.value = local_languages_json, .value = local_languages_json,
.action = "Failed to bind SQLite city languages"}); .action = "Failed to bind SQLite location languages"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLatitudeBindIndex,
.value = location.latitude,
.action = "Failed to bind SQLite location latitude"});
sqlite_export_service_internal::Bind(
insert_location_stmt_,
sqlite_export_service_internal::BoundParam{
.index = sqlite_export_service_internal::kLocationLongitudeBindIndex,
.value = location.longitude,
.action = "Failed to bind SQLite location longitude"});
sqlite_export_service_internal::StepStatement( sqlite_export_service_internal::StepStatement(
db_handle_, insert_city_stmt_, "Failed to insert SQLite city row"); db_handle_, insert_location_stmt_,
"Failed to insert SQLite location row");
const sqlite3_int64 city_id = const sqlite3_int64 location_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_); sqlite_export_service_internal::LastInsertRowId(db_handle_);
city_cache_.emplace(city_key, city_id); location_cache_.emplace(location_key, location_id);
sqlite_export_service_internal::ResetStatement(insert_city_stmt_); sqlite_export_service_internal::ResetStatement(insert_location_stmt_);
return city_id; return location_id;
} }
uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) { uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
@@ -86,14 +107,14 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
throw std::runtime_error("SQLite export service is not initialized"); throw std::runtime_error("SQLite export service is not initialized");
} }
const sqlite3_int64 city_id = ResolveCityId(brewery.address.city); const sqlite3_int64 location_id = ResolveLocationId(brewery.location);
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_brewery_stmt_, insert_brewery_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{ sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kBreweryCityIdBindIndex, .index = sqlite_export_service_internal::kBreweryLocationIdBindIndex,
.value = city_id, .value = location_id,
.action = "Failed to bind SQLite brewery city id"}); .action = "Failed to bind SQLite brewery location id"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_brewery_stmt_, insert_brewery_stmt_,
@@ -128,37 +149,7 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
sqlite_export_service_internal::StepStatement( sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row"); db_handle_, insert_brewery_stmt_, "Failed to insert SQLite brewery row");
const sqlite3_int64 brewery_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_); sqlite_export_service_internal::ResetStatement(insert_brewery_stmt_);
sqlite_export_service_internal::Bind( return sqlite_export_service_internal::LastInsertRowId(db_handle_);
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressBreweryIdBindIndex,
.value = brewery_id,
.action = "Failed to bind SQLite brewery address brewery id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index =
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_brewery_address_stmt_,
"Failed to insert SQLite brewery address row");
sqlite_export_service_internal::ResetStatement(insert_brewery_address_stmt_);
return brewery_id;
} }

View File

@@ -13,8 +13,14 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
throw std::runtime_error("SQLite export service is not initialized"); throw std::runtime_error("SQLite export service is not initialized");
} }
const sqlite3_int64 city_id = ResolveCityId(user.address.city); const sqlite3_int64 location_id = ResolveLocationId(user.location);
sqlite_export_service_internal::Bind(
insert_user_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserLocationIdBindIndex,
.value = location_id,
.action = "Failed to bind SQLite user location id"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_user_stmt_, insert_user_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<std::string_view>{
@@ -67,28 +73,7 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
sqlite_export_service_internal::StepStatement( sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_stmt_, "Failed to insert SQLite user row"); db_handle_, insert_user_stmt_, "Failed to insert SQLite user row");
const sqlite3_int64 user_id =
sqlite_export_service_internal::LastInsertRowId(db_handle_);
sqlite_export_service_internal::ResetStatement(insert_user_stmt_); sqlite_export_service_internal::ResetStatement(insert_user_stmt_);
sqlite_export_service_internal::Bind( return sqlite_export_service_internal::LastInsertRowId(db_handle_);
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressUserIdBindIndex,
.value = user_id,
.action = "Failed to bind SQLite user address user id"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<sqlite3_int64>{
.index = sqlite_export_service_internal::kUserAddressCityIdBindIndex,
.value = city_id,
.action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_,
"Failed to insert SQLite user address row");
sqlite_export_service_internal::ResetStatement(insert_user_address_stmt_);
return user_id;
} }