Compare commits

63 Commits

Author SHA1 Message Date
Aaron Po
5adcb55664 Update viewer 2026-07-23 07:55:16 -04:00
Aaron Po
8f7d4dfbd7 Create biergarten_seed_2026-07-20T05-46-02.993511Z.sqlite 2026-07-20 05:27:20 -04:00
Aaron Po
7e0f1dcb6d Add sqlite load into Database.Seed 2026-07-20 02:32:20 -04:00
Aaron Po
3154c2feed Formatted src/web_client 2026-07-19 06:50:56 -04:00
Aaron Po
3df657dc54 Formatted src/services/xeger 2026-07-19 06:50:56 -04:00
Aaron Po
a238db1d37 Formatted src/services/sqlite/helpers 2026-07-19 06:50:56 -04:00
Aaron Po
5d48d6757a Formatted src/services/sqlite 2026-07-19 06:50:56 -04:00
Aaron Po
34a036d7b3 Formatted src/services/postal_code 2026-07-19 06:50:56 -04:00
Aaron Po
59053fcddd Formatted src/services/logging 2026-07-19 06:50:56 -04:00
Aaron Po
d4c7260127 Formatted src/services/enrichment/wikipedia 2026-07-19 06:50:56 -04:00
Aaron Po
be90522eae Formatted src/services/curated_data 2026-07-19 06:50:56 -04:00
Aaron Po
d055a634c2 Formatted src/services 2026-07-19 06:50:56 -04:00
Aaron Po
933b728135 Formatted src/data_generation/prompt_formatting 2026-07-19 06:50:56 -04:00
Aaron Po
ac98a2a22e Formatted src/data_generation/mock 2026-07-19 06:50:56 -04:00
Aaron Po
d723f53cba Formatted src/data_generation/llama 2026-07-19 06:50:56 -04:00
Aaron Po
1885d9859f Formatted src/biergarten_pipeline_orchestrator 2026-07-19 06:50:56 -04:00
Aaron Po
03467cbb65 Formatted src/application_options 2026-07-19 06:50:56 -04:00
Aaron Po
1f25d25dce Formatted src 2026-07-19 06:50:56 -04:00
Aaron Po
67bc5bfc2a Formatted includes/web_client 2026-07-19 06:50:56 -04:00
Aaron Po
eb32a55831 Formatted includes/services/xeger 2026-07-19 06:50:56 -04:00
Aaron Po
fad1611685 Formatted includes/services/prompting 2026-07-19 06:50:56 -04:00
Aaron Po
74f848d015 Formatted includes/services/postal_code 2026-07-19 06:50:56 -04:00
Aaron Po
86db1c2827 Formatted includes/services/logging 2026-07-19 06:50:56 -04:00
Aaron Po
9bcf4c6879 Formatted includes/services/enrichment 2026-07-19 06:50:56 -04:00
Aaron Po
d9b10df436 Formatted includes/services/datetime 2026-07-19 06:50:56 -04:00
Aaron Po
4f0f56d7d7 Formatted includes/services/database 2026-07-19 06:50:56 -04:00
Aaron Po
6c07edf845 Formatted includes/services/curated_data 2026-07-19 06:50:56 -04:00
Aaron Po
969125e3a5 Formatted includes/json_handling 2026-07-19 06:50:56 -04:00
Aaron Po
ffabd85b6d Formatted includes/data_model 2026-07-19 06:50:56 -04:00
Aaron Po
4fae5ead08 Formatted includes/data_generation/prompt_formatting 2026-07-19 06:50:56 -04:00
Aaron Po
efd0d4f2aa Formatted includes/data_generation 2026-07-19 06:50:56 -04:00
Aaron Po
522381357a Formatted includes/concurrency 2026-07-19 06:50:56 -04:00
Aaron Po
5f3706bb93 Formatted includes 2026-07-19 06:50:56 -04:00
Aaron Po
a52b14c49a update clang format 2026-07-19 06:50:56 -04:00
Aaron Po
4ebd28c06b format docs 2026-07-19 06:50:56 -04:00
Aaron Po
9593a07c13 Format curated data and prompts 2026-07-19 06:50:56 -04:00
Aaron Po
f020671703 Pipeline: City rename, add address/postal-code generation, add xeger generation (#232) 2026-07-19 03:59:06 -04:00
Aaron Po
fbcf438381 Fix mock-only builds (#231) 2026-07-03 02:47:54 -04:00
2b8a900d12 Pipeline: Add LLM and mocked user generation to the pipeline (#230) 2026-07-01 07:47:24 -04:00
Aaron Po
880f73e004 update pipeline docs 2026-06-21 21:58:03 -04:00
Aaron Po
3711591db1 commit csharpier config update 2026-06-20 15:54:53 -04:00
Aaron Po
194509f9c9 update mailkit 2026-06-20 15:54:53 -04:00
Aaron Po
67d6350425 Format ./web/backend/Shared/Shared.Contracts 2026-06-20 15:54:53 -04:00
Aaron Po
cee2917556 Format ./web/backend/Shared/Shared.Application 2026-06-20 15:54:53 -04:00
Aaron Po
6f8e5dc722 Format ./web/backend/API/API.Specs 2026-06-20 15:54:53 -04:00
Aaron Po
db3ed3581d Format ./web/backend/API/API.Core 2026-06-20 15:54:53 -04:00
Aaron Po
b54f70ccbe Format ./web/backend/Domain/Domain.Exceptions 2026-06-20 15:54:53 -04:00
Aaron Po
d4734ae9e7 Format ./web/backend/Domain/Domain.Entities 2026-06-20 15:54:53 -04:00
Aaron Po
27f28d5207 Format ./web/backend/Database/Database.Seed 2026-06-20 15:54:53 -04:00
Aaron Po
bf0d29fa54 Format ./web/backend/Database/Database.Migrations 2026-06-20 15:54:53 -04:00
Aaron Po
4772f59a68 Format ./web/backend/Infrastructure/Infrastructure.Sql 2026-06-20 15:54:53 -04:00
Aaron Po
2e5d4923c0 Format ./web/backend/Infrastructure/Infrastructure.PasswordHashing 2026-06-20 15:54:53 -04:00
Aaron Po
62766b7638 Format ./web/backend/Infrastructure/Infrastructure.Jwt 2026-06-20 15:54:53 -04:00
Aaron Po
744d4c7f8c Format ./web/backend/Infrastructure/Infrastructure.Email.Templates 2026-06-20 15:54:53 -04:00
Aaron Po
2f28ac9350 Format ./web/backend/Infrastructure/Infrastructure.Email 2026-06-20 15:54:53 -04:00
Aaron Po
c8bddae817 Format ./web/backend/Features/Features.UserManagement.Tests 2026-06-20 15:54:53 -04:00
Aaron Po
b298bd02d3 Format ./web/backend/Features/Features.UserManagement 2026-06-20 15:54:53 -04:00
Aaron Po
e2e5eb89e3 Format ./web/backend/Features/Features.Emails.Tests 2026-06-20 15:54:53 -04:00
Aaron Po
3dd44fca5a Format ./web/backend/Features/Features.Emails 2026-06-20 15:54:53 -04:00
Aaron Po
0c55d6c83f Format ./web/backend/Features/Features.Breweries.Tests 2026-06-20 15:54:53 -04:00
Aaron Po
58cbc5b5ba Format ./web/backend/Features/Features.Breweries 2026-06-20 15:54:53 -04:00
Aaron Po
5bef24696b Format ./web/backend/Features/Features.Auth.Tests 2026-06-20 15:54:53 -04:00
Aaron Po
79ae18b3e2 Format ./web/backend/Features/Features.Auth 2026-06-20 15:54:53 -04:00
257 changed files with 44558 additions and 5153 deletions

View File

@@ -13,10 +13,12 @@ low-resource language failures.
- [Model Bias and Language Quality](#model-bias-and-language-quality)
- [Western and Eurocentric Lens](#western-and-eurocentric-lens)
- [Wikipedia Enrichment](#wikipedia-enrichment)
- [Names-by-Country Dataset](#names-by-country-dataset)
- [The "Avoid AI Phrases" Prompt Instruction](#the-avoid-ai-phrases-prompt-instruction)
- [Known Issues](#known-issues)
- [Hallucinated Brewing Techniques](#hallucinated-brewing-techniques)
- [Low-Resource Language Hallucination](#low-resource-language-hallucination)
- [Synthetic Postal Codes](#synthetic-postal-codes)
---
@@ -91,6 +93,31 @@ generated descriptions.
---
## Names-by-Country Dataset
`tooling/pipeline/forenames-by-country.json` and `surnames-by-country.json`
(used to sample a `Name` per ISO 3166-1 country code for user generation) are
vendored verbatim, unmodified, from
[sigpwned/popular-names-by-country-dataset](https://github.com/sigpwned/popular-names-by-country-dataset)
(the `common-forenames-by-country.json` / `common-surnames-by-country.json`
release assets), released under **CC0** (public domain). That dataset's own
forename/surname lists are pulled from Wikipedia's "Lists of most common
surnames" and "List of most popular given names" as of the week of 2023-07-08 —
see that project's README for full provenance. Names are not LLM-generated; this
is curated fixture data per ROADMAP.md §2. Per-forename gender from the source
data is preserved through to the sampled `Name` (rather than discarded during
loading) so it's available for gender-aware persona/bio generation later.
The full multinational dataset is kept as-is (106 countries for forenames, 75
for surnames) rather than trimmed to `locations.json`'s current country list, so
it doesn't need re-sourcing if more countries are added later. `SampleName()` (a
free helper in `generate_users.cc`) returns no result for a country present in
neither file; of the countries in `locations.json`, that's currently `KE`, `SE`,
`SG`, `TH`, `VN`, and `ZA` — `GenerateUsers` skips cities in those countries the
same way brewery generation skips cities whose enrichment lookup fails.
---
## The "Avoid AI Phrases" Prompt Instruction
The system prompt instructs the model to avoid common AI-generated phrasing
@@ -244,8 +271,7 @@ For languages such as Welsh (Wales), Māori (Aotearoa/New Zealand), or Sicilian
but is semantically incoherent. This comes from limited training-data coverage
rather than prompt engineering.
Output sample:
[./out-sample/french-cities.example](out-sample/french-cities.example)
Output sample: [./french-cities.example](french-cities.example)
#### Proposed Mitigations
@@ -258,6 +284,21 @@ Output sample:
SQLite schema so downstream applications can filter or flag potentially
hallucinated text by language tier.
### Synthetic Postal Codes
When the pipeline is run without `--mocked`, `XegerPostalCodeService` generates
a fresh postal code per brewery/user address by reverse-matching one of the
city's curated postal-code regexes from `locations.json` (falling back to the
country format regex), rather than returning one of `MockPostalCodeService`'s
curated real-world examples.
The generated code is **format-conformant by construction** — it always
satisfies the regex it was generated from — but it is not looked up from any
real postal registry. It does not correspond to a real address, business, or
resident, and should not be treated as accurate geocoding or delivery data. This
is consistent with the rest of the dataset: fixture data for a proof-of-concept,
not a source of truth.
---
## Footnotes

View File

@@ -2,7 +2,8 @@
A C++20 command-line pipeline that samples city records from local JSON,
enriches each with Wikipedia context, and generates bilingual brewery names and
descriptions via a local GGUF model or a deterministic mock.
descriptions plus locale-grounded user profiles via a local GGUF model or a
deterministic mock.
> **This pipeline produces AI-generated data.** It is not a source of truth for
> brewing techniques, cultural representation, or local-language accuracy. See
@@ -39,12 +40,13 @@ The pipeline is a data ingestion layer. It sits outside the web app runtime and
produces seed records the app imports at startup or during a dedicated seed
step.
| Planned app area | Pipeline contribution |
| -------------------------------- | ------------------------------------------------------------------ |
| Brewery discovery and management | Sampled city records, localized names, long-form descriptions |
| Beer reviews and ratings | Stable brewery fixtures with enough context to anchor review pages |
| Social follow relationships | Repeatable brewery entities for feeds, follows, and saved lists |
| Geospatial brewery experiences | Latitude, longitude, and country-level metadata |
| Planned app area | Pipeline contribution |
| -------------------------------- | ------------------------------------------------------------------------------------------------- |
| Brewery discovery and management | Sampled city records, localized names, long-form descriptions |
| Beer reviews and ratings | Stable brewery fixtures with enough context to anchor review pages |
| Social follow relationships | Repeatable brewery entities for feeds, follows, and saved lists |
| Geospatial brewery experiences | Latitude, longitude, and country-level metadata |
| User accounts and profiles | Locale-grounded names, bios, and an auth-ready email/date-of-birth pair for seeding real accounts |
---
@@ -63,10 +65,9 @@ cmake --build build
CMake automatically detects whether a compatible llama.cpp installation is
present on the system (`libllama`, `libggml`, `libggml-base`, and `llama.h`
visible on the default search paths). If found, it links against those
libraries and skips the FetchContent build. If not found, it fetches and builds
llama.cpp from source at tag `b9012`. No additional flags are required in
either case.
visible on the default search paths). If found, it links against those libraries
and skips the FetchContent build. If not found, it fetches and builds llama.cpp
from source at tag `b9012`. No additional flags are required in either case.
Metal is enabled automatically on Apple Silicon. CUDA or HIP/ROCm is detected
automatically on Linux when the relevant toolkit is present.
@@ -94,24 +95,27 @@ Each run writes a fresh dated SQLite file such as
./biergarten-pipeline \
--model ../models/google_gemma-4-E4B-it-Q6_K.gguf \
--prompt-dir prompts \
--location-count 25 \
--temperature 1.0 --top-p 0.95 --top-k 64 --n-ctx 8192 --seed -1
```
#### CLI Flags
| Flag | Purpose |
| --------------- | ---------------------------------------------------------------------------------------------------- |
| `--mocked` | Deterministic mock generator, no model required. |
| `--model, -m` | Path to a GGUF file. Required unless `--mocked` is set. |
| `--prompt-dir` | Directory containing prompt files (e.g. `BREWERY_GENERATION.md`). Required unless `--mocked` is set. |
| `--output, -o` | Directory for generated SQLite artifacts. Default: `output`. |
| `--log-path` | Path for application logs. Default: `pipeline.log`. |
| `--temperature` | Sampling temperature. Default: `1.0`. |
| `--top-p` | Nucleus sampling. Default: `0.95`. |
| `--top-k` | Top-k sampling. Default: `64`. |
| `--n-ctx` | Context window size. Default: `8192`. |
| `--seed` | Random seed. Default: `-1` (random at runtime). |
| `--help, -h` | Print usage and exit. |
| Flag | Purpose |
| ------------------ | ---------------------------------------------------------------------------------------------------- |
| `--mocked` | Deterministic mock generator, no model required. |
| `--model, -m` | Path to a GGUF file. Required unless `--mocked` is set. |
| `--prompt-dir` | Directory containing prompt files (e.g. `BREWERY_GENERATION.md`). Required unless `--mocked` is set. |
| `--output, -o` | Directory for generated SQLite artifacts. Default: `output`. |
| `--log-path` | Path for application logs. Default: `pipeline.log`. |
| `--location-count` | Number of cities to sample from `locations.json` per run. Default: `10`. |
| `--temperature` | Sampling temperature. Default: `1.0`. |
| `--top-p` | Nucleus sampling. Default: `0.95`. |
| `--top-k` | Top-k sampling. Default: `64`. |
| `--n-ctx` | Context window size. Default: `8192`. |
| `--seed` | Random seed. Default: `-1` (random at runtime). |
| `--n-gpu-layers` | Number of model layers to offload to GPU. Default: `0`. |
| `--help, -h` | Print usage and exit. |
`--mocked` and `--model` are mutually exclusive. Omitting both exits with an
error before the pipeline starts. Sampling flags are ignored when `--mocked` is
@@ -130,17 +134,21 @@ NVIDIA GPU.
### How it works
The container uses a two-stage build. The first stage pulls prebuilt
`libllama`, `libggml`, and backend plugin libraries (including `libggml-cuda.so`
and the CPU variant plugins) from `ghcr.io/ggml-org/llama.cpp:full-cuda`. The
second stage copies those libraries into `/usr/local/lib` and runs `ldconfig` so
the dynamic linker and `dlopen` calls from `ggml_backend_load_all()` can resolve
the CUDA backend plugin at runtime. llama.cpp headers are cloned at the matching
tag and installed into `/usr/local/include`. CMake auto-detects both and skips
the FetchContent source build entirely, keeping image build times short.
The container uses a two-stage build. The builder stage installs CMake/Ninja,
clones the matching llama.cpp release tag for its headers only (installed into
`/usr/local/include`), and copies prebuilt shared libraries (`libllama`,
`libggml`, and CUDA/CPU backend plugins) from
`ghcr.io/ggml-org/llama.cpp:full-cuda` into `/usr/local/lib`. With both headers
and libraries present, CMake's system-library detection (see [Build](#build)
above) finds them and skips the FetchContent source build, keeping image build
times short.
`GGML_BACKEND_PATH` is set to `/usr/local/lib` so llama.cpp knows where to scan
for backend plugins.
The runtime stage copies the compiled binary, the same prebuilt shared
libraries, and config/prompt assets into a slim CUDA runtime image. It sets
`LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH` so the dynamic linker resolves
`libllama`/`libggml` at startup, and also co-locates `libggml-cuda.so` and the
CPU backend plugins next to the binary for `ggml_backend_load_all()`'s `dlopen`
scan.
### Build the image
@@ -165,44 +173,46 @@ docker build \
Look for `[biergarten] Found system llama.cpp — skipping FetchContent` in the
output to confirm the fast path was taken.
### Run in mocked mode
### Run the container
No model or GPU required. Useful for validating the pipeline logic and SQLite
export path.
```bash
docker run --rm \
-e BIERGARTEN_MODE=mocked \
-v "$PWD/output:/workspace/output" \
-v "$PWD/logs:/workspace/logs" \
biergarten-pipeline:latest
```
### Run in live mode
Mount your GGUF model before starting. The container validates the model path
before launching the binary.
The container always runs the model-backed path; there is no `--mocked`
container mode (use a native build for that — see [Quick Start](#quick-start)).
The entrypoint, `runpod/start.sh`, downloads the GGUF model automatically if it
is not already present at the configured path.
```bash
docker run --rm \
--runtime=nvidia \
-e BIERGARTEN_MODE=live \
-e GGML_BACKEND_PATH="/usr/local/lib/libggml-cuda.so" \
-v "$PWD/models:/workspace/models" \
-v "$PWD/output:/workspace/output" \
-v "$PWD/logs:/workspace/logs" \
biergarten-pipeline:latest
```
The model must be present at `./models/google_gemma-4-E4B-it-Q6_K.gguf` on the
host. See [Model](#model) above for the download command.
By default this downloads `google_gemma-4-E4B-it-Q6_K.gguf` to `./models/` on
first run if it isn't already there. To use a pre-downloaded model, place it at
that path first — see [Model](#model) above.
#### Environment variables
| Variable | Purpose |
| ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ |
| `BIERGARTEN_MODEL_PATH` | GGUF model path. Default: `/workspace/models/google_gemma-4-E4B-it-Q6_K.gguf`. |
| `BIERGARTEN_OUTPUT_DIR` | SQLite output directory. Default: `/workspace/output`. |
| `BIERGARTEN_LOG_PATH` | Log file path. Default: `/workspace/logs/pipeline.log`. |
| `BIERGARTEN_GL_LAYERS` | GPU layers to offload (`--n-gpu-layers`). Default: `40`. |
| `BIERGARTEN_TEMPERATURE`, `BIERGARTEN_TOP_P`, `BIERGARTEN_TOP_K`, `BIERGARTEN_N_CTX`, `BIERGARTEN_SEED` | Optional sampling overrides, unset by default (binary defaults apply). |
| `BIERGARTEN_EXTRA_ARGS` | Additional raw CLI args appended verbatim. |
`--prompt-dir` is hardcoded to `/app/prompts` inside the container and is not
configurable via environment variable.
### RunPod deployment
Use a GPU pod template. Mount persistent storage for `/workspace/models`,
`/workspace/output`, and `/workspace/logs`. Set `BIERGARTEN_MODE=live` in the
template environment. See `tooling/pipeline/runpod/pod-template.yaml` for a
starter template.
`/workspace/output`, and `/workspace/logs`. See
`tooling/pipeline/runpod/pod-template.yaml` for a starter template — set the
environment variables listed above to match your run.
---
@@ -210,52 +220,76 @@ starter template.
### Pipeline Stages
| Stage | Implementation |
| -------- | --------------------------------------------------------------------------------------------------------------------------------------- |
| Load | `JsonLoader::LoadLocations()` reads `locations.json` into typed `Location` records. |
| Sample | `BiergartenDataGenerator::QueryCitiesWithCountries()` samples up to 50 locations per run. |
| Enrich | `WikipediaService` fetches city and beer context. Keeps going when a lookup fails. |
| Generate | `MockGenerator` or `LlamaGenerator` produces brewery names and descriptions in English and the local language. |
| Store | `SqliteExportService` writes each successful brewery into a fresh dated `.sqlite` database with normalized location and brewery tables. |
| Log | `spdlog` writes results and warnings to the console. |
| Stage | Implementation |
| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Load | `ICuratedDataService` (`CuratedJsonDataService`) reads `locations.json`, `personas.json`, `forenames-by-country.json`, and `surnames-by-country.json` (paths supplied via a `CuratedDataFilePaths` DTO at construction) into typed records, caching each after its first load. `--mocked` runs use `MockCuratedDataService`'s fixed in-memory dataset instead. |
| Sample | `BiergartenPipelineOrchestrator::QueryCitiesWithCountries()` samples `--location-count` locations per run (default `10`). |
| Enrich | `WikipediaEnrichmentService` fetches brewing and beer-related context. Keeps going when a lookup fails. `--mocked` runs use `MockEnrichmentService` instead and skip Wikipedia entirely. |
| Generate Users | `GenerateUsers()` samples a persona and a forename/surname pair per enriched city (skipping countries with no name data), then `MockGenerator` or `LlamaGenerator` produces a username, bio, and activity weight around the sampled name. |
| Generate Breweries | `MockGenerator` or `LlamaGenerator` produces brewery names and descriptions in English and the local language. |
| Store | `SqliteExportService` writes each successful user and brewery into a fresh dated `.sqlite` database with normalized `locations`, `users`, and `breweries` tables. |
| Log | `spdlog` writes results and warnings to the console. |
If enrichment or generation fails for a city, that city is skipped and the
pipeline continues.
If name sampling, enrichment, or generation fails for a city, that city is
skipped and the pipeline continues. `GenerateUsers()` runs before
`GenerateBreweries()` in `BiergartenPipelineOrchestrator::Run()`.
### Key Components
- `src/main.cc` — argument parsing and Boost.DI composition root.
- `JsonLoader` — validates curated location input.
- `WikipediaService` — queries Wikipedia extracts, caches results, returns empty
context on failure.
- `LlamaGenerator` — formats prompts for Gemma 4, validates JSON output, retries
malformed responses up to three times. If output looks truncated, the retry
raises the token budget before trying again.
- `MockGenerator` — stable hash-based output so the same city input always
produces the same brewery.
- `CuratedJsonDataService` — implements `ICuratedDataService`; takes a
`CuratedDataFilePaths` DTO (locations/personas/forenames/surnames paths) in
its constructor, then parses and validates curated location, persona, and
forename/surname JSON, memoizing each result after its first load on a given
instance. `MockCuratedDataService` is the in-memory substitute (4 fixed
locations, 3 personas, and name data for `US`/`DE`/`FR`/`BE`) used in
`--mocked` runs.
- `WikipediaEnrichmentService` — queries Wikipedia extracts, caches results,
returns empty context on failure. `MockEnrichmentService` is the no-op
substitute used in `--mocked` runs.
- `LlamaGenerator` — formats prompts for Gemma 4, validates JSON output for both
`GenerateBrewery` and `GenerateUser`, retries malformed responses up to three
times with corrective feedback in the retry prompt. The token budget is fixed
across attempts; it is not raised automatically on truncation.
- `MockGenerator` — stable hash-based output so the same city/persona/name input
always produces the same brewery or user.
- `SqliteExportService` — creates a dated SQLite file per run and persists each
successful brewery into normalized tables.
successful user and brewery into normalized tables.
- Brewery payloads include English and local-language name and description
fields.
fields. User payloads carry a sampled first/last name and gender, an
LLM-generated username/bio/activity weight, and a programmatically generated
(not LLM-authored) unique email and date of birth.
### Runtime Behaviour
`WikipediaService` queries city, country, and beer-related Wikipedia extracts
using its configured lookup, then caches the first successful response per query
string. The fetched extract text is included in the prompt as context for
generation.
`WikipediaEnrichmentService` fetches two Wikipedia extracts per city: a generic
"brewing" extract and a "beer in `{country}`" extract. It does not currently
query a city- or region-specific page. Each query string is cached after its
first successful (or empty) lookup.
`GetLocationContext()` returns an empty string when the web client is
unavailable or when lookup/parsing fails.
`LlamaGenerator` validates model output as structured JSON. The retry path
exists as a safety hatch for cases where the reasoning block consumes available
token budget and compresses the JSON output space. All runs to date have
produced valid output on the first pass; the path is kept for resilience.
`LlamaGenerator` validates model output as structured JSON. On validation
failure it retries up to three times, replaying the previous error message in
the next prompt so the model can self-correct. All runs to date have produced
valid output on the first pass; the retry path is kept for resilience.
`MockGenerator` uses stable hashes for repeatable output in demos and Storybook
runs.
`CuratedJsonDataService` memoizes each of `LoadLocations()`, `LoadPersonas()`,
`LoadForenamesByCountry()`, and `LoadSurnamesByCountry()` independently the
first time each is called, since `BiergartenPipelineOrchestrator` owns a single
`ICuratedDataService` instance for the whole run — later calls return the cached
result instead of re-parsing.
`GenerateUsers()` samples a forename/surname pair per city via `SampleName()`,
keyed by the city's ISO 3166-1 code. Countries present in `locations.json` but
absent from either name fixture (currently `KE`, `SE`, `SG`, `TH`, `VN`, `ZA`)
are skipped, the same way a failed enrichment or generation call skips a city —
see ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section.
### Process Flow - Activity Diagram
![An activity diagram](./diagrams/current/output/activity.svg)
@@ -268,9 +302,10 @@ runs.
## Generated Output
Each successful run stores a `GeneratedBrewery` pair with the source location
and a `BreweryResult` payload. The same generated records are also written to a
fresh SQLite export file named with the current UTC timestamp.
Each successful run stores a `BreweryRecord` pair with the source location and a
`BreweryResult` payload, and a `UserRecord` pair with the source location and a
`UserResult` payload. The same generated records are also written to a fresh
SQLite export file named with the current UTC timestamp.
| Field | Meaning |
| ------------------- | ------------------------------------------ |
@@ -279,6 +314,17 @@ fresh SQLite export file named with the current UTC timestamp.
| `name_local` | Brewery name in the local language. |
| `description_local` | Brewery description in the local language. |
| Field | Meaning |
| ----------------- | -------------------------------------------------------------------------------------- |
| `first_name` | Sampled forename, copied from the curated name data (not LLM-invented). |
| `last_name` | Sampled surname, copied from the curated name data (not LLM-invented). |
| `gender` | Gender associated with the sampled forename in the source dataset. |
| `username` | LLM-generated handle. |
| `bio` | LLM-generated short biography. |
| `activity_weight` | Relative check-in/activity weight, reserved for a future J-curve activity profile. |
| `email` | Unique `@thebiergarten.app` address, generated programmatically from the sampled name. |
| `date_of_birth` | Randomized date of birth (age 19-48), generated programmatically. |
The log dump also includes city, country, state or province, ISO subdivision
code, latitude, and longitude for each entry.
@@ -292,7 +338,6 @@ code, latitude, and longitude for each entry.
| `local_languages` | Locale-aware copy selection |
| `name_en`, `description_en` | Default English display content |
| `name_local`, `description_local` | Local-language display content |
| `region_context` | Richer copy for cards and detail pages |
---
@@ -312,11 +357,9 @@ The build fetches Boost.DI, spdlog, and SQLite via CMake. llama.cpp is fetched
only when a system installation is not detected. Metal is enabled on Apple
Silicon; CUDA or HIP/ROCm is detected on Linux when the toolkit is present.
> **Code Style:** Modern C++20 throughout — RAII for ownership,
> `std::unique_ptr` for injected dependencies, `std::optional` for parse
> outcomes, `std::span` for read-only views over generated city data, structured
> bindings in pipeline loops. Formatting follows the Google C++ Style Guide via
> `.clang-format` with a narrow column limit and two-space indentation.
> **Code Style:** Code style generally follows the Google Style Guide and
> features modern C++20 throughout. Formatting follows the Google C++ Style
> Guide, with the exception of indentation width (3 spaces vs. Google's 2).
---
@@ -358,47 +401,62 @@ Silicon; CUDA or HIP/ROCm is detected on Linux when the toolkit is present.
## Fixture Strategy
- `--mocked` for stable fixtures, repeatable screenshots, and Storybook runs.
`MockCuratedDataService` swaps in for `CuratedJsonDataService`, so no fixture
files need to be present on disk.
- `--model` when geographically grounded content matters for demos.
- Keep `locations.json` structured enough to support discovery and future
filtering.
- Treat SQLite output as seed material for the app's brewery domain, not
production data.
- `personas.json`, `forenames-by-country.json`, and `surnames-by-country.json`
are curated/vendored fixture data, not LLM-generated — see
ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section for provenance.
- Treat SQLite output as seed material for the app's brewery and user domains,
not production data.
---
## Repo Layout
| Path | Purpose |
| ---------------------------- | -------------------------------------------------- |
| `includes/` | Public headers and shared models. |
| `src/` | Implementation files. |
| `locations.json` | Curated city input copied into the build tree. |
| `prompts/` | System prompts used by the model-backed path. |
| `diagrams/` | Architecture and pipeline diagrams. |
| `tooling/pipeline/runpod/` | Dockerfile, launcher, and RunPod pod template. |
| `ETHICS-AND-KNOWN-ISSUES.md` | Ethics, bias, hallucination analysis, mitigations. |
| Path | Purpose |
| -------------------------------------------- | ----------------------------------------------------------- |
| `tooling/pipeline/includes/` | Public headers and shared models. |
| `tooling/pipeline/src/` | Implementation files. |
| `tooling/pipeline/locations.json` | Curated city input copied into the build tree. |
| `tooling/pipeline/personas.json` | Curated user persona archetypes copied into the build tree. |
| `tooling/pipeline/forenames-by-country.json` | Vendored (CC0) forename data by ISO 3166-1 country code. |
| `tooling/pipeline/surnames-by-country.json` | Vendored (CC0) surname data by ISO 3166-1 country code. |
| `tooling/pipeline/prompts/` | System prompts used by the model-backed path. |
| `tooling/pipeline/runpod/` | Dockerfile, launcher, and RunPod pod template. |
| `docs/pipeline/diagrams/` | Architecture and pipeline diagrams. |
| `docs/pipeline/ETHICS-AND-KNOWN-ISSUES.md` | Ethics, bias, hallucination analysis, mitigations. |
---
## Code Tour
Paths below are relative to `tooling/pipeline/`.
- `src/main.cc` — argument parsing and DI composition root.
- `src/biergarten_data_generator/` — orchestration, sampling, logging, and
export.
- `src/services/wikipedia/` — enrichment service and cache.
- `src/biergarten_pipeline_orchestrator/` — orchestration, sampling, logging,
and export.
- `src/services/curated_data/` — `CuratedJsonDataService`, the file-backed
`ICuratedDataService`, and `MockCuratedDataService`, the in-memory
`ICuratedDataService` used in `--mocked` runs.
- `src/services/enrichment/wikipedia/` — enrichment service and cache.
- `src/services/sqlite/` — SQLite export implementation.
- `src/data_generation/llama/` — local inference, prompt loading, output
validation.
- `src/data_generation/mock/` — deterministic fallback.
- `tooling/pipeline/runpod/` — container build and runtime launcher.
- `runpod/` — container build and runtime launcher.
---
## Next Steps
The pipeline currently produces city-aware brewery records and dated SQLite
exports. The next passes add additional fixture types so the app can exercise
the full brewery domain without live data.
The pipeline currently produces city-aware brewery and user records and dated
SQLite exports. The next passes add additional fixture types so the app can
exercise the full brewery and social domains without live data. For the detailed
engineering breakdown of what's needed to reach the architecture in
[`diagrams/planned/`](./diagrams/planned/), see [ROADMAP.md](./ROADMAP.md).
### Testing — Very High Priority
@@ -420,12 +478,6 @@ Generate catalog entries with style, ABV, IBU, color, aroma notes, and food
pairing hints. Link beers back to breweries and cities. Keep style coverage wide
enough to exercise search, sort, and category filters.
### User Generation
Generate user profiles with stable names, bios, locale hints, and preference
signals. Include stable IDs for downstream fixture joins. Keep output
deterministic for screenshots while allowing larger randomized batches.
### Check-In System
Produce timestamped check-in events between users and breweries. Use a J-curve

244
docs/pipeline/ROADMAP.md Normal file
View File

@@ -0,0 +1,244 @@
# Pipeline Roadmap — Reaching the Planned Architecture
This is the engineering breakdown for closing the gap between the **current**
pipeline and the **planned** architecture documented in
[`diagrams/planned/class.puml`](./diagrams/planned/class.puml) and
[`diagrams/planned/activity.puml`](./diagrams/planned/activity.puml). Nothing in
`diagrams/planned/` is implemented yet — this file tracks what it would take to
get there. For the current implementation, see [README.md](./README.md).
Items are grouped by layer and roughly ordered by dependency: later groups build
on types and services introduced earlier.
The only concurrency in the current pipeline is the log dispatcher thread
(`main.cc` spawns it to drain a `BoundedChannel<LogEntry>` into spdlog for the
whole run). Sampling, enrichment, generation, and export all happen
synchronously on the main thread inside `BiergartenPipelineOrchestrator::Run()`
— §6 below is what introduces the additional worker threads the planned
architecture needs.
---
## 1. Domain Models
`tooling/pipeline/includes/data_model/generated_models.h` and `models.h`.
- [ ] Add `Completeness` enum (`Full`, `Partial`, `Absent`) and a
`LocationContext` struct (`text`, `completeness`, `char_count`). Replace
`EnrichedCity::region_context` (currently a plain `std::string`) with
`context : LocationContext`.
- [ ] Add `BeerStyle` (`name`, `description`, `min_abv`, `max_abv`, `min_ibu`,
`max_ibu`).
- [ ] Add `BeerResult`, `CheckinResult`, `RatingResult` result payloads.
- [ ] Add `GenerationMetadata` (`generation_id`, `generated_time`,
`context_provided`, `generated_with`).
- [x] Add `first_name`, `last_name`, `gender`, and `activity_weight` to
`UserResult` (currently just `username`, `bio`). `first_name`/
`last_name`/`gender` are copied from the sampled `Name` (see below), not
LLM-invented.
- [x] Add `Name` (`first_name`, `last_name`, `gender`) — the sampled result
handed to `DataGenerator::GenerateUser`. Add `ForenameEntry` (`name`,
`gender`). Landed as a flatter shape than originally planned here: no
`NamesByCountry` wrapper class. `curated_data_service.h` instead declares
`ForenameList = std::vector<ForenameEntry>` and
`SurnameList = std::vector<std::string>`, and `ICuratedDataService`
exposes two maps keyed by ISO 3166-1 code —
`ForenamesByCountryMap = unordered_map<string, ForenameList>` and
`SurnamesByCountryMap = unordered_map<string, SurnameList>` — directly
(see §2). Sampling is a free function,
`SampleName(forenames_by_country, surnames_by_country, iso3166_1, rng)`,
in `generate_users.cc`'s anonymous namespace, not a method on a class.
Pairing a forename with a surname happens at sample time so gender
(present per-forename in the source data) is never discarded the way a
pre-flattened `{first_name, last_name}` list would lose it; see §2.
- [ ] Extend `GeneratedBrewery` with `brewery_id`, `context_completeness`,
`metadata` (currently just `{location, brewery}`).
- [ ] Add `GeneratedBeer`, `GeneratedCheckin`, `GeneratedRating`,
`GeneratedFollow` aggregate structs.
- [x] Add `UserRecord` (`location`, `user : UserResult`, `email`,
`date_of_birth`) as the current stand-in for the planned `GeneratedUser` —
`email` and `date_of_birth` are programmatically generated by the
orchestrator, never LLM-authored, so a downstream auth-account seeding
consumer can register real accounts from the pipeline's SQLite export. No
`password`, `user_id`, or `metadata` field yet, matching `BreweryRecord`'s
equally pre-`metadata` shape. Already wired into
`IExportService`/`SqliteExportService`: a `users` table exists and
`GenerateUsers()` exports each successful record via
`ProcessRecord(const UserRecord&)` (see §5) in addition to logging and
holding `generated_users_` in memory. Still missing vs. the planned
`GeneratedUser`: `user_id`, `password`, and `metadata`.
- [x] Add `UserPersona` (`name`, `description`, `style_affinities`).
## 2. Data Preloading
`tooling/pipeline/includes/services/curated_data/`, fixture files.
- [x] Extract an interface; have `CuratedJsonDataService` implement it. Landed
as `ICuratedDataService` (`services/curated_data/curated_data_service.h`),
not `DataPreloader` — `LoadLocations()`, `LoadPersonas()`,
`LoadForenamesByCountry()`, and `LoadSurnamesByCountry()` are all virtual
methods returning `const&` and take no arguments. `CuratedJsonDataService`
(`services/curated_data/curated_json_data_service.h`, originally landed as
`JsonLoader` under `json_handling/` before being renamed and moved) takes
a `CuratedDataFilePaths` DTO (`locations_path`, `personas_path`,
`forenames_path`, `surnames_path`) in its constructor rather than a path
per `Load*()` call, and memoizes each result in a private `cache` struct
(`locations`, `personas`, `forenames_by_country`, `surnames_by_country`),
so a second call on the same instance returns the cached result instead of
re-parsing — safe because `CuratedJsonDataService` outlives every call
site (owned by `BiergartenPipelineOrchestrator` via
`unique_ptr<ICuratedDataService>` for the whole run). `main.cc`'s DI
injector also gained a `MockCuratedDataService` (fixed in-memory data: 4
locations across `US`/`DE`/`FR`/`BE`, 3 personas, and matching
forename/surname sets for those four countries), bound in place of
`CuratedJsonDataService` under `--mocked`, mirroring the existing
`MockEnrichmentService`/`MockGenerator` pattern.
- [ ] Add `LoadBeerStyles()`. `beer-styles.json` already exists in the repo and
is already copied into the Docker image (`runpod/Dockerfile`), but no
loader reads it yet, and the native CMake build doesn't copy it into
`build/` at all — `CMakeLists.txt`'s "Runtime Assets" step only copies
`locations.json` and `prompts/`.
- [x] Add `LoadPersonas()` and author `personas.json` (doesn't exist yet).
- [x] Add name-by-country loading, parsing
`tooling/pipeline/forenames-by-country.json` and
`surnames-by-country.json`. Landed as two separate methods —
`LoadForenamesByCountry()` and `LoadSurnamesByCountry()` — each returning
a flat `ForenamesByCountryMap` / `SurnamesByCountryMap` (aliases for
`unordered_map<string, ForenameList>` /
`unordered_map<string, SurnameList>`) keyed by ISO 3166-1 code, rather
than one combined `LoadNamesByCountry() : NamesByCountry` call. Both files
are vendored verbatim (unmodified, full multinational coverage) from
`sigpwned/popular-names-by-country-dataset` (CC0) — see
ETHICS-AND-KNOWN-ISSUES.md's Names-by-Country Dataset section for
provenance. Deliberately not pre-paired or filtered down to just the
countries in `locations.json`: keeping the source shape (including
per-forename gender) intact means the loader can support more countries
later, or gender-aware persona/bio generation, without re-sourcing data.
## 3. Policy / Strategy Layer
Entirely new — no `includes/policy/` (or equivalent) directory exists today.
- [ ] `ContextStrategy` interface + `BreweryContextStrategy` /
`BeerContextStrategy`. Today
`WikipediaEnrichmentService::GetLocationContext` hardcodes a generic
`"brewing"` query and a `"beer in {country}"` query directly — no
per-phase strategy selection.
- [ ] `SamplingStrategy` interface + `UniformSamplingStrategy`, replacing the
inline `std::ranges::sample(...)` call in
`BiergartenPipelineOrchestrator::QueryCitiesWithCountries()`.
- [ ] `BeerSelectionStrategy` interface + `RandomBeerSelectionStrategy`, to pick
styles per brewery from the `BeerStyle` palette (depends on §1 and §2).
- [ ] `CheckinDistributionStrategy` interface + `JCurveCheckinStrategy` /
`RandomCheckinStrategy` — this is the "Check-In System" item already
called out in README.md's Next Steps, made concrete.
- [ ] `FollowGenerationStrategy` interface + `RandomFollowStrategy` /
`ActivityWeightedFollowStrategy`.
## 4. Data Generation
`tooling/pipeline/includes/data_generation/`, `src/data_generation/`.
- [ ] Extend the `DataGenerator` interface with `GenerateBeer`,
`GenerateCheckin`, `GenerateRating` (today: only `GenerateBrewery` and
`GenerateUser`).
- [x] Implement `LlamaGenerator::GenerateUser` for real, with a retry loop
mirroring `GenerateBrewery` (GBNF grammar, `ValidateUserJson`, up to 3
attempts with corrective feedback) and a new `prompts/USER_GENERATION.md`.
Changed the `DataGenerator::GenerateUser` signature to
`(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult`,
matching what the activity diagram actually passes.
`MockGenerator::GenerateUser` updated to match the new signature too.
- [ ] Implement `MockGenerator::GenerateBeer` / `GenerateCheckin` /
`GenerateRating`.
- [ ] Add `IPromptFormatter::ExpectedArchitecture()` and
`LlamaGenerator::ValidateModelArchitecture()` so loading a GGUF that
doesn't match the configured chat template fails fast instead of silently
producing degraded output.
- [ ] Add prompt template files for beer/checkin/rating generation next to the
existing `prompts/BREWERY_GENERATION.md`.
## 5. Export Service
`tooling/pipeline/includes/services/database/`, `src/services/sqlite/`.
- [x] `IExportService::ProcessRecord(const UserRecord&)` and
`SqliteExportService`'s `users` table (see `kCreateUsersTableSql` /
`kInsertUserSql` in
`includes/services/database/sqlite_statement_helpers.h`) are implemented —
`GenerateUsers()` exports every successful user alongside its resolved
`location_id`.
- [ ] Extend `IExportService` with `ProcessBeer`, `ProcessCheckin`,
`ProcessRating`, `ProcessFollow` (today
`ProcessRecord(const BreweryRecord&)` and
`ProcessRecord(const UserRecord&)` exist).
- [ ] Add `beers`, `checkins`, `ratings`, `follows` tables to
`SqliteExportService::InitializeSchema()` (`locations`, `breweries`, and
`users` already exist) — see `kCreateLocationsTableSql` /
`kCreateBreweriesTableSql` / `kCreateUsersTableSql` in
`includes/services/database/sqlite_statement_helpers.h` for the existing
pattern to follow.
- [ ] Add a `brewery_cache_` alongside the existing `location_cache_`, and move
from one open transaction for the whole run to per-phase batched commits
(`BEGIN` / `COMMIT & BEGIN` on a batch-size threshold), as shown in the
planned activity diagram.
## 6. Concurrency & Orchestration
`tooling/pipeline/includes/concurrency/`,
`src/biergarten_pipeline_orchestrator/`.
- [ ] `BoundedChannel<T>` already exists and is production-tested — but it's
only wired to the single log channel today. Stand up the per-phase
producer/consumer channels (`loc_ch`, `exp_ch`, etc.) the planned activity
diagram describes, each with a dedicated LLM worker thread and SQLite
worker thread.
- [ ] Rewrite `BiergartenPipelineOrchestrator::Run()` from one synchronous pass
over `generated_breweries_` into phased methods — `RunUserPhase` →
`RunBreweryAndBeerPhase` (with a `RunBeerPhase` sub-step once
`brewery_pool_` is populated) → `RunCheckinPhase` / `RunFollowPhase`
(these two can run in parallel) → `RunRatingPhase` — backed by
`user_pool_`, `brewery_pool_`, `beer_pool_`, `checkin_pool_`,
`follow_pool_`.
- [ ] Honor the structural-concurrency requirement already called out in a
comment on `BiergartenPipelineOrchestrator::Run()` in
`biergarten_pipeline_orchestrator.h`: once real worker threads exist, they
must be structurally joined (e.g. via `std::jthread`) before `Run()`
returns, so no worker logs to a closed channel during teardown.
## 7. Enrichment
- [ ] Decide whether to restore the city/region-specific Wikipedia query that's
currently commented out in
`src/services/enrichment/wikipedia/get_summary.cc` (`GetLocationContext`).
The `ContextStrategy` work in §3 is a natural place to reintroduce it via
`BreweryContextStrategy::QueriesFor()`.
- [ ] Pre-warm caches at startup (`PreWarmBeerStyleCache`,
`PreWarmLocationCache` in the planned activity diagram) instead of
fetching lazily per record, so the streaming phases never block on a cold
cache mid-run.
## 8. Fixtures / Build
- [x] Author `personas.json`, `forenames-by-country.json`, and
`surnames-by-country.json` (see §2).
- [ ] Fix the `beer-styles.json` build-tree gap: add it to the "Runtime Assets"
`configure_file` step in `CMakeLists.txt` so native builds and the Docker
image agree on what's available at runtime.
---
## Suggested build order
The planned activity diagram's own phase notes ("brewery*pool* is now fully
populated. Phase 1b may begin.", etc.) imply a dependency order. Roughly:
1. Domain models (§1) and data preloading (§2) — nothing else compiles without
these.
2. Export schema (§5) — so every later generation phase has somewhere to land
its output.
3. Policy/strategy layer (§3) and generator interface (§4).
4. Concurrency/orchestration rewrite (§6), which is the only piece that actually
wires §1–§5 into the phased, parallel pipeline shown in the planned diagrams.
5. Enrichment cache pre-warming and the city/region query decision (§7) — useful
at any point, but most valuable once phases run concurrently.

View File

@@ -18,23 +18,40 @@ skinparam ActivityBarColor #628A5B
skinparam SwimlaneBorderColor #547461
skinparam SwimlaneBorderThickness 0.3
title The Biergarten Data Pipeline (Streaming Architecture)
title The Biergarten Data Pipeline (Current — Synchronous Data Path)
|#F2F6F0|main.cc|
start
:Create BoundedChannel<LogEntry> log_channel;
:Spawn log dispatcher thread\n(LogDispatcher::Run());
note right
The only concurrency in the current pipeline:
a dedicated thread drains log_channel and
forwards entries to spdlog for the entire run.
Joined during shutdown after log_channel closes.
end note
:ParseArguments(argc, argv);
if (Are arguments valid?) then (no)
:spdlog::error usage info;
:Log error / usage info;
stop
else (yes)
endif
:Init OpenSSL global state & LlamaBackendState;
:di::make_injector(...);
:injector.create<std::unique_ptr<BiergartenDataGenerator>>();
:BiergartenDataGenerator::Run();
note right
Binds ILogger, IEnrichmentService, DataGenerator,
IExportService, IPromptFormatter, WebClient based
on --mocked vs. model-backed mode.
end note
:injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
:BiergartenPipelineOrchestrator::Run();
note right
Run() itself is synchronous — sampling, enrichment,
generation, and export all happen on this thread.
end note
|#EAF0E8|BiergartenDataGenerator|
|#EAF0E8|BiergartenPipelineOrchestrator|
:Initialize SQLite export;
|#E0EAE0|SqliteExportService|
@@ -48,23 +65,106 @@ note right
Begins Transaction
end note
|#EAF0E8|BiergartenDataGenerator|
|#EAF0E8|BiergartenPipelineOrchestrator|
:QueryCitiesWithCountries();
|#E2EBDC|JsonLoader|
:JsonLoader::LoadLocations("locations.json");
:std::ranges::sample(all_locations, 50);
:std::ranges::sample(all_locations, --location-count);
note right
--location-count defaults to 10.
end note
|#EAF0E8|BiergartenDataGenerator|
|#EAF0E8|BiergartenPipelineOrchestrator|
while (For each sampled Location?) is (Remaining cities)
|#DCE8D8|WikipediaService|
|#DCE8D8|IEnrichmentService|
:GetLocationContext(loc);
:FetchExtracts(City, Country, Beer);
|#EAF0E8|BiergartenDataGenerator|
:Store EnrichedCity{Location, region_context};
note right
WikipediaEnrichmentService fetches a generic
"brewing" extract and a "beer in {country}" extract
(not a city/region-specific page). MockEnrichmentService
(--mocked) returns an empty string instead.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
if (Lookup failed?) then (yes)
:Log warning, skip city;
else (no)
:Store EnrichedCity{Location, region_context};
endif
endwhile (Done)
|#EAF0E8|BiergartenDataGenerator|
|#EAF0E8|BiergartenPipelineOrchestrator|
:GenerateUsers(enriched_cities);
|#E2EBDC|JsonLoader|
:LoadPersonas("personas.json");
:LoadForenamesByCountry("forenames-by-country.json");
:LoadSurnamesByCountry("surnames-by-country.json");
note right
Each call is cached after its first parse.
MockCuratedDataService (--mocked) returns a
fixed in-memory dataset instead and skips
these three JSON files entirely.
end note
|#EAF0E8|BiergartenPipelineOrchestrator|
while (For each EnrichedCity?) is (Remaining cities)
:SampleName(forenames_by_country,\n surnames_by_country, location.iso3166_1);
if (Names available for country?) then (no)
:Log warning, skip city;
else (yes)
:Sample UserPersona (uniform random);
|#E5EDE1|DataGenerator|
if (Generator Mode) then (MockGenerator)
:DeterministicHash & Format;
else (LlamaGenerator)
:prompt_directory_->Load("USER_GENERATION");
note right
Resolves to USER_GENERATION.md inside --prompt-dir.
end note
repeat
:Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
:ValidateUserJson(raw, user);
if (Is JSON Valid?) then (yes)
break
else (no)
:Attempt++;
:Append validation error to retry prompt;
endif
repeat while (Attempt < 3?) is (yes)
endif
|#EAF0E8|BiergartenPipelineOrchestrator|
if (Generation successful?) then (yes)
:BuildEmail(name, used_email_local_parts);
:GenerateDateOfBirth(rng);
note right
email and date_of_birth are generated
programmatically, never LLM-authored.
end note
|#E0EAE0|SqliteExportService|
:ProcessRecord(UserRecord);
if (Location in cache?) then (yes)
:Reuse location_id;
else (no)
:Insert Location & Cache ID;
endif
:Insert User (FK: location_id);
if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenPipelineOrchestrator|
:Log warning "SQLite export failed";
else (no)
endif
else (no)
:Log warning "Generation failed, skipping...";
endif
endif
endwhile (Done)
|#EAF0E8|BiergartenPipelineOrchestrator|
:GenerateBreweries(enriched_cities);
|#E5EDE1|DataGenerator|
@@ -73,7 +173,10 @@ while (For each EnrichedCity?) is (Remaining cities)
:DeterministicHash & Format;
else (LlamaGenerator)
:PrepareRegionContext;
:LoadBrewerySystemPrompt("prompts/system.md");
:prompt_directory_->Load("BREWERY_GENERATION");
note right
Resolves to BREWERY_GENERATION.md inside --prompt-dir.
end note
repeat
:Infer(system_prompt, user_prompt, max_tokens, kBreweryJsonGrammar);
:ValidateBreweryJson(raw, brewery);
@@ -81,11 +184,16 @@ while (For each EnrichedCity?) is (Remaining cities)
break
else (no)
:Attempt++;
:Append validation error to retry prompt;
endif
repeat while (Attempt < 3?) is (yes)
note right
max_tokens is fixed across retries —
it is not raised on truncation.
end note
endif
|#EAF0E8|BiergartenDataGenerator|
|#EAF0E8|BiergartenPipelineOrchestrator|
if (Generation successful?) then (yes)
|#E0EAE0|SqliteExportService|
:ProcessRecord(GeneratedBrewery);
@@ -97,8 +205,8 @@ while (For each EnrichedCity?) is (Remaining cities)
:Insert Brewery (FK: location_id);
if (Exception caught during insert?) then (yes)
|#EAF0E8|BiergartenDataGenerator|
:spdlog::warn "Failed to stream record to SQLite export";
|#EAF0E8|BiergartenPipelineOrchestrator|
:Log warning "SQLite export failed";
note right
Data loss is prevented per-record.
The pipeline continues running.
@@ -106,7 +214,7 @@ while (For each EnrichedCity?) is (Remaining cities)
else (no)
endif
else (no)
:spdlog::warn "Generation failed, skipping...";
:Log warning "Generation failed, skipping...";
endif
|#E5EDE1|DataGenerator|
endwhile (Done)
@@ -119,6 +227,8 @@ note right
end note
|#F2F6F0|main.cc|
:Close log_channel;
:Join log dispatcher thread;
:Return 0;
stop

View File

@@ -25,15 +25,19 @@ skinparam note {
title The Biergarten Data Pipeline - Class Diagram
class BiergartenDataGenerator {
class BiergartenPipelineOrchestrator {
- logger_ : std::shared_ptr<ILogger>
- context_service_ : std::unique_ptr<IEnrichmentService>
- generator_ : std::unique_ptr<DataGenerator>
- exporter_ : std::unique_ptr<IExportService>
- generated_breweries_ : std::vector<GeneratedBrewery>
- curated_data_service_ : std::unique_ptr<ICuratedDataService>
- application_options_ : ApplicationOptions
- generated_breweries_ : std::vector<BreweryRecord>
- generated_users_ : std::vector<UserRecord>
+ Run() : bool
- QueryCitiesWithCountries() : std::vector<Location>
- GenerateBreweries(cities : std::span<const EnrichedCity>) : void
- GenerateUsers(cities : std::span<const EnrichedCity>) : void
- LogResults() : void
}
@@ -54,21 +58,29 @@ class PipelinePhase <<enumeration>> {
Teardown
}
struct LogEntry {
+ timestamp : std::chrono::system_clock::time_point
struct LogDTO {
+ level : LogLevel
+ phase : PipelinePhase
+ message : std::string
}
struct LogEntry {
+ timestamp : std::chrono::system_clock::time_point
+ origin : std::source_location
+ thread_id : std::thread::id
+ level : LogLevel
+ phase : PipelinePhase
+ message : std::string
+ worker : std::optional<std::string>
}
interface ILogger <<interface>> {
+ Log(entry : const LogEntry&) : void
+ Log(payload : LogDTO) : void
- {abstract} DoLog(entry : LogEntry) : void
}
class LogProducer {
- channel_ : BoundedChannel<LogEntry>&
+ Log(entry : const LogEntry&) : void
- DoLog(entry : LogEntry) : void
}
class LogDispatcher {
@@ -81,7 +93,11 @@ interface IEnrichmentService <<interface>> {
+ GetLocationContext(loc : const Location&) : std::string
}
class WikipediaService {
class MockEnrichmentService {
+ GetLocationContext(loc : const Location&) : std::string
}
class WikipediaEnrichmentService {
- client_ : std::unique_ptr<WebClient>
- extract_cache_ : std::unordered_map<std::string, std::string>
+ GetLocationContext(loc : const Location&) : std::string
@@ -100,26 +116,27 @@ class HttpWebClient {
interface DataGenerator <<interface>> {
+ GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
+ GenerateUser(locale : const std::string&) : UserResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
}
class MockGenerator {
+ GenerateBrewery(...) : BreweryResult
+ GenerateUser(...) : UserResult
+ GenerateBrewery(location : const Location&, region_context : const std::string&) : BreweryResult
+ GenerateUser(city : const EnrichedCity&, persona : const UserPersona&, name : const Name&) : UserResult
- DeterministicHash(location : const Location&) : size_t
- DeterministicHash(location : const Location&, persona : const UserPersona&, name : const Name&) : size_t
}
class LlamaGenerator {
- model_ : ModelHandle
- context_ : ContextHandle
- prompt_formatter_ : std::unique_ptr<IPromptFormatter>
- prompt_directory_ : std::unique_ptr<IPromptDirectory>
- rng_ : std::mt19937
+ GenerateBrewery(...) : BreweryResult
+ GenerateUser(...) : UserResult
- Load(model_path : const std::string&) : void
- Infer(...) : std::string
- InferFormatted(...) : std::string
- LoadBrewerySystemPrompt(...) : std::string
}
interface IPromptFormatter <<interface>> {
@@ -130,13 +147,59 @@ class Gemma4JinjaPromptFormatter {
+ Format(system_prompt : std::string_view, user_prompt : std::string_view) : std::string
}
class JsonLoader {
+ {static} LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
interface IPromptDirectory <<interface>> {
+ Load(key : std::string_view) : std::string
}
class PromptDirectory {
- prompt_dir_ : std::filesystem::path
- cache_ : std::unordered_map<std::string, std::string>
+ Load(key : std::string_view) : std::string
}
interface ICuratedDataService <<interface>> {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
class JsonLoader {
- cache_ : cache
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
note right of JsonLoader
Each Load* method memoizes its result in
cache_ on first call; later calls on the
same instance return the cached value
without re-parsing.
end note
class MockCuratedDataService {
- locations_ : std::vector<Location>
- personas_ : std::vector<UserPersona>
- forenames_by_country_ : std::unordered_map<std::string, forename_list>
- surnames_by_country_ : std::unordered_map<std::string, surname_list>
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
note right of MockCuratedDataService
Fixed 4-location / 3-persona dataset
(US, DE, FR, BE) used in --mocked runs;
filepath arguments are ignored.
end note
interface IExportService <<interface>> {
+ Initialize() : void
+ ProcessRecord(brewery : const GeneratedBrewery&) : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+ ProcessRecord(user : const UserRecord&) : uint64_t
+ Finalize() : void
}
@@ -144,15 +207,18 @@ class SqliteExportService {
- date_time_provider_ : std::unique_ptr<IDateTimeProvider>
- run_timestamp_utc_ : std::string
- database_path_ : std::filesystem::path
- db_handle_ : sqlite3*
- insert_location_stmt_ : sqlite3_stmt*
- insert_brewery_stmt_ : sqlite3_stmt*
- db_handle_ : SqliteDatabaseHandle
- insert_location_stmt_ : SqliteStatementHandle
- insert_brewery_stmt_ : SqliteStatementHandle
- insert_user_stmt_ : SqliteStatementHandle
- transaction_open_ : bool
- location_cache_ : std::unordered_map<std::string, sqlite3_int64>
+ Initialize() : void
+ ProcessRecord(brewery : const GeneratedBrewery&) : void
+ ProcessRecord(brewery : const BreweryRecord&) : uint64_t
+ ProcessRecord(user : const UserRecord&) : uint64_t
+ Finalize() : void
- InitializeSchema() : void
- ResolveLocationId(location : const Location&) : sqlite3_int64
}
interface IDateTimeProvider <<interface>> {
@@ -164,19 +230,23 @@ class SystemDateTimeProvider {
}
' Structural Relationships / Dependency Injection
BiergartenDataGenerator *-- ILogger : owns
BiergartenDataGenerator *-- IEnrichmentService : owns
BiergartenDataGenerator *-- DataGenerator : owns
BiergartenDataGenerator *-- IExportService : owns
BiergartenPipelineOrchestrator *-- ILogger : owns
BiergartenPipelineOrchestrator *-- IEnrichmentService : owns
BiergartenPipelineOrchestrator *-- DataGenerator : owns
BiergartenPipelineOrchestrator *-- IExportService : owns
BiergartenPipelineOrchestrator *-- ICuratedDataService : owns
LogEntry *-- LogLevel
LogEntry *-- PipelinePhase
LogDTO *-- LogLevel
LogDTO *-- PipelinePhase
ILogger <|.. LogProducer : implements
LogProducer ..> LogEntry : emits
LogDispatcher ..> LogEntry : consumes
IEnrichmentService <|.. WikipediaService : implements
WikipediaService *-- WebClient : owns
IEnrichmentService <|.. WikipediaEnrichmentService : implements
IEnrichmentService <|.. MockEnrichmentService : implements
WikipediaEnrichmentService *-- WebClient : owns
WebClient <|.. HttpWebClient : implements
@@ -184,10 +254,13 @@ DataGenerator <|.. MockGenerator : implements
DataGenerator <|.. LlamaGenerator : implements
LlamaGenerator *-- IPromptFormatter : uses
LlamaGenerator *-- IPromptDirectory : uses
IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements
IPromptDirectory <|.. PromptDirectory : implements
BiergartenDataGenerator ..> JsonLoader : uses
ICuratedDataService <|.. JsonLoader : implements
ICuratedDataService <|.. MockCuratedDataService : implements
IExportService <|.. SqliteExportService : implements
SqliteExportService *-- IDateTimeProvider : owns

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 44 KiB

After

Width:  |  Height:  |  Size: 86 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 44 KiB

After

Width:  |  Height:  |  Size: 87 KiB

View File

@@ -43,10 +43,18 @@ fork again
:EnrichmentService::PreWarmLocationCache(sampled_locations);
end fork
fork
:JsonLoader::LoadNamesByCountry("names-by-country.json");
:JsonLoader::LoadForenamesByCountry(\n "forenames-by-country.json");
fork again
:JsonLoader::LoadSurnamesByCountry(\n "surnames-by-country.json");
fork again
:JsonLoader::LoadPersonas("personas.json");
end fork
note right
Each call is memoized after its first parse
(implemented today). MockCuratedDataService
returns a fixed in-memory dataset instead
under --mocked, skipping these JSON files.
end note
' ═══════════════════════════════════════════
' PHASE 0 — USER GENERATION
@@ -82,11 +90,15 @@ fork again
ibu_preference, checkin_weight.
end note
:NamesByCountry::SampleName(\n location.iso3166_1);
:SampleName(forenames_by_country,\n surnames_by_country, location.iso3166_1, rng);
note right
Deterministic lookup -- no LLM involved.
Name selected from pre-keyed table
Free function (implemented today in
generate_users.cc), not a class method.
Name selected from pre-keyed maps
and passed into the generation prompt.
Skips the city if either map has no
entry for iso3166_1.
end note
:GenerateUser(enriched_city, persona, sampled_name)\nvia DataGenerator;

View File

@@ -67,11 +67,50 @@ package "Domain: Models" {
}
class UserResult {
+ first_name : std::string
+ last_name : std::string
+ gender : std::string
+ username : std::string
+ bio : std::string
+ activity_weight : float
}
class Name {
+ first_name : std::string
+ last_name : std::string
+ gender : std::string
}
class ForenameEntry {
+ name : std::string
+ gender : std::string
}
note right of ForenameEntry
forename_list = std::unordered_set<ForenameEntry>
surname_list = std::unordered_set<std::string>
(aliases declared in curated_data_service.h).
ICuratedDataService::LoadForenamesByCountry() /
LoadSurnamesByCountry() each return a flat
std::unordered_map<std::string, forename_list>
or <std::string, surname_list> keyed by ISO
3166-1 country code -- no NamesByCountry
wrapper class. SampleName(forenames_by_country,
surnames_by_country, iso3166_1, rng) is a free
function, not a method, so it can be called by
any per-city worker without owning the maps.
Sourced from popular-names-by-country-dataset
(CC0) -- see ETHICS-AND-KNOWN-ISSUES.md. Gender
is preserved per forename so it can be threaded
through to persona/bio generation later; surnames
carry no gender in the source data. Pairing
happens at sample time, not at fixture-authoring
time, so no information from the source dataset
is discarded up front.
end note
ForenameEntry ..> Name : SampleName() produces
class CheckinResult {
+ checked_in_at : std::string
+ note : std::string
@@ -110,9 +149,20 @@ package "Domain: Models" {
+ user_id : uint64_t
+ location : Location
+ user : UserResult
+ email : std::string
+ date_of_birth : std::string
+ password : std::string
+ metadata : GenerationMetadata
}
note right of GeneratedUser
email, date_of_birth, and password are
programmatically generated by the orchestrator
(not LLM-authored) so a downstream auth-account
seeding consumer can register real accounts from
this export. See ROADMAP.md §1.
end note
class GeneratedCheckin {
+ checkin_id : uint64_t
+ user_id : uint64_t
@@ -318,20 +368,43 @@ package "Infrastructure: Pipeline Channel" {
package "Infrastructure: Data Preloading" {
interface DataPreloader <<interface>> {
+ LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
+ LoadBeerStyles(filepath : const std::filesystem::path&) : std::vector<BeerStyle>
+ LoadPersonas(filepath : const std::filesystem::path&) : std::vector<Persona>
+ LoadNamesByCountry(filepath : const std::filesystem::path&) : NamesByCountry
interface ICuratedDataService <<interface>> {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
class JsonLoader {
+ LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
+ LoadBeerStyles(filepath : const std::filesystem::path&) : std::vector<BeerStyle>
+ LoadPersonas(filepath : const std::filesystem::path&) : std::vector<Persona>
+ LoadNamesByCountry(filepath : const std::filesystem::path&) : NamesByCountry
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
note right of JsonLoader
Each Load* call is memoized after its first
parse -- implemented today, ahead of the rest
of this package (LoadBeerStyles is still planned).
end note
class MockCuratedDataService {
+ LoadLocations(filepath : const std::filesystem::path&) : const std::vector<Location>&
+ LoadBeerStyles(filepath : const std::filesystem::path&) : const std::vector<BeerStyle>&
+ LoadPersonas(filepath : const std::filesystem::path&) : const std::vector<UserPersona>&
+ LoadForenamesByCountry(forenames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, forename_list>&
+ LoadSurnamesByCountry(surnames_filepath : const std::filesystem::path&) : const std::unordered_map<std::string, surname_list>&
}
note right of MockCuratedDataService
Fixed in-memory dataset used in --mocked mode,
implemented today for Locations/Personas/
Forenames/Surnames; would need a LoadBeerStyles
fixture too once that method is implemented.
end note
}
package "Infrastructure: Enrichment" {
@@ -380,7 +453,7 @@ package "Infrastructure: Data Generation" {
interface DataGenerator <<interface>> {
+ GenerateBrewery(location : const Location&,\n context : const LocationContext&) : BreweryResult
+ GenerateBeer(brewery_id : uint64_t,\n location : const Location&,\n context : const LocationContext&,\n style : const BeerStyle&) : BeerResult
+ GenerateUser(location : const Location&) : UserResult
+ GenerateUser(city : const EnrichedCity&,\n persona : const UserPersona&,\n name : const Name&) : UserResult
+ GenerateCheckin(user : const GeneratedUser&,\n brewery : const GeneratedBrewery&,\n timestamp : const std::string&) : CheckinResult
+ GenerateRating(user : const GeneratedUser&,\n beer : const GeneratedBeer&,\n checkin_id : uint64_t) : RatingResult
}
@@ -473,7 +546,7 @@ package "Infrastructure: Data Export" {
}
class BiergartenPipelineOrchestrator {
- preloader_ : std::unique_ptr<DataPreloader>
- curated_data_service_ : std::unique_ptr<ICuratedDataService>
- enrichment_service_ : std::unique_ptr<EnrichmentService>
- generator_ : std::unique_ptr<DataGenerator>
- logger_ : std::unique_ptr<Logger>
@@ -501,7 +574,7 @@ class BiergartenPipelineOrchestrator {
}
' --- Orchestration Aggregations (Services & Strategies) ---
BiergartenPipelineOrchestrator *-- DataPreloader
BiergartenPipelineOrchestrator *-- ICuratedDataService
BiergartenPipelineOrchestrator *-- EnrichmentService
BiergartenPipelineOrchestrator *-- DataGenerator
BiergartenPipelineOrchestrator *-- ExportService
@@ -520,7 +593,8 @@ BiergartenPipelineOrchestrator *-- "0..*" GeneratedCheckin : checkin_pool_
BiergartenPipelineOrchestrator *-- "0..*" GeneratedFollow : follow_pool_
' --- Interfaces & Implementations ---
DataPreloader <|.. JsonLoader
ICuratedDataService <|.. JsonLoader
ICuratedDataService <|.. MockCuratedDataService
Logger <|.. PipelineLogger
ContextStrategy <|.. BreweryContextStrategy
ContextStrategy <|.. BeerContextStrategy

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 106 KiB

After

Width:  |  Height:  |  Size: 142 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 184 KiB

After

Width:  |  Height:  |  Size: 240 KiB

View File

@@ -0,0 +1,24 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

View File

@@ -0,0 +1,74 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Pipeline Results Viewer</title>
</head>
<body>
<div class="dialog" id="window">
<div class="dialog-header">
<p>Pipeline Results Viewer</p>
</div>
<div class="tab-strip" role="tablist">
<button
type="button"
class="tab is-active"
id="tab-tables"
role="tab"
aria-selected="true"
aria-controls="panel-tables"
>
Tables
</button>
<button
type="button"
class="tab"
id="tab-query"
role="tab"
aria-selected="false"
aria-controls="panel-query"
>
Query
</button>
</div>
<div class="dialog-body">
<div
class="tab-panel"
id="panel-tables"
role="tabpanel"
aria-labelledby="tab-tables"
>
<div class="field-group">
<label for="table-select">Table</label>
<select id="table-select"></select>
</div>
<p id="status"></p>
<div id="table-container"></div>
</div>
<div
class="tab-panel"
id="panel-query"
role="tabpanel"
aria-labelledby="tab-query"
hidden
>
<div class="field-group field-group--stacked">
<label for="query-input">SQL Query</label>
<textarea
id="query-input"
rows="4"
spellcheck="false"
placeholder="SELECT * FROM ..."
></textarea>
</div>
<button type="button" id="query-run" class="btn">Run Query</button>
<p id="query-status"></p>
<div id="query-container"></div>
</div>
</div>
</div>
<script type="module" src="/src/main.ts"></script>
</body>
</html>

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,20 @@
{
"name": "pipeline-results-viewer",
"version": "0.0.0",
"private": true,
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"preview": "vite preview"
},
"devDependencies": {
"@types/sql.js": "^1.4.11",
"sass": "^1.101.0",
"typescript": "^6.0.3",
"vite": "^8.0.12"
},
"dependencies": {
"sql.js": "^1.14.1"
}
}

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 9.3 KiB

View File

@@ -0,0 +1,147 @@
import initSqlJs, { type Database, type SqlValue } from "sql.js";
import sqlWasmUrl from "sql.js/dist/sql-wasm.wasm?url";
import "./style.scss";
const DB_URL = "/biergarten.sqlite";
const tabs = document.querySelectorAll<HTMLButtonElement>("[role='tab']");
const panels = document.querySelectorAll<HTMLDivElement>("[role='tabpanel']");
const tableSelect = document.querySelector<HTMLSelectElement>("#table-select")!;
const tableContainer =
document.querySelector<HTMLDivElement>("#table-container")!;
const status = document.querySelector<HTMLParagraphElement>("#status")!;
const queryInput = document.querySelector<HTMLTextAreaElement>("#query-input")!;
const queryRun = document.querySelector<HTMLButtonElement>("#query-run")!;
const queryStatus = document.querySelector<HTMLParagraphElement>("#query-status")!;
const queryContainer =
document.querySelector<HTMLDivElement>("#query-container")!;
for (const tab of tabs) {
tab.addEventListener("click", () => {
for (const otherTab of tabs) {
const isActive = otherTab === tab;
otherTab.classList.toggle("is-active", isActive);
otherTab.setAttribute("aria-selected", String(isActive));
}
for (const panel of panels) {
panel.hidden = panel.id !== tab.getAttribute("aria-controls");
}
});
}
const renderResultTable = (
container: HTMLDivElement,
columns: string[],
values: SqlValue[][],
) => {
const table = document.createElement("table");
const head = table.createTHead().insertRow();
for (const column of columns) {
const th = document.createElement("th");
th.textContent = column;
head.append(th);
}
const body = table.createTBody();
for (const row of values) {
const tr = body.insertRow();
for (const cell of row) {
const td = document.createElement("td");
td.textContent = String(cell ?? "");
tr.append(td);
}
}
container.append(table);
};
const renderTable = (db: Database, name: string) => {
const result = db.exec(`SELECT * FROM "${name}"`);
tableContainer.replaceChildren();
if (result.length === 0) {
tableContainer.append(
Object.assign(document.createElement("p"), { textContent: "No rows." }),
);
return;
}
const { columns, values } = result[0];
renderResultTable(tableContainer, columns, values);
};
const runQuery = (db: Database) => {
const sql = queryInput.value.trim();
queryContainer.replaceChildren();
if (!sql) {
queryStatus.textContent = "Enter a query to run.";
return;
}
try {
const results = db.exec(sql);
if (results.length === 0) {
queryStatus.textContent = "Query executed. No rows returned.";
return;
}
const { columns, values } = results[results.length - 1];
renderResultTable(queryContainer, columns, values);
queryStatus.textContent = `${values.length} row${values.length === 1 ? "" : "s"} returned.`;
} catch (error) {
queryStatus.textContent = `Error: ${error instanceof Error ? error.message : String(error)}`;
}
};
async function main(): Promise<void> {
status.textContent = "Loading database...";
const sqlJs = await initSqlJs({ locateFile: () => sqlWasmUrl });
const response = await fetch(DB_URL);
const buffer = await response.arrayBuffer();
const db = new sqlJs.Database(new Uint8Array(buffer));
const tables =
db
.exec(
`SELECT name
FROM sqlite_master
WHERE
type = 'table'
AND
name NOT LIKE 'sqlite_%'
ORDER BY name`,
)[0]
?.values.map((row) => String(row[0])) ?? [];
for (const name of tables) {
tableSelect.append(new Option(name, name));
}
tableSelect.addEventListener("change", () =>
renderTable(db, tableSelect.value),
);
if (tables.length > 0) {
renderTable(db, tables[0]);
}
status.textContent = "";
queryRun.addEventListener("click", () => runQuery(db));
queryInput.addEventListener("keydown", (event) => {
if ((event.metaKey || event.ctrlKey) && event.key === "Enter") {
event.preventDefault();
runQuery(db);
}
});
}
main().catch((error: unknown) => {
status.textContent = `Failed to load database: ${String(error)}`;
});

View File

@@ -0,0 +1,267 @@
$gray-100: #dedede;
$gray-300: #c3c3c3;
$gray-500: #808080;
$white: #ffffff;
$black: #000000;
$navy: #000080;
$blue: #1084d0;
$teal: #008080;
$color-surface: $gray-300;
$color-surface-alt: $gray-100;
$color-surface-sunken: $white;
$color-border-highlight: $gray-100;
$color-border-lowlight: $gray-500;
$color-border-light: $white;
$color-border-dark: $black;
$color-text: $black;
$color-text-on-accent: $white;
$color-text-status: $navy;
$color-accent-start: $navy;
$color-accent-end: $blue;
$color-desktop-bg: $teal;
$color-row-stripe: $gray-100;
$color-shadow: rgba($black, 0.4);
$color-focus-ring: $black;
$bevel-size: 2px;
$font-stack: Tahoma, "MS Sans Serif", Verdana, sans-serif;
@mixin bevel-raised($size: $bevel-size) {
border-style: solid;
border-width: $size;
border-top-color: $color-border-highlight;
border-left-color: $color-border-highlight;
border-right-color: $color-border-dark;
border-bottom-color: $color-border-dark;
box-shadow:
inset $size $size 0 0 $color-border-light,
inset (-$size) (-$size) 0 0 $color-border-lowlight;
}
@mixin bevel-sunken($size: $bevel-size) {
border-style: solid;
border-width: $size;
border-top-color: $color-border-lowlight;
border-left-color: $color-border-lowlight;
border-right-color: $color-border-light;
border-bottom-color: $color-border-light;
box-shadow:
inset $size $size 0 0 $color-border-dark,
inset (-$size) (-$size) 0 0 $color-surface-alt;
}
* {
box-sizing: border-box;
}
body {
margin: 0;
padding: 24px;
background: $color-desktop-bg;
font-family: $font-stack;
color: $color-text;
min-height: 100vh;
}
#window {
width: 100%;
max-width: 1600px;
margin: 0 auto;
background: $color-surface;
padding: 6px;
display: flex;
flex-direction: column;
gap: 6px;
@include bevel-raised(3px);
box-shadow: 8px 8px 0 0 $color-shadow;
}
.dialog-header {
background: linear-gradient(90deg, $color-accent-start, $color-accent-end);
color: $color-text-on-accent;
padding: 6px 8px;
display: flex;
align-items: center;
p {
margin: 0;
font-size: 14px;
font-weight: bold;
letter-spacing: 0.5px;
text-transform: uppercase;
}
}
.tab-strip {
display: flex;
gap: 4px;
padding: 8px 8px 0;
}
.tab {
position: relative;
top: 3px;
margin-bottom: -2px;
font-family: $font-stack;
font-size: 12px;
font-weight: bold;
text-transform: uppercase;
padding: 6px 14px;
background: $color-surface;
color: $color-text;
border-style: solid;
border-width: 2px;
border-top-color: $color-border-light;
border-left-color: $color-border-light;
border-right-color: $color-border-dark;
border-bottom-color: $color-border-dark;
border-radius: 0;
cursor: pointer;
&.is-active {
top: 0;
z-index: 1;
border-bottom-color: $color-surface;
}
&:focus {
outline: 1px dotted $color-focus-ring;
outline-offset: -4px;
}
}
.dialog-body {
border-top: 2px solid $color-border-dark;
padding: 12px;
}
.tab-panel {
display: flex;
flex-direction: column;
gap: 12px;
&[hidden] {
display: none;
}
}
.field-group {
display: flex;
align-items: center;
gap: 8px;
label {
font-size: 12px;
font-weight: bold;
text-transform: uppercase;
}
&.field-group--stacked {
flex-direction: column;
align-items: stretch;
}
}
select#table-select {
font-family: $font-stack;
font-size: 13px;
padding: 3px 6px;
background: $color-surface-sunken;
color: $color-text;
border-radius: 0;
@include bevel-sunken();
&:focus {
outline: 1px dotted $color-focus-ring;
outline-offset: 2px;
}
}
textarea#query-input {
font-family: $font-stack;
font-size: 13px;
padding: 6px;
background: $color-surface-sunken;
color: $color-text;
border-radius: 0;
resize: vertical;
@include bevel-sunken();
&:focus {
outline: 1px dotted $color-focus-ring;
outline-offset: 2px;
}
}
.btn {
align-self: flex-start;
font-family: $font-stack;
font-size: 12px;
font-weight: bold;
text-transform: uppercase;
padding: 6px 16px;
background: $color-surface;
color: $color-text;
cursor: pointer;
@include bevel-raised();
&:active {
@include bevel-sunken();
}
&:focus {
outline: 1px dotted $color-focus-ring;
outline-offset: -4px;
}
}
#status,
#query-status {
margin: 0;
font-size: 12px;
font-style: italic;
color: $color-text-status;
}
#table-container,
#query-container {
background: $color-surface-sunken;
padding: 4px;
overflow-x: auto;
@include bevel-sunken();
}
table {
border-collapse: collapse;
margin: 0 auto;
font-family: $font-stack;
}
th,
td {
border: 1px solid $color-border-lowlight;
padding: 4px 8px;
text-align: left;
font-size: 13px;
}
th {
background: $color-surface;
white-space: nowrap;
font-weight: bold;
text-transform: uppercase;
@include bevel-raised(1px);
}
td {
min-width: 120px;
max-width: 800px;
white-space: normal;
word-break: break-word;
background: $color-surface-sunken;
}
tbody tr:nth-child(even) td {
background: $color-row-stripe;
}

View File

@@ -0,0 +1,24 @@
{
"compilerOptions": {
"target": "es2023",
"module": "esnext",
"lib": ["ES2023", "DOM"],
"types": ["vite/client"],
"skipLibCheck": true,
/* Bundler mode */
"moduleResolution": "bundler",
"esModuleInterop": true,
"allowImportingTsExtensions": true,
"verbatimModuleSyntax": true,
"moduleDetection": "force",
"noEmit": true,
/* Linting */
"noUnusedLocals": true,
"noUnusedParameters": true,
"erasableSyntaxOnly": true,
"noFallthroughCasesInSwitch": true
},
"include": ["src"]
}

View File

@@ -1,5 +1,5 @@
---
BasedOnStyle: Google
ColumnLimit: 80
IndentWidth: 2
IndentWidth: 3
...

View File

@@ -1,12 +1,6 @@
Checks: >
-*,
bugprone-*,
google-*,
modernize-*,
readability-*,
cppcoreguidelines-*,
-modernize-use-trailing-return-type,
-google-runtime-references
-*, bugprone-*, google-*, modernize-*, readability-*, cppcoreguidelines-*,
-modernize-use-trailing-return-type, -google-runtime-references
CheckOptions:
# Enforce Google Naming Conventions with valid clang-tidy strings

View File

@@ -111,6 +111,9 @@ if(NOT BIERGARTEN_MOCK_ONLY)
)
FetchContent_MakeAvailable(llama-cpp)
endif()
if (NOT APPLE)
find_package(OpenMP REQUIRED)
endif()
endif()
# spdlog
@@ -139,6 +142,7 @@ FetchContent_MakeAvailable(cpp-httplib)
# 5. Executable & Sources
add_executable(${PROJECT_NAME}
includes/services/enrichment/mock_enrichment.h
includes/services/curated_data/mock_curated_data_service.h
includes/json_handling/pretty_print.h)
# --- Entry point ---
@@ -148,7 +152,12 @@ target_sources(${PROJECT_NAME} PRIVATE
# --- json_handling ---
target_sources(${PROJECT_NAME} PRIVATE
src/json_handling/json_loader.cc
src/services/curated_data/curated_json_data_service.cc
)
# --- services: curated_data ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/curated_data/mock_curated_data_service.cc
)
# --- application_options ---
@@ -161,6 +170,7 @@ target_sources(${PROJECT_NAME} PRIVATE
src/biergarten_pipeline_orchestrator/log_results.cc
src/biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
src/biergarten_pipeline_orchestrator/generate_breweries.cc
src/biergarten_pipeline_orchestrator/generate_users.cc
src/biergarten_pipeline_orchestrator/run.cc
src/biergarten_pipeline_orchestrator/query_cities_with_countries.cc
)
@@ -201,9 +211,22 @@ target_sources(${PROJECT_NAME} PRIVATE
src/services/enrichment/wikipedia/get_summary.cc
)
# --- services: postal_code ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/postal_code/xeger_postal_code_service.cc
)
# --- services: xeger ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/xeger/xeger.cc
src/services/xeger/xeger_parser.cc
src/services/xeger/xeger_generator.cc
)
# --- services: sqlite ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/sqlite/process_record.cc
src/services/sqlite/process_user_record.cc
src/services/sqlite/sqlite_export_service.cc
src/services/sqlite/finalize.cc
src/services/sqlite/initialize.cc
@@ -227,8 +250,10 @@ target_include_directories(${PROJECT_NAME} PRIVATE
includes
)
target_link_libraries(${PROJECT_NAME} PRIVATE
$<$<NOT:$<BOOL:${BIERGARTEN_MOCK_ONLY}>>:llama>
$<TARGET_NAME_IF_EXISTS:OpenMP::OpenMP_CXX>
boost::di
Boost::json
Boost::program_options
@@ -260,6 +285,21 @@ configure_file(
${CMAKE_BINARY_DIR}/locations.json
COPYONLY
)
configure_file(
${CMAKE_SOURCE_DIR}/personas.json
${CMAKE_BINARY_DIR}/personas.json
COPYONLY
)
configure_file(
${CMAKE_SOURCE_DIR}/forenames-by-country.json
${CMAKE_BINARY_DIR}/forenames-by-country.json
COPYONLY
)
configure_file(
${CMAKE_SOURCE_DIR}/surnames-by-country.json
${CMAKE_BINARY_DIR}/surnames-by-country.json
COPYONLY
)
add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory
${CMAKE_SOURCE_DIR}/prompts

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,72 @@
#!/bin/bash
#
# Walks every directory under src/ and includes/ (including nested
# subdirectories), runs clang-format on the C/C++ files in that directory
# only (non-recursive), and commits the result with its own commit.
#
# Usage: ./format-per-directory.sh [-y]
# -y skip the confirmation prompt
set -euo pipefail
SKIP_CONFIRM=false
if [[ "${1:-}" == "-y" ]]; then
SKIP_CONFIRM=true
fi
cd "$(dirname "$0")"
if [ ! -f .clang-format ]; then
echo "ERROR: .clang-format file not found."
exit 1
fi
if ! command -v clang-format &>/dev/null; then
echo "ERROR: clang-format not found."
exit 1
fi
if ! command -v git &>/dev/null; then
echo "ERROR: git not found."
exit 1
fi
echo "WARNING: This script will format .cpp, .h, .cxx, .cc, .c, .hpp files"
echo "directory-by-directory under src/ and includes/, committing after each"
echo "directory is formatted."
if [[ "$SKIP_CONFIRM" == false ]]; then
read -p "Do you want to continue? (y/n) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo "Aborted."
exit 1
fi
fi
if [[ -n "$(git status --porcelain)" ]]; then
echo "ERROR: working tree is not clean. Commit or stash your changes first."
exit 1
fi
dirs=$(find src includes -type d | sort)
for dir in $dirs; do
files=$(find "$dir" -maxdepth 1 -type f \( -name "*.cpp" -o -name "*.hpp" -o -name "*.h" -o -name "*.c" -o -name "*.cc" -o -name "*.cxx" \))
if [[ -z "$files" ]]; then
continue
fi
echo "Formatting $dir..."
echo "$files" | xargs clang-format -i
if [[ -n "$(git status --porcelain -- "$dir")" ]]; then
git add -- "$dir"
git commit -m "Formatted $dir"
else
echo "No changes in $dir, skipping commit."
fi
done
echo "Done."

View File

@@ -0,0 +1,80 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_
/**
* @file biergarten_pipeline.h
* @brief Master umbrella header — includes every public header in the
* Biergarten pipeline.
*/
// --- orchestrator ---
#include "biergarten_pipeline_orchestrator.h"
// --- concurrency ---
#include "concurrency/bounded_channel.h"
// --- data_generation ---
#include "data_generation/data_generator.h"
#include "data_generation/json_grammars.h"
#ifndef BIERGARTEN_MOCK_ONLY
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#endif
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
// --- data_model ---
#include "data_model/generated_models.h"
#include "data_model/models.h"
// --- json_handling ---
#include "json_handling/pretty_print.h"
#include "services/curated_data/curated_json_data_service.h"
// --- llama backend ---
#ifndef BIERGARTEN_MOCK_ONLY
#include "llama_backend_state.h"
#endif
// --- services: curated_data ---
#include "services/curated_data/curated_data_service.h"
#include "services/curated_data/mock_curated_data_service.h"
// --- services: database ---
#include "services/database/export_service.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
// --- services: datetime ---
#include "services/datetime/date_time_provider.h"
#include "services/datetime/timer.h"
// --- services: enrichment ---
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
// --- services: postal_code ---
#include "services/postal_code/mock_postal_code_service.h"
#include "services/postal_code/postal_code_service.h"
#include "services/postal_code/xeger_postal_code_service.h"
#include "services/xeger/xeger.h"
// --- services: logging ---
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
// --- services: prompting ---
#include "services/prompting/prompt_directory.h"
// --- web_client ---
#include "web_client/http_web_client.h"
#include "web_client/web_client.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_H_

View File

@@ -1,12 +1,9 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_
/**
* @file biergarten_data_generator.h
* @brief Orchestration for end-to-end brewery data generation pipeline.
*
* Intent: Coordinates location loading, enrichment, and generation phases
* to produce a complete dataset. Coordinates dependencies via composition root.
* @file biergarten_pipeline_orchestrator.h
* @brief Orchestration for end-to-end brewery and user data generation.
*/
#include <memory>
@@ -15,88 +12,110 @@
#include "data_generation/data_generator.h"
#include "data_model/generated_models.h"
#include "services/curated_data/curated_data_service.h"
#include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
#include "services/postal_code/postal_code_service.h"
/**
* @brief Main data generator class for the Biergarten pipeline.
* @brief Main orchestrator for the Biergarten data generation pipeline.
*
* This class encapsulates the core logic for generating brewery data.
* It handles location loading, city enrichment, and brewery generation.
* Handles location loading, city enrichment, and brewery/user generation.
*/
class BiergartenPipelineOrchestrator {
public:
/**
* @brief Constructs the orchestrator with injected pipeline dependencies.
*
* @param context_service Provides regional context for locations.
* @param generator Implementation (Llama or Mock) for brewery/user generation.
* @param exporter Database backend for persisting generated records.
* @param application_options CLI configuration and paths.
*/
BiergartenPipelineOrchestrator(
std::shared_ptr<ILogger> logger,
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
const ApplicationOptions& application_options);
public:
/**
* @brief Constructs the orchestrator with injected pipeline dependencies.
*
* @param logger Sink for pipeline diagnostics.
* @param context_service Provides regional context for locations.
* @param generator Implementation (Llama or Mock) for brewery/user
* generation.
* @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name
* data used to seed generation.
* @param postal_code_service Resolves a postal code for a city from its
* curated postal-code data.
* @param application_options CLI configuration and paths.
*/
BiergartenPipelineOrchestrator(
std::shared_ptr<ILogger> logger,
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options);
/**
* @brief Run the data generation pipeline.
*
* Performs the following steps:
* 1. Load curated locations from JSON
* 2. Resolve context for each city using the injected context service
* 3. Generate brewery data for sampled cities
*
* @note STRUCTURAL CONCURRENCY REQUIREMENT:
* When transitioned to a multithreaded design, this method MUST structurally
* enforce that all deployed worker threads are joined before returning (e.g.
* by using std::jthread or a structured concurrency primitive). This ensures
* workers do not attempt to log to a closed channel during application teardown.
*
* @return true if successful, false if not
*/
bool Run();
/**
* @brief Run the data generation pipeline.
*
* Performs the following steps:
* 1. Load curated locations from JSON
* 2. Resolve context for each city using the injected context service
* 3. Generate brewery and user data for sampled cities
*
* @note STRUCTURAL CONCURRENCY REQUIREMENT:
* When transitioned to a multithreaded design, this method MUST
* structurally enforce that all deployed worker threads are joined before
* returning (e.g. by using std::jthread or a structured concurrency
* primitive). This ensures workers do not attempt to log to a closed
* channel during application teardown.
*
* @return true if successful, false if not
*/
bool Run();
private:
/// @brief Logger instance for emitting pipeline messages.
std::shared_ptr<ILogger> logger_;
private:
std::shared_ptr<ILogger> logger_;
std::unique_ptr<IEnrichmentService> context_service_;
/// @brief Owning context provider dependency.
std::unique_ptr<IEnrichmentService> context_service_;
/**
* @brief Generator implementation selected at the composition root (Llama
* or Mock).
*/
std::unique_ptr<DataGenerator> generator_;
/// @brief Generator dependency selected in the composition root.
std::unique_ptr<DataGenerator> generator_;
/// @brief Storage backend for generated brewery records.
/**
* @brief Storage backend for generated brewery and user records.
*/
std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_;
std::unique_ptr<IPostalCodeService> postal_code_service_;
/// @brief CLI configuration: paths, model settings, generation parameters.
ApplicationOptions application_options_;
/**
* @brief Load locations from JSON and sample cities.
*
* @return Vector of sampled locations capped at 50 entries.
*/
std::vector<Location> QueryCitiesWithCountries();
/**
* @brief Load locations from JSON and sample cities.
*
* @return Vector of locations randomly sampled per
* PipelineOptions::location_count.
*/
std::vector<City> QueryLocations();
/**
* @brief Generate breweries for enriched cities.
*
* @param cities Span of enriched city data.
*/
void GenerateBreweries(std::span<const EnrichedCity> cities);
/**
* @brief Generate breweries for enriched cities.
*
* @param cities Span of enriched city data.
*/
void GenerateBreweries(std::span<const EnrichedCity> cities);
/**
* @brief Log the generated brewery results.
*/
void LogResults() const;
/**
* @brief Generate users grounded in sampled names and personas for
* enriched cities.
*
* @param cities Span of enriched city data.
*/
void GenerateUsers(std::span<const EnrichedCity> cities);
/// @brief Stores generated brewery data.
std::vector<GeneratedBrewery> generated_breweries_;
/**
* @brief Log the generated brewery and user results.
*/
void LogResults() const;
std::vector<BreweryRecord> generated_breweries_;
std::vector<UserRecord> generated_users_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_DATA_GENERATOR_H_
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_H_

View File

@@ -9,10 +9,12 @@
/**
* @file bounded_channel.h
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous channel.
* @brief Thread-safe, bounded multi-producer/multi-consumer synchronous
* channel.
*
* Intent: Enables asynchronous inter-thread communication with backpressure.
* Models a synchronous channel where producers/consumers block on capacity limits.
* Models a synchronous channel where producers/consumers block on capacity
* limits.
*/
/**
@@ -24,47 +26,45 @@
*/
template <typename T>
class BoundedChannel {
// -------------------------------------------------------------------------
// Internal state — all access must be guarded by mutex_.
// -------------------------------------------------------------------------
// Internal state — all access must be guarded by mutex_.
std::queue<T> queue_;
std::queue<T> queue_;
std::mutex mutex_;
std::mutex mutex_;
std::condition_variable not_full_;
std::condition_variable not_full_;
std::condition_variable not_empty_;
std::condition_variable not_empty_;
std::size_t capacity_;
std::size_t capacity_;
bool closed_ = false;
bool closed_ = false;
public:
/**
* @brief Construct a bounded channel with the given capacity.
* @param capacity Maximum number of items the channel may hold.
*/
explicit BoundedChannel(std::size_t capacity) : capacity_(capacity) {}
public:
/**
* @brief Construct a bounded channel with the given capacity.
* @param capacity Maximum number of items the channel may hold.
*/
explicit BoundedChannel(std::size_t capacity) : capacity_(capacity) {}
/**
* @brief Send an item into the channel. Blocks when the channel is full.
* @param item Move-only item to enqueue.
*/
void Send(T item);
/**
* @brief Send an item into the channel. Blocks when the channel is full.
* @param item Move-only item to enqueue.
*/
void Send(T item);
/**
* @brief Receive an item from the channel. Blocks when the channel is
* empty.
* @return std::optional<T> containing the item, or std::nullopt when the
* channel is closed and drained.
*/
std::optional<T> Receive();
/**
* @brief Receive an item from the channel. Blocks when the channel is
* empty.
* @return std::optional<T> containing the item, or std::nullopt when the
* channel is closed and drained.
*/
std::optional<T> Receive();
/**
* @brief Close the channel and unblock all waiting threads. Idempotent.
*/
void Close();
/**
* @brief Close the channel and unblock all waiting threads. Idempotent.
*/
void Close();
};
// Include the template implementation

View File

@@ -1,4 +1,4 @@
#include "bounded_channel.h"
#include "concurrency/bounded_channel.h"
template <typename T>
void BoundedChannel<T>::Send(T item) {

View File

@@ -14,26 +14,28 @@
* @brief Interface for data generator implementations.
*/
class DataGenerator {
public:
virtual ~DataGenerator() = default;
public:
virtual ~DataGenerator() = default;
/**
* @brief Generates brewery data for a location.
*
* @param location Location data
* @param region_context Additional regional context text.
* @return Brewery generation result.
*/
virtual BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) = 0;
/**
* @brief Generates brewery data for a location.
*
* @param city Enriched city the brewery is associated with.
* @return Brewery generation result.
*/
virtual BreweryResult GenerateBrewery(const EnrichedCity& city) = 0;
/**
* @brief Generates a user profile for a locale.
*
* @param locale Locale hint used by generator.
* @return User generation result.
*/
virtual UserResult GenerateUser(const std::string& locale) = 0;
/**
* @brief Generates a user profile grounded in a sampled name and persona.
*
* @param city Enriched city the user is associated with.
* @param persona Persona archetype grounding the generated bio.
* @param name Sampled first/last name (and gender)
* @return User generation result.
*/
virtual UserResult GenerateUser(const EnrichedCity& city,
const UserPersona& persona,
const Name& name) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_DATA_GENERATOR_H_

View File

@@ -0,0 +1,52 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_
/**
* @file data_generation/json_grammars.h
* @brief GBNF grammars constraining structured JSON output from
* LlamaGenerator inference calls.
*/
#include <string_view>
// GBNF grammar for structured user JSON output.
// thought-block permits the model to emit free-form reasoning before the
// JSON object (the prompts explicitly invite this); only the "{...}" tail is
// constrained to the expected shape.
inline constexpr std::string_view kUserJsonGrammar = R"json_user(
root ::= thought-block (
"{" ws
"\"username\"" ws ":" ws string ws "," ws
"\"bio\"" ws ":" ws string ws "," ws
"\"activity_weight\"" ws ":" ws number ws
"}" ws
)
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)?
)json_user";
// GBNF grammar for structured brewery JSON output (see thought-block note
// above).
inline constexpr std::string_view kBreweryJsonGrammar = R"json_brewery(
root ::= thought-block (
"{" ws
"\"name_en\"" ws ":" ws string ws "," ws
"\"description_en\"" ws ":" ws string ws "," ws
"\"name_local\"" ws ":" ws string ws "," ws
"\"description_local\"" ws ":" ws string ws
"}" ws
)
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
)json_brewery";
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_JSON_GRAMMARS_H_

View File

@@ -1,24 +1,23 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_
#include <filesystem>
/**
* @file data_generation/llama_generator.h
* @brief llama.cpp-backed implementation of DataGenerator.
*/
#include <cstdint>
#include <filesystem>
#include <memory>
#include <random>
#include <string>
#include <string_view>
#include "../services/prompting/prompt_directory.h"
#include "data_generation/data_generator.h"
#include "data_generation/prompt_formatting/prompt_formatter.h"
#include "data_model/models.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
struct llama_model;
struct llama_context;
@@ -27,113 +26,112 @@ struct llama_context;
* @brief Data generator implementation backed by llama.cpp.
*/
class LlamaGenerator final : public DataGenerator {
public:
/**
* @brief Constructs a generator using parsed application options and loads
* the configured model immediately.
*
* @param options Parsed application options.
* @param model_path Filesystem path to GGUF model assets.
* @param prompt_formatter Formatter that produces model-specific prompts.
* @param prompt_directory Directory service for loading named prompt files.
*/
LlamaGenerator(const ApplicationOptions& options,
const std::string& model_path, std::shared_ptr<ILogger> logger,
std::unique_ptr<IPromptFormatter> prompt_formatter,
std::unique_ptr<IPromptDirectory> prompt_directory);
public:
/**
* @brief Constructs a generator using parsed application options and loads
* the configured model immediately.
*
* @param options Parsed application options.
* @param model_path Filesystem path to GGUF model assets.
* @param prompt_formatter Formatter that produces model-specific prompts.
* @param prompt_directory Directory service for loading named prompt files.
*/
LlamaGenerator(const ApplicationOptions& options,
const std::string& model_path,
std::shared_ptr<ILogger> logger,
std::unique_ptr<IPromptFormatter> prompt_formatter,
std::unique_ptr<IPromptDirectory> prompt_directory);
~LlamaGenerator() override;
~LlamaGenerator() override;
// disable copy constructor
LlamaGenerator(const LlamaGenerator&) = delete;
LlamaGenerator(const LlamaGenerator&) = delete;
LlamaGenerator& operator=(const LlamaGenerator&) = delete;
LlamaGenerator(LlamaGenerator&&) = delete;
LlamaGenerator& operator=(LlamaGenerator&&) = delete;
// disable copy assignment operator
LlamaGenerator& operator=(const LlamaGenerator&) = delete;
// disable move constructor
LlamaGenerator(LlamaGenerator&&) = delete;
// disable move assignment operator
LlamaGenerator& operator=(LlamaGenerator&&) = delete;
/**
* @brief Generates brewery data for a specific location.
*
* @param location City object.
* @param region_context Additional regional context.
* @return Generated brewery result.
*/
BreweryResult GenerateBrewery(const EnrichedCity&) override;
/**
* @brief Generates brewery data for a specific location.
*
* @param location Location object.
* @param region_context Additional regional context.
* @return Generated brewery result.
*/
BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) override;
/**
* @brief Generates a user profile grounded in a sampled name and persona.
*
* @param city Enriched city the user is associated with.
* @param persona Persona archetype grounding the generated bio.
* @param name Sampled first/last name -- not LLM-invented.
* @return Generated user profile.
*/
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
const Name& name) override;
/**
* @brief Generates a user profile for the provided locale.
*
* @param locale Locale hint.
* @return Generated user profile.
*/
UserResult GenerateUser(const std::string& locale) override;
private:
static constexpr int32_t kDefaultMaxTokens = 10000;
static constexpr float kDefaultSamplingTopP = 0.95F;
static constexpr uint32_t kDefaultSamplingTopK = 64;
static constexpr uint32_t kDefaultContextSize = 8192;
private:
static constexpr int32_t kDefaultMaxTokens = 10000;
static constexpr float kDefaultSamplingTopP = 0.95F;
static constexpr uint32_t kDefaultSamplingTopK = 64;
static constexpr uint32_t kDefaultContextSize = 8192;
struct ModelDeleter {
void operator()(llama_model* model) const noexcept;
};
struct ContextDeleter {
void operator()(llama_context* context) const noexcept;
};
struct ModelDeleter {
void operator()(llama_model* model) const noexcept;
};
struct ContextDeleter {
void operator()(llama_context* context) const noexcept;
};
using ModelHandle = std::unique_ptr<llama_model, ModelDeleter>;
using ContextHandle = std::unique_ptr<llama_context, ContextDeleter>;
using ModelHandle = std::unique_ptr<llama_model, ModelDeleter>;
using ContextHandle = std::unique_ptr<llama_context, ContextDeleter>;
/**
* @brief Loads model and prepares inference context.
*
* @param model_path Filesystem path to GGUF model.
*/
void Load(const std::string& model_path);
/**
* @brief Loads model and prepares inference context.
*
* @param model_path Filesystem path to GGUF model.
*/
void Load(const std::string& model_path);
/**
* @brief Infers text from separate system and user prompts.
*
* This helps chat-capable models preserve system-role behavior instead of
* concatenating system text into user input.
*
* @param system_prompt System role prompt.
* @param prompt User prompt.
* @param max_tokens Maximum tokens to generate.
* @param grammar Optional GBNF grammar constraining generated output.
* @return Generated text.
*/
std::string Infer(const std::string& system_prompt,
const std::string& prompt,
int max_tokens = kDefaultMaxTokens,
std::string_view grammar = {});
/**
* @brief Infers text from separate system and user prompts.
*
* This helps chat-capable models preserve system-role behavior instead of
* concatenating system text into user input.
*
* @param system_prompt System role prompt.
* @param prompt User prompt.
* @param max_tokens Maximum tokens to generate.
* @param grammar Optional GBNF grammar constraining generated output.
* @return Generated text.
*/
std::string Infer(const std::string& system_prompt, const std::string& prompt,
int max_tokens = kDefaultMaxTokens,
std::string_view grammar = {});
/**
* @brief Runs inference on an already-formatted prompt.
*
* @param formatted_prompt Prompt preformatted for model chat template.
* @param max_tokens Maximum tokens to generate.
* @param grammar Optional GBNF grammar constraining generated output.
* @return Generated text.
*/
std::string InferFormatted(const std::string& formatted_prompt,
int max_tokens = kDefaultMaxTokens,
std::string_view grammar = {});
/**
* @brief Runs inference on an already-formatted prompt.
*
* @param formatted_prompt Prompt preformatted for model chat template.
* @param max_tokens Maximum tokens to generate.
* @param grammar Optional GBNF grammar constraining generated output.
* @return Generated text.
*/
std::string InferFormatted(const std::string& formatted_prompt,
int max_tokens = kDefaultMaxTokens,
std::string_view grammar = {});
ModelHandle model_;
ContextHandle context_;
float sampling_temperature_ = 1.0F;
float sampling_top_p_ = kDefaultSamplingTopP;
uint32_t sampling_top_k_ = kDefaultSamplingTopK;
std::mt19937 rng_;
uint32_t n_ctx_ = kDefaultContextSize;
int n_gpu_layers_ = 0;
std::shared_ptr<ILogger> logger_;
std::unique_ptr<IPromptFormatter> prompt_formatter_;
std::unique_ptr<IPromptDirectory> prompt_directory_;
ModelHandle model_;
ContextHandle context_;
float sampling_temperature_ = 1.0F;
float sampling_top_p_ = kDefaultSamplingTopP;
uint32_t sampling_top_k_ = kDefaultSamplingTopK;
std::mt19937 rng_;
uint32_t n_ctx_ = kDefaultContextSize;
int n_gpu_layers_ = 0;
std::shared_ptr<ILogger> logger_;
std::unique_ptr<IPromptFormatter> prompt_formatter_;
std::unique_ptr<IPromptDirectory> prompt_directory_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_H_

View File

@@ -47,4 +47,18 @@ void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out);
/**
* @brief Validates and parses user JSON output.
*
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
* and `last_name` are not LLM-authored and are set separately from the
* sampled Name.
*
* @param raw Raw model output.
* @param user_out Parsed user payload.
* @return Validation error message if invalid, or std::nullopt on success.
*/
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_HELPERS_H_

View File

@@ -16,115 +16,129 @@
* @brief Mock generator used for deterministic, model-free outputs.
*/
class MockGenerator final : public DataGenerator {
public:
/**
* @brief Generates deterministic brewery data for a location.
*
* @param location City and country names.
* @param region_context Unused for mock generation.
* @return Generated brewery result.
*/
BreweryResult GenerateBrewery(const Location& location,
const std::string& region_context) override;
public:
/**
* @brief Generates deterministic brewery data for a location.
*
* @param enriched_city
* @return Generated brewery result.
*/
/**
* @brief Generates deterministic user data for a locale.
*
* @param locale Locale hint.
* @return Generated user result.
*/
UserResult GenerateUser(const std::string& locale) override;
BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override;
/**
* @brief Generates deterministic user data grounded in a sampled name and
* persona.
private:
/**
* @brief Combines two strings into a stable hash value.
*
* @param location City and country names.
* @return Deterministic hash value.
*/
static size_t DeterministicHash(const Location& location);
* @return Generated user result.
*/
UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
const Name& name) override;
// Hash stride constants for deterministic distribution across fixed-size
// arrays. These coprime strides spread hash values uniformly without
// clustering, ensuring diverse output across different hash inputs.
static constexpr size_t kNounHashStride = 7;
static constexpr size_t kDescriptionHashStride = 13;
static constexpr size_t kBioHashStride = 11;
private:
/**
* @brief Combines two strings into a stable hash value.
*
* @param location City and country names.
* @return Deterministic hash value.
*/
static size_t DeterministicHash(const City& location);
static constexpr std::array<std::string_view, 18> kBreweryAdjectives = {
"Craft", "Heritage", "Local", "Artisan", "Pioneer", "Golden",
"Modern", "Classic", "Summit", "Northern", "Riverstone", "Barrel",
"Hinterland", "Harbor", "Wild", "Granite", "Copper", "Maple"};
/**
* @brief Combines city, persona, and name into a stable hash value.
*
* @param location City and country names.
* @param persona Persona archetype.
* @param name Sampled first/last name.
* @return Deterministic hash value.
*/
static size_t DeterministicHash(const City& location,
const UserPersona& persona,
const Name& name);
static constexpr std::array<std::string_view, 18> kBreweryNouns = {
"Brewing Co.", "Brewery", "Bier Haus", "Taproom", "Works",
"House", "Fermentery", "Ale Co.", "Cellars", "Collective",
"Project", "Foundry", "Malthouse", "Public House", "Co-op",
"Lab", "Beer Hall", "Guild"};
// Hash stride constants for deterministic distribution across fixed-size
// arrays. These coprime strides spread hash values uniformly without
// clustering, ensuring diverse output across different hash inputs.
static constexpr size_t kNounHashStride = 7;
static constexpr size_t kDescriptionHashStride = 13;
static constexpr size_t kBioHashStride = 11;
static constexpr size_t kActivityWeightHashStride = 17;
static constexpr size_t kActivityWeightHashRange = 1000;
static constexpr std::array<std::string_view, 18> kBreweryDescriptions = {
"Handcrafted pale ales and seasonal IPAs with local ingredients.",
"Traditional lagers and experimental sours in small batches.",
"Award-winning stouts and wildly hoppy blonde ales.",
"Craft brewery specializing in Belgian-style triples and dark "
"porters.",
"Modern brewery blending tradition with bold experimental flavors.",
"Neighborhood-focused taproom pouring crisp pilsners and citrusy "
"pale "
"ales.",
"Small-batch brewery known for barrel-aged releases and smoky "
"lagers.",
"Independent brewhouse pairing farmhouse ales with rotating food "
"pop-ups.",
"Community brewpub making balanced bitters, saisons, and hazy IPAs.",
"Experimental nanobrewery exploring local yeast and regional "
"grains.",
"Family-run brewery producing smooth amber ales and robust porters.",
"Urban brewery crafting clean lagers and bright, fruit-forward "
"sours.",
"Riverfront brewhouse featuring oak-matured ales and seasonal "
"blends.",
"Modern taproom focused on sessionable lagers and classic pub "
"styles.",
"Brewery rooted in tradition with a lineup of malty reds and crisp "
"lagers.",
"Creative brewery offering rotating collaborations and limited "
"draft-only "
"pours.",
"Locally inspired brewery serving approachable ales with bold hop "
"character.",
"Destination taproom known for balanced IPAs and cocoa-rich "
"stouts."};
static constexpr std::array<std::string_view, 18> kBreweryAdjectives = {
"Craft", "Heritage", "Local", "Artisan", "Pioneer", "Golden",
"Modern", "Classic", "Summit", "Northern", "Riverstone", "Barrel",
"Hinterland", "Harbor", "Wild", "Granite", "Copper", "Maple"};
static constexpr std::array<std::string_view, 18> kUsernames = {
"hopseeker", "malttrail", "yeastwhisper", "lagerlane",
"barrelbound", "foamfinder", "taphunter", "graingeist",
"brewscout", "aleatlas", "caskcompass", "hopsandmaps",
"mashpilot", "pintnomad", "fermentfriend", "stoutsignal",
"sessionwander", "kettlekeeper"};
static constexpr std::array<std::string_view, 18> kBreweryNouns = {
"Brewing Co.", "Brewery", "Bier Haus", "Taproom", "Works",
"House", "Fermentery", "Ale Co.", "Cellars", "Collective",
"Project", "Foundry", "Malthouse", "Public House", "Co-op",
"Lab", "Beer Hall", "Guild"};
static constexpr std::array<std::string_view, 18> kBios = {
"Always chasing balanced IPAs and crisp lagers across local taprooms.",
"Weekend brewery explorer with a soft spot for dark, roasty stouts.",
"Documenting tiny brewpubs, fresh pours, and unforgettable beer "
"gardens.",
"Fan of farmhouse ales, food pairings, and long tasting flights.",
"Collecting favorite pilsners one city at a time.",
"Hops-first drinker who still saves room for classic malt-forward "
"styles.",
"Finding hidden tap lists and sharing the best seasonal releases.",
"Brewery road-tripper focused on local ingredients and clean "
"fermentation.",
"Always comparing house lagers and ranking patio pint vibes.",
"Curious about yeast strains, barrel programs, and cellar experiments.",
"Believes every neighborhood deserves a great community taproom.",
"Looking for session beers that taste great from first sip to last.",
"Belgian ale enthusiast who never skips a new saison.",
"Hazy IPA critic with deep respect for a perfectly clear pilsner.",
"Visits breweries for the stories, stays for the flagship pours.",
"Craft beer fan mapping tasting notes and favorite brew routes.",
"Always ready to trade recommendations for underrated local breweries.",
"Keeping a running list of must-try collab releases and tap takeovers."};
static constexpr std::array<std::string_view, 18> kBreweryDescriptions = {
"Handcrafted pale ales and seasonal IPAs with local ingredients.",
"Traditional lagers and experimental sours in small batches.",
"Award-winning stouts and wildly hoppy blonde ales.",
"Craft brewery specializing in Belgian-style triples and dark "
"porters.",
"Modern brewery blending tradition with bold experimental flavors.",
"Neighborhood-focused taproom pouring crisp pilsners and citrusy "
"pale "
"ales.",
"Small-batch brewery known for barrel-aged releases and smoky "
"lagers.",
"Independent brewhouse pairing farmhouse ales with rotating food "
"pop-ups.",
"Community brewpub making balanced bitters, saisons, and hazy IPAs.",
"Experimental nanobrewery exploring local yeast and regional "
"grains.",
"Family-run brewery producing smooth amber ales and robust porters.",
"Urban brewery crafting clean lagers and bright, fruit-forward "
"sours.",
"Riverfront brewhouse featuring oak-matured ales and seasonal "
"blends.",
"Modern taproom focused on sessionable lagers and classic pub "
"styles.",
"Brewery rooted in tradition with a lineup of malty reds and crisp "
"lagers.",
"Creative brewery offering rotating collaborations and limited "
"draft-only "
"pours.",
"Locally inspired brewery serving approachable ales with bold hop "
"character.",
"Destination taproom known for balanced IPAs and cocoa-rich "
"stouts."};
static constexpr std::array<std::string_view, 18> kUsernames = {
"hopseeker", "malttrail", "yeastwhisper", "lagerlane",
"barrelbound", "foamfinder", "taphunter", "graingeist",
"brewscout", "aleatlas", "caskcompass", "hopsandmaps",
"mashpilot", "pintnomad", "fermentfriend", "stoutsignal",
"sessionwander", "kettlekeeper"};
static constexpr std::array<std::string_view, 18> kBios = {
"Always chasing balanced IPAs and crisp lagers across local taprooms.",
"Weekend brewery explorer with a soft spot for dark, roasty stouts.",
"Documenting tiny brewpubs, fresh pours, and unforgettable beer "
"gardens.",
"Fan of farmhouse ales, food pairings, and long tasting flights.",
"Collecting favorite pilsners one city at a time.",
"Hops-first drinker who still saves room for classic malt-forward "
"styles.",
"Finding hidden tap lists and sharing the best seasonal releases.",
"Brewery road-tripper focused on local ingredients and clean "
"fermentation.",
"Always comparing house lagers and ranking patio pint vibes.",
"Curious about yeast strains, barrel programs, and cellar experiments.",
"Believes every neighborhood deserves a great community taproom.",
"Looking for session beers that taste great from first sip to last.",
"Belgian ale enthusiast who never skips a new saison.",
"Hazy IPA critic with deep respect for a perfectly clear pilsner.",
"Visits breweries for the stories, stays for the flagship pours.",
"Craft beer fan mapping tasting notes and favorite brew routes.",
"Always ready to trade recommendations for underrated local breweries.",
"Keeping a running list of must-try collab releases and tap "
"takeovers."};
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_MOCK_GENERATOR_H_

View File

@@ -7,12 +7,13 @@
#include "data_generation/prompt_formatting/prompt_formatter.h"
class Gemma4JinjaPromptFormatter final : public IPromptFormatter {
public:
Gemma4JinjaPromptFormatter() = default;
~Gemma4JinjaPromptFormatter() override = default;
public:
Gemma4JinjaPromptFormatter() = default;
~Gemma4JinjaPromptFormatter() override = default;
[[nodiscard]] std::string Format(std::string_view system_prompt,
std::string_view user_prompt) const override;
[[nodiscard]] std::string Format(
std::string_view system_prompt,
std::string_view user_prompt) const override;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_PROMPT_FORMATTING_GEMMA4_JINJA_PROMPT_FORMATTER_H_

View File

@@ -5,16 +5,16 @@
#include <string_view>
class IPromptFormatter {
public:
IPromptFormatter() = default;
IPromptFormatter(const IPromptFormatter&) = delete;
IPromptFormatter& operator=(const IPromptFormatter&) = delete;
IPromptFormatter(IPromptFormatter&&) = delete;
IPromptFormatter& operator=(IPromptFormatter&&) = delete;
virtual ~IPromptFormatter() = default;
public:
IPromptFormatter() = default;
IPromptFormatter(const IPromptFormatter&) = delete;
IPromptFormatter& operator=(const IPromptFormatter&) = delete;
IPromptFormatter(IPromptFormatter&&) = delete;
IPromptFormatter& operator=(IPromptFormatter&&) = delete;
virtual ~IPromptFormatter() = default;
[[nodiscard]] virtual std::string Format(
std::string_view system_prompt, std::string_view user_prompt) const = 0;
[[nodiscard]] virtual std::string Format(
std::string_view system_prompt, std::string_view user_prompt) const = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_PROMPT_FORMATTING_PROMPT_FORMATTER_H_

View File

@@ -3,64 +3,128 @@
/**
* @file data_model/generated_models.h
* @brief Generated output models from the pipeline: brewery/user results, enriched data,
* and complete generation results.
* @brief Generated output models from the pipeline: brewery/user results,
* enriched data, and complete generation results.
*/
#include <string>
#include "data_model/models.h"
// ============================================================================
// Generation Output Models
// ============================================================================
/**
* @brief Generated brewery payload.
*/
struct BreweryResult {
/// @brief Brewery display name in English.
std::string name_en;
/**
* @brief Brewery display name in English.
*/
std::string name_en;
/// @brief Brewery description text in English.
std::string description_en;
/**
* @brief Brewery description text in English.
*/
std::string description_en;
/// @brief Brewery display name in the local language.
std::string name_local;
/**
* @brief Brewery display name in the local language.
*/
std::string name_local;
/// @brief Brewery description text in the local language.
std::string description_local;
/**
* @brief Brewery description text in the local language.
*/
std::string description_local;
};
/**
* @brief Generated user profile payload.
*/
struct UserResult {
/// @brief Username handle.
std::string username{};
/**
* @brief First (given) name, copied from the sampled Name -- not
* LLM-invented.
*/
std::string first_name{};
/// @brief Short user biography.
std::string bio{};
/**
* @brief Last (family) name, copied from the sampled Name -- not
* LLM-invented.
*/
std::string last_name{};
/**
* @brief Gender associated with the sampled first name, copied from the
* sampled Name -- not LLM-invented.
*/
std::string gender{};
/**
* @brief Username handle.
*/
std::string username{};
/**
* @brief Short user biography.
*/
std::string bio{};
/**
* @brief Relative check-in/activity weight for this user, used to drive
* a J-curve activity profile in later pipeline phases.
*/
float activity_weight{};
};
// ============================================================================
// Pipeline Data Models
// ============================================================================
/**
* @brief Enriched city data with Wikipedia context.
*/
struct EnrichedCity {
Location location;
std::string region_context{};
City location;
std::string region_context{};
};
/**
* @brief A physical address: the city plus a concrete postal code selected
* from that city's curated postal-code examples.
*/
struct BreweryAddress {
City city;
std::string postal_code{};
};
/**
* @brief A physical address for a generated user: the city plus a concrete
* postal code selected from that city's curated postal-code examples.
*/
struct UserAddress {
City city;
std::string postal_code{};
};
/**
* @brief Helper struct to store generated brewery data.
*/
struct GeneratedBrewery {
Location location;
BreweryResult brewery;
struct BreweryRecord {
BreweryAddress address;
BreweryResult brewery;
};
/**
* @brief Helper struct to store generated user data.
*
* `email` and `date_of_birth` are programmatically generated by the
* orchestrator (never LLM-authored) so a downstream auth-account seeding
* consumer can register real accounts from the pipeline's SQLite export.
*/
struct UserRecord {
UserAddress address;
UserResult user;
std::string email{};
std::string date_of_birth{};
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_GENERATED_MODELS_H_

View File

@@ -10,6 +10,7 @@
#include <boost/program_options.hpp>
#include <cstdint>
#include <filesystem>
#include <functional>
#include <memory>
#include <optional>
#include <string>
@@ -20,126 +21,225 @@ class ILogger;
namespace prog_opts = boost::program_options;
// ============================================================================
// Location Models
// ============================================================================
// City Models
/**
* @brief Canonical location record for city-level generation.
* @brief Curated postal-code metadata for a city, mirroring the
* `postal_code` object in the locations dataset.
*/
struct Location {
/// @brief City name.
std::string city{};
struct PostalCodeSpec {
/**
* @brief Permissive country-wide postal-code format regex, used as the
* validation fallback for a selected example.
*/
std::string country_format_regex{};
/// @brief State or province name.
std::string state_province{};
/**
* @brief City-specific postal-code regexes (more precise than the country
* format).
*/
std::vector<std::string> city_regexes{};
/// @brief ISO 3166-2 subdivision code.
std::string iso3166_2{};
/// @brief Country name.
std::string country{};
/// @brief ISO 3166-1 country code.
std::string iso3166_1{};
/// @brief Local language codes in priority order.
std::vector<std::string> local_languages{};
/// @brief Latitude in decimal degrees.
double latitude{};
/// @brief Longitude in decimal degrees.
double longitude{};
/**
* @brief Concrete example postal codes for this city. The postal code
* service selects from these rather than synthesizing codes.
*/
std::vector<std::string> examples{};
};
/**
* @brief Non-owning brewery location input.
* @brief Canonical city record for city-level generation.
*/
struct BreweryLocation {
/// @brief City name.
std::string_view city_name;
struct City {
std::string city{};
std::string state_province{};
/// @brief Country name.
std::string_view country_name;
/**
* @brief ISO 3166-2 subdivision code.
*/
std::string iso3166_2{};
std::string country{};
/**
* @brief ISO 3166-1 country code.
*/
std::string iso3166_1{};
/**
* @brief Local language codes in priority order.
*/
std::vector<std::string> local_languages{};
/**
* @brief Postal-code metadata (format regex, city regexes, and concrete
* examples) sourced directly from the curated locations dataset.
*/
PostalCodeSpec postal_code{};
};
// Name / Persona Models
/**
* @brief A sampled first/last name pair, with the source forename's gender.
*
* Produced by the SampleName() helper in generate_users.cc.
*/
struct Name {
std::string first_name{};
std::string last_name{};
/**
* @brief Gender associated with the sampled forename (e.g. "M", "F"), as
* reported by the source dataset.
*/
std::string gender{};
};
/**
* @brief A single forename entry from the names-by-country fixture data.
*/
struct ForenameEntry {
/**
* @brief Romanized forename.
*/
std::string name{};
/**
* @brief Gender associated with this forename, as reported by the source
* dataset (e.g. "M", "F").
*/
std::string gender{};
bool operator==(const ForenameEntry& other) const {
return name == other.name && gender == other.gender;
}
};
namespace std {
template <>
struct hash<ForenameEntry> {
size_t operator()(const ForenameEntry& entry) const noexcept {
const size_t name_hash = std::hash<std::string>{}(entry.name);
const size_t gender_hash = std::hash<std::string>{}(entry.gender);
return name_hash ^ (gender_hash << 1);
}
};
} // namespace std
/**
* @brief A persona archetype used to ground LLM-generated user bios.
*/
struct UserPersona {
/**
* @brief Persona display name (e.g. "Hophead Explorer").
*/
std::string name{};
/**
* @brief Short description of the persona's interests and voice.
*/
std::string description{};
/**
* @brief Beer styles this persona gravitates toward.
*/
std::vector<std::string> style_affinities{};
};
// ============================================================================
// Configuration Models
// ============================================================================
/**
* @brief LLM sampling parameters.
*/
struct SamplingOptions {
/// @brief LLM sampling temperature (0.0 to 1.0, higher = more random).
float temperature = 1.0F;
/**
* @brief LLM sampling temperature (higher = more random).
*/
float temperature = 1.0F;
/// @brief LLM nucleus sampling top-p parameter.
float top_p = 0.95F;
/**
* @brief LLM nucleus sampling top-p parameter.
*/
float top_p = 0.95F;
/// @brief LLM top-k sampling parameter.
uint32_t top_k = 64;
/**
* @brief LLM top-k sampling parameter.
*/
uint32_t top_k = 64;
/// @brief Context window size (tokens).
uint32_t n_ctx = 8192;
/**
* @brief Context window size (tokens).
*/
uint32_t n_ctx = 8192;
/// @brief Random seed (-1 for random, otherwise non-negative).
int seed = -1;
/**
* @brief Random seed (-1 for random, otherwise non-negative).
*/
int seed = -1;
/// @brief Number of layers to offload to GPU.
int n_gpu_layers = 0;
/**
* @brief Number of layers to offload to GPU.
*/
int n_gpu_layers = 0;
};
/**
* @brief Configuration for the LLM generator component.
*/
struct GeneratorOptions {
/// @brief Path to the LLM model file (gguf format).
std::filesystem::path model_path;
/**
* @brief Path to the LLM model file (gguf format).
*/
std::filesystem::path model_path;
/// @brief Use mocked generator instead of actual LLM inference.
bool use_mocked = false;
/**
* @brief Use mocked generator instead of actual LLM inference.
*/
bool use_mocked = false;
/// @brief Specific sampling parameters for this generator.
/// If nullopt, the application should use global defaults.
std::optional<SamplingOptions> sampling;
/**
* @brief Specific sampling parameters for this generator.
* If nullopt, the application should use global defaults.
*/
std::optional<SamplingOptions> sampling;
};
/**
* @brief Configuration for the pipeline execution and output.
*/
struct PipelineOptions {
/// @brief Directory for generated artifacts.
std::filesystem::path output_path;
/**
* @brief Directory for generated artifacts.
*/
std::filesystem::path output_path;
/// @brief Directory that contains named prompt files (e.g.
/// BREWERY_GENERATION.md).
std::filesystem::path prompt_dir;
/**
* @brief Directory that contains named prompt files (e.g.
* BREWERY_GENERATION.md).
*/
std::filesystem::path prompt_dir;
/// @brief Path for application logs.
std::filesystem::path log_path;
std::filesystem::path log_path;
/// @brief Number of locations to sample from the dataset
/// More locations -> more users/more breweries
uint32_t location_count;
/**
* @brief Number of locations to sample from the dataset
* More locations -> more users/more breweries
*/
uint32_t location_count;
};
/**
* @brief Root configuration object for the Biergarten pipeline.
*/
struct ApplicationOptions {
GeneratorOptions generator;
PipelineOptions pipeline;
GeneratorOptions generator;
PipelineOptions pipeline;
};
// ============================================================================
// Function Declarations
// ============================================================================
std::optional<ApplicationOptions> ParseArguments(const int argc, char** argv,
std::shared_ptr<ILogger> logger = nullptr);
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger = nullptr);
#endif // BIERGARTEN_PIPELINE_INCLUDES_DATA_MODEL_MODELS_H_

View File

@@ -1,25 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
/**
* @file json_handling/json_loader.h
* @brief Loader API for curated location data.
*/
#include <filesystem>
#include <memory>
#include <vector>
#include "data_model/models.h"
#include "services/logging/logger.h"
/// @brief Loads curated world locations from a JSON file into memory.
class JsonLoader {
public:
/// @brief Parses a JSON array file and returns all location records.
static std::vector<Location> LoadLocations(
const std::filesystem::path& filepath,
std::shared_ptr<ILogger> logger = nullptr);
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

View File

@@ -27,83 +27,84 @@
inline void PrettyPrint(std::ostream& outstream,
boost::json::value const& json_val,
std::string* indent = nullptr) {
std::string str;
if (indent == nullptr) {
indent = &str;
}
switch (json_val.kind()) {
case boost::json::kind::object: {
outstream << "{\n";
indent->append(4, ' ');
auto const& obj = json_val.get_object();
if (!obj.empty()) {
const auto* iter = obj.begin();
for (;;) {
outstream << *indent << boost::json::serialize(iter->key()) << " : ";
PrettyPrint(outstream, iter->value(), indent);
iter = std::next(iter);
if (iter == obj.end()) {
break;
}
std::string str;
if (indent == nullptr) {
indent = &str;
}
switch (json_val.kind()) {
case boost::json::kind::object: {
outstream << "{\n";
indent->append(4, ' ');
auto const& obj = json_val.get_object();
if (!obj.empty()) {
const auto* iter = obj.begin();
for (;;) {
outstream << *indent << boost::json::serialize(iter->key())
<< " : ";
PrettyPrint(outstream, iter->value(), indent);
iter = std::next(iter);
if (iter == obj.end()) {
break;
}
outstream << ",\n";
}
outstream << ",\n";
}
}
outstream << "\n";
indent->resize(indent->size() - 4);
outstream << *indent << "}";
break;
}
case boost::json::kind::array: {
outstream << "[\n";
indent->append(4, ' ');
auto const& arr = json_val.get_array();
if (!arr.empty()) {
const auto* iter = arr.begin();
for (;;) {
outstream << *indent;
PrettyPrint(outstream, *iter, indent);
iter = std::next(iter);
if (iter == arr.end()) {
break;
}
outstream << ",\n";
}
}
outstream << "\n";
indent->resize(indent->size() - 4);
outstream << *indent << "]";
break;
}
case boost::json::kind::string: {
outstream << serialize(json_val.get_string());
break;
}
case boost::json::kind::uint64:
case boost::json::kind::int64:
case boost::json::kind::double_:
outstream << json_val;
break;
case boost::json::kind::bool_:
if (json_val.get_bool()) {
outstream << "true";
} else {
outstream << "false";
}
break;
case boost::json::kind::null:
outstream << "null";
break;
}
if (indent->empty()) {
outstream << "\n";
indent->resize(indent->size() - 4);
outstream << *indent << "}";
break;
}
case boost::json::kind::array: {
outstream << "[\n";
indent->append(4, ' ');
auto const& arr = json_val.get_array();
if (!arr.empty()) {
const auto* iter = arr.begin();
for (;;) {
outstream << *indent;
PrettyPrint(outstream, *iter, indent);
iter = std::next(iter);
if (iter == arr.end()) {
break;
}
outstream << ",\n";
}
}
outstream << "\n";
indent->resize(indent->size() - 4);
outstream << *indent << "]";
break;
}
case boost::json::kind::string: {
outstream << serialize(json_val.get_string());
break;
}
case boost::json::kind::uint64:
case boost::json::kind::int64:
case boost::json::kind::double_:
outstream << json_val;
break;
case boost::json::kind::bool_:
if (json_val.get_bool()) {
outstream << "true";
} else {
outstream << "false";
}
break;
case boost::json::kind::null:
outstream << "null";
break;
}
if (indent->empty()) {
outstream << "\n";
}
}
}
#endif
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_PRETTY_PRINT_H_

View File

@@ -15,18 +15,12 @@
* it alive for application lifetime.
*/
class LlamaBackendState {
public:
/// @brief Initializes global llama backend state.
LlamaBackendState() { llama_backend_init(); }
public:
LlamaBackendState() { llama_backend_init(); }
~LlamaBackendState() { llama_backend_free(); }
/// @brief Cleans up global llama backend state.
~LlamaBackendState() { llama_backend_free(); }
/// @brief Non-copyable type.
LlamaBackendState(const LlamaBackendState&) = delete;
/// @brief Non-copyable type.
LlamaBackendState& operator=(const LlamaBackendState&) = delete;
LlamaBackendState(const LlamaBackendState&) = delete;
LlamaBackendState& operator=(const LlamaBackendState&) = delete;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_LLAMA_BACKEND_STATE_H_

View File

@@ -0,0 +1,50 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
/**
* @file services/curated_data/curated_data_service.h
* @brief Abstraction for loading curated location, persona, and name data.
*/
#include <filesystem>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "data_model/models.h"
using ForenameList = std::vector<ForenameEntry>;
using SurnameList = std::vector<std::string>;
using LocationsList = std::vector<City>;
using PersonasList = std::vector<UserPersona>;
using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;
/**
* @brief Interface for services that load curated data used to seed
* brewery/user generation.
*/
class ICuratedDataService {
public:
ICuratedDataService() = default;
virtual ~ICuratedDataService() = default;
ICuratedDataService(const ICuratedDataService&) = delete;
ICuratedDataService& operator=(const ICuratedDataService&) = delete;
ICuratedDataService(ICuratedDataService&&) = delete;
ICuratedDataService& operator=(ICuratedDataService&&) = delete;
/**
* @brief Loads all curated location records.
*/
virtual const LocationsList& LoadLocations() = 0;
/**
* @brief Loads all curated persona records.
*/
virtual const PersonasList& LoadPersonas() = 0;
virtual const ForenamesByCountryMap& LoadForenamesByCountry() = 0;
virtual const SurnamesByCountryMap& LoadSurnamesByCountry() = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_

View File

@@ -0,0 +1,63 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
/**
* @file json_handling/json_loader.h
* @brief JSON-backed implementation of ICuratedDataService.
*/
#include <filesystem>
#include "data_model/models.h"
#include "services/curated_data/curated_data_service.h"
/**
* @brief File locations for the curated JSON fixtures consumed by
* CuratedJsonDataService.
*/
struct CuratedDataFilePaths {
std::filesystem::path locations_path;
std::filesystem::path personas_path;
std::filesystem::path forenames_path;
std::filesystem::path surnames_path;
};
/**
* @brief Loads curated location, persona, and name data from JSON files.
*/
class CuratedJsonDataService final : public ICuratedDataService {
struct cache {
LocationsList locations;
PersonasList personas;
ForenamesByCountryMap forenames_by_country;
SurnamesByCountryMap surnames_by_country;
cache() = default;
~cache() = default;
};
CuratedDataFilePaths filepaths_;
cache cache_;
public:
explicit CuratedJsonDataService(CuratedDataFilePaths filepaths);
/**
* @brief Parses a JSON array file and returns all location records.
*/
const LocationsList& LoadLocations() override;
/**
* @brief Parses a JSON array file and returns all persona records.
*/
const PersonasList& LoadPersonas() override;
const ForenamesByCountryMap& LoadForenamesByCountry() override;
/**
* @brief Parses a JSON file and returns all the forenames per country.
*/
const SurnamesByCountryMap& LoadSurnamesByCountry() override;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

View File

@@ -0,0 +1,38 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
/**
* @file services/curated_data/mock_curated_data_service.h
* @brief In-memory ICuratedDataService backed by a small fixed dataset, used
* when file-backed curated data is disabled (mock mode).
*/
#include <filesystem>
#include "data_model/models.h"
#include "services/curated_data/curated_data_service.h"
/**
* @brief Curated data service returning a small fixed in-memory dataset in
* place of the JSON fixture files used by JsonLoader.
*/
class MockCuratedDataService final : public ICuratedDataService {
public:
MockCuratedDataService();
const LocationsList& LoadLocations() override;
const PersonasList& LoadPersonas() override;
const ForenamesByCountryMap& LoadForenamesByCountry() override;
const SurnamesByCountryMap& LoadSurnamesByCountry() override;
private:
LocationsList locations_;
PersonasList personas_;
ForenamesByCountryMap forenames_by_country_;
SurnamesByCountryMap surnames_by_country_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_

View File

@@ -14,29 +14,38 @@
* @brief Interface for services that persist generated brewery records.
*/
class IExportService {
public:
IExportService() = default;
public:
IExportService() = default;
virtual ~IExportService() = default;
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~IExportService() = default;
IExportService(const IExportService&) = delete;
IExportService& operator=(const IExportService&) = delete;
IExportService(IExportService&&) = delete;
IExportService& operator=(IExportService&&) = delete;
IExportService(const IExportService&) = delete;
IExportService& operator=(const IExportService&) = delete;
IExportService(IExportService&&) = delete;
IExportService& operator=(IExportService&&) = delete;
/**
* @brief Prepares the export destination for a new run.
*/
virtual void Initialize() = 0;
/// @brief Prepares the export destination for a new run.
virtual void Initialize() = 0;
/**
* @brief Persists one generated brewery record.
*
* @param brewery Generated brewery payload to store.
*/
virtual uint64_t ProcessRecord(const BreweryRecord& brewery) = 0;
/**
* @brief Persists one generated brewery record.
*
* @param brewery Generated brewery payload to store.
*/
virtual uint64_t ProcessRecord(const GeneratedBrewery& brewery) = 0;
/**
* @brief Persists one generated user record.
*
* @param user Generated user payload to store.
*/
virtual uint64_t ProcessRecord(const UserRecord& user) = 0;
/// @brief Finalizes the export destination.
virtual void Finalize() = 0;
/**
* @brief Finalizes the export destination.
*/
virtual void Finalize() = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_EXPORT_SERVICE_H_

View File

@@ -12,7 +12,7 @@
#include <string>
#include <string_view>
#include "sqlite_handle_types.h"
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal {

View File

@@ -12,49 +12,62 @@
#include <unordered_map>
#include "data_model/models.h"
#include "../datetime/date_time_provider.h"
#include "export_service.h"
#include "sqlite_export_service_helpers.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service_helpers.h"
#include "services/datetime/date_time_provider.h"
/**
* @brief Persists generated brewery records into a fresh SQLite database.
*/
class SqliteExportService final : public IExportService {
public:
explicit SqliteExportService(const ApplicationOptions& options);
~SqliteExportService() override;
public:
explicit SqliteExportService(const ApplicationOptions& options);
~SqliteExportService() override;
SqliteExportService(const SqliteExportService&) = delete;
SqliteExportService& operator=(const SqliteExportService&) = delete;
SqliteExportService(SqliteExportService&&) = delete;
SqliteExportService& operator=(SqliteExportService&&) = delete;
SqliteExportService(const SqliteExportService&) = delete;
SqliteExportService& operator=(const SqliteExportService&) = delete;
SqliteExportService(SqliteExportService&&) = delete;
SqliteExportService& operator=(SqliteExportService&&) = delete;
void Initialize() override;
uint64_t ProcessRecord(const GeneratedBrewery& brewery) override;
void Finalize() override;
void Initialize() override;
uint64_t ProcessRecord(const BreweryRecord& brewery) override;
uint64_t ProcessRecord(const UserRecord& user) override;
void Finalize() override;
private:
using SqliteDatabaseHandle =
sqlite_export_service_internal::SqliteDatabaseHandle;
using SqliteStatementHandle =
sqlite_export_service_internal::SqliteStatementHandle;
private:
using SqliteDatabaseHandle =
sqlite_export_service_internal::SqliteDatabaseHandle;
using SqliteStatementHandle =
sqlite_export_service_internal::SqliteStatementHandle;
void InitializeSchema() const;
void PrepareStatements();
void RollbackAndCloseNoThrow() noexcept;
void InitializeSchema() const;
void PrepareStatements();
void RollbackAndCloseNoThrow() noexcept;
[[nodiscard]] std::filesystem::path BuildDatabasePath() const;
[[nodiscard]] static std::string BuildLocationKey(const Location& location);
[[nodiscard]] std::filesystem::path BuildDatabasePath() const;
[[nodiscard]] static std::string BuildCityKey(const City& city);
std::unique_ptr<IDateTimeProvider> date_time_provider_;
std::filesystem::path output_path_;
std::string run_timestamp_utc_;
std::filesystem::path database_path_;
SqliteDatabaseHandle db_handle_;
SqliteStatementHandle insert_location_stmt_;
SqliteStatementHandle insert_brewery_stmt_;
bool transaction_open_ = false;
std::unordered_map<std::string, sqlite3_int64> location_cache_;
/**
* @brief Returns the row id for @p city, inserting it first if it has
* not already been seen during this run.
*
* Shared by both ProcessRecord() overloads so brewery and user addresses
* referencing the same city resolve to the same row.
*/
[[nodiscard]] sqlite3_int64 ResolveCityId(const City& city);
std::unique_ptr<IDateTimeProvider> date_time_provider_;
std::filesystem::path output_path_;
std::string run_timestamp_utc_;
std::filesystem::path database_path_;
SqliteDatabaseHandle db_handle_;
SqliteStatementHandle insert_city_stmt_;
SqliteStatementHandle insert_brewery_stmt_;
SqliteStatementHandle insert_brewery_address_stmt_;
SqliteStatementHandle insert_user_stmt_;
SqliteStatementHandle insert_user_address_stmt_;
bool transaction_open_ = false;
std::unordered_map<std::string, sqlite3_int64> city_cache_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_H_

View File

@@ -3,8 +3,8 @@
/* Umbrella header for backward compatibility. */
#include "sqlite_connection_helpers.h"
#include "sqlite_handle_types.h"
#include "sqlite_statement_helpers.h"
#include "services/database/sqlite_connection_helpers.h"
#include "services/database/sqlite_handle_types.h"
#include "services/database/sqlite_statement_helpers.h"
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_

View File

@@ -13,22 +13,24 @@
namespace sqlite_export_service_internal {
struct SqliteDatabaseDeleter {
void operator()(sqlite3* handle) const noexcept;
void operator()(sqlite3* handle) const noexcept;
};
struct SqliteStatementDeleter {
void operator()(sqlite3_stmt* statement) const noexcept;
void operator()(sqlite3_stmt* statement) const noexcept;
};
using SqliteDatabaseHandle = std::unique_ptr<sqlite3, SqliteDatabaseDeleter>;
using SqliteStatementHandle =
std::unique_ptr<sqlite3_stmt, SqliteStatementDeleter>;
// Represents a parameter that is bound to a prepared SQLite statement.
// N.B. indices are 1 based.
template <typename T>
struct BindParam {
int index;
T value;
std::string_view action;
struct BoundParam {
int index;
T value;
std::string_view action;
};
} // namespace sqlite_export_service_internal

View File

@@ -13,13 +13,13 @@
#include <string_view>
#include <vector>
#include "sqlite_handle_types.h"
#include "services/database/sqlite_handle_types.h"
namespace sqlite_export_service_internal {
inline constexpr std::string_view kCreateLocationsTableSql = R"sql(
inline constexpr std::string_view kCreateCitiesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS locations (
CREATE TABLE IF NOT EXISTS cities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
city TEXT NOT NULL,
state_province TEXT NOT NULL,
@@ -27,9 +27,9 @@ CREATE TABLE IF NOT EXISTS locations (
country TEXT NOT NULL,
iso3166_1 TEXT NOT NULL,
local_languages_json TEXT NOT NULL,
latitude REAL NOT NULL,
longitude REAL NOT NULL,
UNIQUE(city, state_province, iso3166_2, country, latitude, longitude)
postal_code_country_format_regex TEXT NOT NULL,
postal_code_city_regex_json TEXT NOT NULL,
UNIQUE(city, state_province, iso3166_2, country)
);
)sql";
@@ -38,55 +38,159 @@ inline constexpr std::string_view kCreateBreweriesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS breweries (
id INTEGER PRIMARY KEY AUTOINCREMENT,
location_id INTEGER NOT NULL,
name_en TEXT NOT NULL,
description_en TEXT NOT NULL,
name_local TEXT NOT NULL,
description_local TEXT NOT NULL,
FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
description_local TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id);
)sql";
inline constexpr std::string_view kInsertLocationSql = R"sql(
INSERT INTO locations (
inline constexpr std::string_view kCreateBreweryAddressesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS brewery_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
brewery_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_brewery_id
ON brewery_addresses(brewery_id);
CREATE INDEX IF NOT EXISTS idx_brewery_addresses_city_id
ON brewery_addresses(city_id);
)sql";
inline constexpr std::string_view kCreateUsersTableSql = R"sql(
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
first_name TEXT NOT NULL,
last_name TEXT NOT NULL,
gender TEXT NOT NULL,
username TEXT NOT NULL,
bio TEXT NOT NULL,
activity_weight REAL NOT NULL,
email TEXT NOT NULL UNIQUE,
date_of_birth TEXT NOT NULL
);
)sql";
inline constexpr std::string_view kCreateUserAddressesTableSql = R"sql(
CREATE TABLE IF NOT EXISTS user_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_user_addresses_user_id
ON user_addresses(user_id);
CREATE INDEX IF NOT EXISTS idx_user_addresses_city_id
ON user_addresses(city_id);
)sql";
inline constexpr std::string_view kInsertCitySql = R"sql(
INSERT INTO cities (
city,
state_province,
iso3166_2,
country,
iso3166_1,
local_languages_json,
latitude,
longitude
postal_code_country_format_regex,
postal_code_city_regex_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertBrewerySql = R"sql(
INSERT INTO breweries (
location_id,
name_en,
description_en,
name_local,
description_local
) VALUES (?, ?, ?, ?, ?);
) VALUES (?, ?, ?, ?);
)sql";
inline constexpr int kLocationCityBindIndex = 1;
inline constexpr int kLocationStateProvinceBindIndex = 2;
inline constexpr int kLocationIso31662BindIndex = 3;
inline constexpr int kLocationCountryBindIndex = 4;
inline constexpr int kLocationIso31661BindIndex = 5;
inline constexpr int kLocationLanguagesBindIndex = 6;
inline constexpr int kLocationLatitudeBindIndex = 7;
inline constexpr int kLocationLongitudeBindIndex = 8;
inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
INSERT INTO brewery_addresses (
brewery_id,
city_id,
postal_code
) VALUES (?, ?, ?);
)sql";
inline constexpr int kBreweryLocationIdBindIndex = 1;
inline constexpr int kBreweryEnglishNameBindIndex = 2;
inline constexpr int kBreweryEnglishDescriptionBindIndex = 3;
inline constexpr int kBreweryLocalNameBindIndex = 4;
inline constexpr int kBreweryLocalDescriptionBindIndex = 5;
inline constexpr std::string_view kInsertUserSql = R"sql(
INSERT INTO users (
first_name,
last_name,
gender,
username,
bio,
activity_weight,
email,
date_of_birth
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
)sql";
inline constexpr std::string_view kInsertUserAddressSql = R"sql(
INSERT INTO user_addresses (
user_id,
city_id,
postal_code
) VALUES (?, ?, ?);
)sql";
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
// placeholder order in the SQL above.
enum CityBindIndex {
kCityCityBindIndex = 1,
kCityStateProvinceBindIndex,
kCityIso31662BindIndex,
kCityCountryBindIndex,
kCityIso31661BindIndex,
kCityLanguagesBindIndex,
kCityPostalCodeCountryFormatRegexBindIndex,
kCityPostalCodeCityRegexJsonBindIndex,
};
enum BreweryBindIndex {
kBreweryEnglishNameBindIndex = 1,
kBreweryEnglishDescriptionBindIndex,
kBreweryLocalNameBindIndex,
kBreweryLocalDescriptionBindIndex,
};
enum BreweryAddressBindIndex {
kBreweryAddressBreweryIdBindIndex = 1,
kBreweryAddressCityIdBindIndex,
kBreweryAddressPostalCodeBindIndex,
};
enum UserBindIndex {
kUserFirstNameBindIndex = 1,
kUserLastNameBindIndex,
kUserGenderBindIndex,
kUserUsernameBindIndex,
kUserBioBindIndex,
kUserActivityWeightBindIndex,
kUserEmailBindIndex,
kUserDateOfBirthBindIndex,
};
enum UserAddressBindIndex {
kUserAddressUserIdBindIndex = 1,
kUserAddressCityIdBindIndex,
kUserAddressPostalCodeBindIndex,
};
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
std::string_view sql,
@@ -95,13 +199,13 @@ SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
void ResetStatement(SqliteStatementHandle& statement);
void Bind(const SqliteStatementHandle& statement,
const BindParam<std::string_view>& param);
const BoundParam<std::string_view>& param);
void Bind(const SqliteStatementHandle& statement,
const BindParam<double>& param);
const BoundParam<double>& param);
void Bind(const SqliteStatementHandle& statement,
const BindParam<sqlite3_int64>& param);
const BoundParam<sqlite3_int64>& param);
void StepStatement(const SqliteDatabaseHandle& db_handle,
const SqliteStatementHandle& statement,

View File

@@ -15,52 +15,54 @@
/**
* @brief Interface for UTC timestamp providers.
*
* @todo Yagni, consider removing this and just use SystemDateTimeProvider
* directly
*/
class IDateTimeProvider {
public:
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~IDateTimeProvider() = default;
public:
virtual ~IDateTimeProvider() = default;
IDateTimeProvider() = default;
IDateTimeProvider(const IDateTimeProvider&) = delete;
IDateTimeProvider& operator=(const IDateTimeProvider&) = delete;
IDateTimeProvider(IDateTimeProvider&&) = delete;
IDateTimeProvider& operator=(IDateTimeProvider&&) = delete;
IDateTimeProvider() = default;
IDateTimeProvider(const IDateTimeProvider&) = delete;
IDateTimeProvider& operator=(const IDateTimeProvider&) = delete;
IDateTimeProvider(IDateTimeProvider&&) = delete;
IDateTimeProvider& operator=(IDateTimeProvider&&) = delete;
/**
* @brief Returns the current UTC timestamp in a file-safe format.
*
* @return UTC timestamp string.
*/
virtual std::string GetUtcTimestamp() = 0;
/**
* @brief Returns the current UTC timestamp in a file-safe format.
*
* @return UTC timestamp string.
*/
virtual std::string GetUtcTimestamp() = 0;
};
/**
* @brief Timestamp provider backed by the system clock.
*/
class SystemDateTimeProvider final : public IDateTimeProvider {
public:
std::string GetUtcTimestamp() override {
constexpr int kFractionalSecondWidth = 6;
public:
std::string GetUtcTimestamp() override {
constexpr int kFractionalSecondWidth = 6;
const auto now = std::chrono::system_clock::now();
const auto now_time = std::chrono::system_clock::to_time_t(now);
const std::tm* utc_time_ptr = std::gmtime(&now_time);
if (utc_time_ptr == nullptr) {
throw std::runtime_error("Failed to format UTC timestamp");
}
const auto now = std::chrono::system_clock::now();
const auto now_time = std::chrono::system_clock::to_time_t(now);
const std::tm* utc_time_ptr = std::gmtime(&now_time);
if (utc_time_ptr == nullptr) {
throw std::runtime_error("Failed to format UTC timestamp");
}
const auto fractional_seconds =
std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()) %
std::chrono::seconds(1);
const auto fractional_seconds =
std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()) %
std::chrono::seconds(1);
std::ostringstream output;
output << std::put_time(utc_time_ptr, "%Y-%m-%dT%H-%M-%S");
output << '.' << std::setw(kFractionalSecondWidth) << std::setfill('0')
<< fractional_seconds.count() << 'Z';
return output.str();
}
std::ostringstream output;
output << std::put_time(utc_time_ptr, "%Y-%m-%dT%H-%M-%S");
output << '.' << std::setw(kFractionalSecondWidth) << std::setfill('0')
<< fractional_seconds.count() << 'Z';
return output.str();
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATETIME_DATE_TIME_PROVIDER_H_

View File

@@ -8,28 +8,28 @@
* @brief Simple timer utility for measuring elapsed time.
*/
class Timer {
std::chrono::steady_clock::time_point start_time =
std::chrono::steady_clock::now();
std::chrono::steady_clock::time_point start_time =
std::chrono::steady_clock::now();
public:
Timer(const Timer&) = delete;
Timer& operator=(const Timer&) = delete;
Timer(Timer&&) = delete;
Timer& operator=(Timer&&) = delete;
Timer() = default;
~Timer() = default;
public:
Timer(const Timer&) = delete;
Timer& operator=(const Timer&) = delete;
Timer(Timer&&) = delete;
Timer& operator=(Timer&&) = delete;
Timer() = default;
~Timer() = default;
[[nodiscard]] int64_t Elapsed() const {
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - start_time)
.count();
}
[[nodiscard]] int64_t Elapsed() const {
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - start_time)
.count();
}
[[nodiscard]] int64_t Reset() {
auto previous_elapsed = Elapsed();
start_time = std::chrono::steady_clock::now();
return previous_elapsed;
}
[[nodiscard]] int64_t Reset() {
auto previous_elapsed = Elapsed();
start_time = std::chrono::steady_clock::now();
return previous_elapsed;
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATETIME_TIMER_H_

View File

@@ -14,17 +14,16 @@
* @brief Interface for services that can enrich a location with context.
*/
class IEnrichmentService {
public:
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~IEnrichmentService() = default;
public:
virtual ~IEnrichmentService() = default;
/**
* @brief Resolves contextual enrichment for a location.
*
* @param loc Location to enrich.
* @return Context text, or an empty string if unavailable.
*/
virtual std::string GetLocationContext(const Location& loc) = 0;
/**
* @brief Resolves contextual enrichment for a location.
*
* @param loc City to enrich.
* @return Context text, or an empty string if unavailable.
*/
virtual std::string GetLocationContext(const City& loc) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_ENRICHMENT_SERVICE_H_

View File

@@ -1,17 +1,20 @@
//
// Created by aaronpo on 13/05/2026.
//
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
/**
* @file services/enrichment/mock_enrichment.h
* @brief No-op IEnrichmentService used when network enrichment is disabled.
*/
#include <string>
#include "enrichment_service.h"
#include "services/enrichment/enrichment_service.h"
/**
* @brief Enrichment service that returns no context for any location.
*/
class MockEnrichmentService final : public IEnrichmentService {
public:
std::string GetLocationContext(const Location& /*loc*/) override {
return {};
}
public:
std::string GetLocationContext(const City& /*loc*/) override { return {}; }
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_

View File

@@ -11,26 +11,38 @@
#include <string_view>
#include <unordered_map>
#include "enrichment_service.h"
#include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h"
#include "web_client/web_client.h"
/// @brief Provides Wikipedia summary lookups backed by cached raw extracts.
/**
* @brief Provides Wikipedia summary lookups backed by cached raw extracts.
*/
class WikipediaEnrichmentService final : public IEnrichmentService {
public:
/// @brief Creates a new Wikipedia service with the provided web client.
explicit WikipediaEnrichmentService(std::unique_ptr<WebClient> client,
std::shared_ptr<ILogger> logger);
public:
/**
* @brief Creates a new Wikipedia service with the provided web client.
*/
explicit WikipediaEnrichmentService(std::unique_ptr<WebClient> client,
std::shared_ptr<ILogger> logger);
/// @brief Returns the Wikipedia-derived context for a location.
[[nodiscard]] std::string GetLocationContext(const Location& loc) override;
/**
* @brief Returns the Wikipedia-derived context for a location.
*/
[[nodiscard]] std::string GetLocationContext(const City& loc) override;
private:
std::string FetchExtract(std::string_view query);
std::unique_ptr<WebClient> client_;
std::shared_ptr<ILogger> logger_;
/// @brief Canonical cache for raw Wikipedia query extracts.
std::unordered_map<std::string, std::string> extract_cache_;
private:
std::string FetchExtract(std::string_view query);
std::unique_ptr<WebClient> client_;
std::shared_ptr<ILogger> logger_;
/**
* @brief Cache for raw Wikipedia query extracts, keyed by query string.
*
* GetLocationContext() always queries "brewing" and reuses "beer in
* {country}" for every city in that country, so caching avoids refetching
* the same extract across locations in a run.
*/
std::unordered_map<std::string, std::string> extract_cache_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_WIKIPEDIA_SERVICE_H_

View File

@@ -22,32 +22,32 @@
* and exit once the channel has been closed and drained.
*/
class LogDispatcher {
public:
/**
* @brief Construct a log dispatcher.
*
* @param channel Reference to the bounded channel used for log retrieval.
*/
explicit LogDispatcher(BoundedChannel<LogEntry>& channel);
public:
/**
* @brief Construct a log dispatcher.
*
* @param channel Reference to the bounded channel used for log retrieval.
*/
explicit LogDispatcher(BoundedChannel<LogEntry>& channel);
LogDispatcher(const LogDispatcher&) = delete;
LogDispatcher& operator=(const LogDispatcher&) = delete;
LogDispatcher(LogDispatcher&&) = delete;
LogDispatcher& operator=(LogDispatcher&&) = delete;
~LogDispatcher() = default;
LogDispatcher(const LogDispatcher&) = delete;
LogDispatcher& operator=(const LogDispatcher&) = delete;
LogDispatcher(LogDispatcher&&) = delete;
LogDispatcher& operator=(LogDispatcher&&) = delete;
~LogDispatcher() = default;
/**
* @brief Drain the channel and forward entries to spdlog.
*
* Intended to be called once on a dedicated thread. The loop returns after
* the channel has been closed and all queued entries have been processed.
*/
void Run();
/**
* @brief Drain the channel and forward entries to spdlog.
*
* Intended to be called once on a dedicated thread. The loop returns after
* the channel has been closed and all queued entries have been processed.
*/
void Run();
private:
BoundedChannel<LogEntry>& channel_;
private:
BoundedChannel<LogEntry>& channel_;
static spdlog::level::level_enum ToSpdlogLevel(LogLevel level);
static spdlog::level::level_enum ToSpdlogLevel(LogLevel level);
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_DISPATCHER_H_

View File

@@ -20,10 +20,22 @@
* @brief Severity levels supported by the logging infra.
*/
enum class LogLevel {
Debug, ///< Development/debugging information.
Info, ///< General informational messages.
Warn, ///< Warning conditions.
Error, ///< Error conditions.
/**
* @brief Development/debugging information.
*/
Debug,
/**
* @brief General informational messages.
*/
Info,
/**
* @brief Warning conditions.
*/
Warn,
/**
* @brief Error conditions.
*/
Error,
};
/**
@@ -34,23 +46,49 @@ enum class LogLevel {
* pipeline that emitted it.
*/
enum class PipelinePhase {
Startup, ///< Initialization and validation.
UserGeneration, ///< User profile generation.
BreweryAndBeerGeneration, ///< Brewery and beer data generation.
CheckinGeneration, ///< Checkin (visit) record generation.
RatingGeneration, ///< Rating and review generation.
FollowGeneration, ///< Follow relationship generation.
Teardown, ///< Finalization and cleanup.
/**
* @brief Initialization and validation.
*/
Startup,
/**
* @brief City/context enrichment (e.g. Wikipedia lookups).
*/
Enrichment,
/**
* @brief User profile generation.
*/
UserGeneration,
/**
* @brief Brewery and beer data generation.
*/
BreweryAndBeerGeneration,
/**
* @brief Checkin (visit) record generation.
*/
CheckinGeneration,
/**
* @brief Rating and review generation.
*/
RatingGeneration,
/**
* @brief Follow relationship generation.
*/
FollowGeneration,
/**
* @brief Finalization and cleanup.
*/
Teardown,
};
/**
* @struct LogDTO
* @brief User-provided subset of log fields. Used to capture call-site info transparently.
* @brief User-provided subset of log fields. Used to capture call-site info
* transparently.
*/
struct LogDTO {
LogLevel level;
PipelinePhase phase;
std::string message;
LogLevel level;
PipelinePhase phase;
std::string message;
};
/**
@@ -64,25 +102,35 @@ struct LogDTO {
* before the entry is dispatched.
*/
struct LogEntry {
/// @brief Timestamp when the entry was created.
std::chrono::system_clock::time_point timestamp{};
/**
* @brief Timestamp when the entry was created.
*/
std::chrono::system_clock::time_point timestamp{};
/// @brief Source location where the log call was made.
std::source_location origin{};
/**
* @brief Source location where the log call was made.
*/
std::source_location origin{};
/// @brief Thread responsible for emitting the log.
std::thread::id thread_id{};
/**
* @brief Thread responsible for emitting the log.
*/
std::thread::id thread_id{};
/**
* @brief Severity level of this entry.
*/
LogLevel level;
/// @brief Severity level of this entry.
LogLevel level;
/// @brief Pipeline phase associated with the entry.
PipelinePhase phase;
/// @brief Log message text.
std::string message;
/**
* @brief Pipeline phase associated with the entry.
*/
PipelinePhase phase;
/**
* @brief Log message text.
*/
std::string message;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_

View File

@@ -6,8 +6,8 @@
* a bounded channel for later processing by the dispatcher.
*/
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
#include <string_view>
@@ -23,31 +23,31 @@
* remain valid for the lifetime of the producer.
*/
class LogProducer final : public ILogger {
public:
/**
* @brief Construct a channel-backed producer.
*
* @param channel Reference to the bounded channel used for log transfer.
*/
explicit LogProducer(BoundedChannel<LogEntry>& channel);
public:
/**
* @brief Construct a channel-backed producer.
*
* @param channel Reference to the bounded channel used for log transfer.
*/
explicit LogProducer(BoundedChannel<LogEntry>& channel);
LogProducer(const LogProducer&) = delete;
LogProducer& operator=(const LogProducer&) = delete;
LogProducer(LogProducer&&) = delete;
LogProducer& operator=(LogProducer&&) = delete;
LogProducer(const LogProducer&) = delete;
LogProducer& operator=(const LogProducer&) = delete;
LogProducer(LogProducer&&) = delete;
LogProducer& operator=(LogProducer&&) = delete;
~LogProducer() override = default;
~LogProducer() override = default;
/**
* @brief Queue a log message for asynchronous processing.
*
* Blocks while the channel applies backpressure. This blocking behavior
* under heavy load is an accepted trade-off for simplicity.
*/
void DoLog(LogEntry log_entry) override;
/**
* @brief Queue a log message for asynchronous processing.
*
* Blocks while the channel applies backpressure. This blocking behavior
* under heavy load is an accepted trade-off for simplicity.
*/
void DoLog(LogEntry log_entry) override;
private:
BoundedChannel<LogEntry>& channel_;
private:
BoundedChannel<LogEntry>& channel_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_

View File

@@ -23,42 +23,43 @@
* by the composition root and injected into services that emit diagnostics.
*/
class ILogger {
public:
ILogger() = default;
ILogger(const ILogger&) = delete;
ILogger& operator=(const ILogger&) = delete;
ILogger(ILogger&&) = delete;
ILogger& operator=(ILogger&&) = delete;
virtual ~ILogger() = default;
public:
ILogger() = default;
ILogger(const ILogger&) = delete;
ILogger& operator=(const ILogger&) = delete;
ILogger(ILogger&&) = delete;
ILogger& operator=(ILogger&&) = delete;
virtual ~ILogger() = default;
/**
* @brief Submit a log message to the logging subsystem.
*
* @param payload User-provided log data (level, phase, message).
* @param origin Auto-captured source location of the call site.
*/
void Log(LogDTO payload,
std::source_location origin = std::source_location::current(),
std::chrono::system_clock::time_point timestamp = std::chrono::system_clock::now(),
std::thread::id thread_id = std::this_thread::get_id()) {
LogEntry entry;
entry.timestamp = timestamp;
entry.thread_id = thread_id;
entry.level = payload.level;
entry.phase = payload.phase;
entry.message = std::move(payload.message);
entry.origin = origin;
DoLog(std::move(entry));
}
/**
* @brief Submit a log message to the logging subsystem.
*
* @param payload User-provided log data (level, phase, message).
* @param origin Auto-captured source location of the call site.
*/
void Log(LogDTO payload,
std::source_location origin = std::source_location::current(),
std::chrono::system_clock::time_point timestamp =
std::chrono::system_clock::now(),
std::thread::id thread_id = std::this_thread::get_id()) {
LogEntry entry;
entry.timestamp = timestamp;
entry.thread_id = thread_id;
entry.level = payload.level;
entry.phase = payload.phase;
entry.message = std::move(payload.message);
entry.origin = origin;
DoLog(std::move(entry));
}
protected:
/**
* @brief Underlying implementation to transport the log entry.
*
* Implementations must be thread-safe as DoLog can be called concurrently
* from multiple worker threads.
*/
virtual void DoLog(LogEntry log_entry) = 0;
protected:
/**
* @brief Underlying implementation to transport the log entry.
*
* Implementations must be thread-safe as DoLog can be called concurrently
* from multiple worker threads.
*/
virtual void DoLog(LogEntry log_entry) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOGGER_H_

View File

@@ -0,0 +1,80 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/mock_postal_code_service.h
* @brief Data-driven postal code service that selects the first curated
* example for a city and validates it against the city's postal-code regex.
*/
#include <format>
#include <regex>
#include <stdexcept>
#include <string>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service backed entirely by the curated locations
* dataset: it returns the city's first example postal code after checking it
* against the curated postal-code regex.
*
* A precise city regex is preferred when present; otherwise the permissive
* country format regex is used as the fallback. A city with no examples, or an
* example that fails validation, throws so the caller can skip the city.
*/
class MockPostalCodeService final : public IPostalCodeService {
public:
std::string GeneratePostalCode(const City& city) override {
const PostalCodeSpec& spec = city.postal_code;
if (spec.examples.empty()) {
throw std::runtime_error(std::format(
"No postal-code examples available for city '{}'", city.city));
}
const std::string& candidate = spec.examples.front();
if (!MatchesAnyPattern(candidate, spec)) {
throw std::runtime_error(
std::format("Postal code '{}' for city '{}' does not match the "
"curated postal-code regex",
candidate, city.city));
}
return candidate;
}
private:
/**
* @brief Returns true if @p candidate matches any city regex, or the
* country format regex when no city regex matches. When the spec carries no
* usable regex at all, validation is skipped (returns true).
*/
static bool MatchesAnyPattern(const std::string& candidate,
const PostalCodeSpec& spec) {
bool has_pattern = false;
for (const std::string& pattern : spec.city_regexes) {
if (pattern.empty()) {
continue;
}
has_pattern = true;
if (std::regex_match(candidate, std::regex(pattern))) {
return true;
}
}
if (!spec.country_format_regex.empty()) {
has_pattern = true;
if (std::regex_match(candidate,
std::regex(spec.country_format_regex))) {
return true;
}
}
return !has_pattern;
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_

View File

@@ -0,0 +1,35 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/postal_code_service.h
* @brief Abstraction for resolving a postal code for a city.
*/
#include <string>
#include "data_model/models.h"
/**
* @brief Interface for services that resolve a postal code for a city.
*/
class IPostalCodeService {
public:
IPostalCodeService() = default;
virtual ~IPostalCodeService() = default;
IPostalCodeService(const IPostalCodeService&) = delete;
IPostalCodeService& operator=(const IPostalCodeService&) = delete;
IPostalCodeService(IPostalCodeService&&) = delete;
IPostalCodeService& operator=(IPostalCodeService&&) = delete;
/**
* @brief Resolves a postal code for @p city.
*
* @param city City whose curated postal-code examples supply the result.
* @return A postal code string, or an empty string if none is available.
*/
virtual std::string GeneratePostalCode(const City& city) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_

View File

@@ -0,0 +1,32 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/xeger_postal_code_service.h
* @brief Postal code service that generates a fresh, format-conformant code
* per call from the curated regex patterns, instead of returning a fixed
* example.
*/
#include <random>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service using regex-driven generation (xeger).
*
* Picks among the city's curated `city_regexes` (falls back to
* `country_format_regex`), then generates a random string matching that pattern
* via GenerateStringFromRegex().
*/
class XegerPostalCodeService final : public IPostalCodeService {
public:
XegerPostalCodeService();
std::string GeneratePostalCode(const City& city) override;
private:
std::mt19937 rng_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_

View File

@@ -24,22 +24,22 @@
* @brief Interface for loading named prompt files.
*/
class IPromptDirectory {
public:
IPromptDirectory() = default;
IPromptDirectory(const IPromptDirectory&) = delete;
IPromptDirectory& operator=(const IPromptDirectory&) = delete;
IPromptDirectory(IPromptDirectory&&) = delete;
IPromptDirectory& operator=(IPromptDirectory&&) = delete;
virtual ~IPromptDirectory() = default;
public:
IPromptDirectory() = default;
IPromptDirectory(const IPromptDirectory&) = delete;
IPromptDirectory& operator=(const IPromptDirectory&) = delete;
IPromptDirectory(IPromptDirectory&&) = delete;
IPromptDirectory& operator=(IPromptDirectory&&) = delete;
virtual ~IPromptDirectory() = default;
/**
* @brief Loads the prompt associated with @p key.
*
* @param key Logical prompt key, e.g. "BREWERY_GENERATION".
* @return Prompt text.
* @throws std::runtime_error if the prompt file cannot be found or read.
*/
[[nodiscard]] virtual std::string Load(std::string_view key) = 0;
/**
* @brief Loads the prompt associated with @p key.
*
* @param key Logical prompt key, e.g. "BREWERY_GENERATION".
* @return Prompt text.
* @throws std::runtime_error if the prompt file cannot be found or read.
*/
[[nodiscard]] virtual std::string Load(std::string_view key) = 0;
};
/**
@@ -50,33 +50,33 @@ class IPromptDirectory {
* at construction time; individual file absence is reported lazily at Load().
*/
class PromptDirectory final : public IPromptDirectory {
public:
/**
* @brief Constructs a PromptDirectory rooted at @p prompt_dir.
*
* @param prompt_dir Absolute or relative path to the prompt directory.
* @throws std::runtime_error if @p prompt_dir does not exist or is not a
* directory.
*/
explicit PromptDirectory(const std::filesystem::path& prompt_dir);
PromptDirectory(const std::filesystem::path& prompt_dir,
std::shared_ptr<ILogger> logger);
public:
/**
* @brief Constructs a PromptDirectory rooted at @p prompt_dir.
*
* @param prompt_dir Absolute or relative path to the prompt directory.
* @throws std::runtime_error if @p prompt_dir does not exist or is not a
* directory.
*/
explicit PromptDirectory(const std::filesystem::path& prompt_dir);
PromptDirectory(const std::filesystem::path& prompt_dir,
std::shared_ptr<ILogger> logger);
/**
* @brief Loads the prompt for @p key, caching the result.
*
* Maps @p key → <prompt_dir>/<key>.md.
*
* @param key Logical prompt key.
* @return Prompt text.
* @throws std::runtime_error if the file does not exist or is empty.
*/
[[nodiscard]] std::string Load(std::string_view key) override;
/**
* @brief Loads the prompt for @p key, caching the result.
*
* Maps @p key → <prompt_dir>/<key>.md.
*
* @param key Logical prompt key.
* @return Prompt text.
* @throws std::runtime_error if the file does not exist or is empty.
*/
[[nodiscard]] std::string Load(std::string_view key) override;
private:
std::filesystem::path prompt_dir_;
std::shared_ptr<ILogger> logger_;
std::unordered_map<std::string, std::string> cache_;
private:
std::filesystem::path prompt_dir_;
std::shared_ptr<ILogger> logger_;
std::unordered_map<std::string, std::string> cache_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_PROMPTING_PROMPT_DIRECTORY_H_

View File

@@ -0,0 +1,38 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_
#include <random>
#include <string>
#include <string_view>
/**
* @brief Generates a random string matching @p pattern.
*
* Supported syntax:
* - literals
* - `.`
* - alternation `|`
* - groups `(...)` and non-capturing groups `(?:...)`
* - quantifiers `*` `+` `?` `{n}` `{n,}` `{n,m}`
* - character classes `[...]` / `[^...]`
* - shorthand escapes `\d \D \w \W \s \S \n \t \r`
*
* Not supported: lookaround, backreferences, and lazy quantifiers.
*
* @param pattern Regular expression to generate a matching
* string for.
* @param rng Random source driving alternation/quantifier/
* class choices.
* @param unbounded_repeat_cap Extra repeats allowed on top of the declared
* minimum for unbounded quantifiers
* (`*`, `+`, `{n,}`).
*
* @return A string satisfying `std::regex_match(result, std::regex(pattern))`
* for any @p pattern within the supported syntax subset.
*
* @throws std::runtime_error if @p pattern cannot be parsed.
*/
std::string GenerateStringFromRegex(std::string_view pattern, std::mt19937& rng,
int unbounded_repeat_cap = 8);
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_H_

View File

@@ -0,0 +1,201 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_
/**
* @file services/xeger/xeger_internal.h
* @brief Declarations shared by the xeger parser and generator.
*
* 1. Parser Turns the pattern text into an AST (Literal, AnyChar,
* CharClass, Concat, Alternation, Repeat nodes).
* 2. Visitor Double-dispatch contract -- each AST-walking
* operation implements this.
* 3. XegerGenerator a Visitor that emits a random character wherever the
* AST says a character belongs, respecting alternation
* choices and repeat counts.
*/
#include <memory>
#include <optional>
#include <random>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xeger_internal {
// Regex syntax characters
constexpr char kAnchorStart = '^';
constexpr char kAnchorEnd = '$';
constexpr char kAlternation = '|';
constexpr char kGroupOpen = '(';
constexpr char kGroupClose = ')';
constexpr char kNonCapturingMarker = '?';
constexpr char kNonCapturingColon = ':';
constexpr char kClassOpen = '[';
constexpr char kClassClose = ']';
constexpr char kClassNegate = '^';
constexpr char kClassRange = '-';
constexpr char kAnyChar = '.';
constexpr char kEscape = '\\';
constexpr char kStar = '*';
constexpr char kPlus = '+';
constexpr char kOptional = '?';
constexpr char kBraceOpen = '{';
constexpr char kBraceClose = '}';
constexpr char kBraceComma = ',';
// AST
struct Literal;
struct AnyChar;
struct CharClass;
struct Concat;
struct Alternation;
struct Repeat;
struct Visitor {
virtual void Visit(const Literal&) = 0;
virtual void Visit(const AnyChar&) = 0;
virtual void Visit(const CharClass&) = 0;
virtual void Visit(const Concat&) = 0;
virtual void Visit(const Alternation&) = 0;
virtual void Visit(const Repeat&) = 0;
virtual ~Visitor() = default;
};
struct Node {
virtual void Accept(Visitor&) const = 0;
virtual ~Node() = default;
};
using NodePtr = std::unique_ptr<Node>;
template <typename Derived>
struct Visitable : Node {
void Accept(Visitor& visitor) const override {
visitor.Visit(static_cast<const Derived&>(*this));
}
};
// One fixed character, e.g. 'a'.
struct Literal : Visitable<Literal> {
char ch;
explicit Literal(const char c) : ch(c) {}
};
// The "." wildcard: any single printable character.
struct AnyChar : Visitable<AnyChar> {};
// A set of allowed characters, stored as inclusive ranges. A single
// character 'x' is the range {'x', 'x'}. `negated` flips the meaning to
// "any character NOT listed" (i.e. `[^...]`).
struct CharClass : Visitable<CharClass> {
std::vector<std::pair<char, char>> ranges;
bool negated = false;
};
// "Do these children one after another."
struct Concat : Visitable<Concat> {
std::vector<NodePtr> parts;
};
// "Pick exactly one of these options at random" -- the `|` operator.
struct Alternation : Visitable<Alternation> {
std::vector<NodePtr> options;
};
// "Repeat one child between min and max times."
// a* -> min 0, max infinite
// a+ -> min 1, max infinite
// a? -> min 0, max 1
// a{2,5} -> min 2, max 5
struct Repeat : Visitable<Repeat> {
NodePtr child;
unsigned int min = 0;
std::optional<unsigned int> max; // std::nullopt means "no upper bound"
};
// Parser (recursive descent)
//
// Precedence, loosest to tightest binding:
// ParseAlternation ::= a|b|c
// ParseConcat ::= abc
// ParseRepeat ::= a* a{2,5}
// ParseAtom ::= a . [..] (..) \d
class Parser {
public:
explicit Parser(std::string_view pattern);
NodePtr Parse();
private:
std::string src_;
size_t pos_ = 0;
// Strips anchors (leading '^' and trailing '$') as they are not needed for
// generation.
static std::string StripAnchors(std::string_view pattern);
[[nodiscard]] bool AtEnd() const;
[[nodiscard]] char Peek() const;
char Advance();
bool Match(char c);
void Expect(char c);
// Level 1: alternation -- ( concat | concat | concat )
NodePtr ParseAlternation();
// Level 2: concatenation -- a run of items back to back
NodePtr ParseConcat();
// Level 3: an atom plus an optional quantifier
NodePtr ParseRepeat();
// Curly-brace quantifiers: {n}, {n,}, {n,m}
NodePtr ParseBrace(NodePtr atom);
// Level 4: an atom -- the smallest standalone unit
NodePtr ParseAtom();
static NodePtr MakeClass(std::vector<std::pair<char, char>> ranges,
bool negated);
NodePtr ParseEscape();
NodePtr ParseCharClass();
// Reads one character inside [ ], turning \] \n etc. into the real
// character they represent.
char ClassChar();
};
// Generator -- a Visitor that emits a random matching string
class XegerGenerator final : public Visitor {
public:
XegerGenerator(std::mt19937& rng, int unbounded_cap);
std::string Run(const Node& root);
void Visit(const Literal& node) override;
void Visit(const AnyChar&) override;
void Visit(const CharClass& node) override;
void Visit(const Concat& node) override;
void Visit(const Alternation& node) override;
void Visit(const Repeat& node) override;
private:
std::mt19937& rng_;
int unbounded_cap_;
std::string out_;
static bool InRanges(const CharClass& node, char c);
char PickFromClass(const CharClass& node);
};
} // namespace xeger_internal
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_XEGER_XEGER_INTERNAL_H_

View File

@@ -1,19 +1,18 @@
/**
* @file web_client/http_web_client.h
* @brief cpp-httplib implementation of the WebClient interface.
*/
* @file web_client/http_web_client.h
* @brief cpp-httplib implementation of the WebClient interface.
*/
#ifndef BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
#define BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_
#include "web_client/web_client.h"
#include "services/logging/logger.h"
#include <memory>
#include <string>
#include <utility>
#include "services/logging/logger.h"
#include "web_client/web_client.h"
/**
* @brief WebClient implementation backed by cpp-httplib.
*
@@ -26,31 +25,31 @@
* bound to a single origin at construction time.
*/
class HttpWebClient final : public WebClient {
public:
explicit HttpWebClient(std::shared_ptr<ILogger> logger)
: logger_(std::move(logger)) {}
~HttpWebClient() override = default;
public:
explicit HttpWebClient(std::shared_ptr<ILogger> logger)
: logger_(std::move(logger)) {}
~HttpWebClient() override = default;
/**
* @brief Executes a blocking HTTP/HTTPS GET request against a full URL.
*
* @param url Fully-qualified URL, e.g. "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
* @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
*/
std::string Get(const std::string& url) override;
/**
* @brief Executes a blocking HTTP/HTTPS GET request against a full URL.
*
* @param url Fully-qualified URL, e.g.
* "https://en.wikipedia.org/api/rest_v1/page/summary/Berlin"
* @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
*/
std::string Get(const std::string& url) override;
/**
* @brief Percent-encodes a single URI component (query parameter value or
* path segment). Delegates to httplib::encode_uri_component().
*
* @param value Raw string to encode.
* @return Percent-encoded string safe for use in a URL.
*/
std::string EncodeURL(const std::string& value) override;
/**
* @brief Percent-encodes a single URI component (query parameter value or
* path segment). Delegates to httplib::encode_uri_component().
*
* @param value Raw string to encode.
* @return Percent-encoded string safe for use in a URL.
*/
std::string EncodeURL(const std::string& value) override;
private:
std::shared_ptr<ILogger> logger_;
private:
std::shared_ptr<ILogger> logger_;
};
#endif
#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_HTTP_WEB_CLIENT_H_

View File

@@ -12,25 +12,24 @@
* @brief Abstract web client interface.
*/
class WebClient {
public:
/// @brief Virtual destructor for polymorphic cleanup.
virtual ~WebClient() = default;
public:
virtual ~WebClient() = default;
/**
* @brief Executes an HTTP GET request.
*
* @param url Request URL.
* @return Response body.
*/
virtual std::string Get(const std::string& url) = 0;
/**
* @brief Executes an HTTP GET request.
*
* @param url Request URL.
* @return Response body.
*/
virtual std::string Get(const std::string& url) = 0;
/**
* @brief URL-encodes a string value.
*
* @param value Raw string value.
* @return Encoded value safe for URL usage.
*/
virtual std::string EncodeURL(const std::string& value) = 0;
/**
* @brief URL-encodes a string value.
*
* @param value Raw string value.
* @return Encoded value safe for URL usage.
*/
virtual std::string EncodeURL(const std::string& value) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_WEB_CLIENT_WEB_CLIENT_H_

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,72 @@
[
{
"name": "Hophead Explorer",
"description": "Chases the newest hop varieties and double-digit IBUs, tracking hazy releases and limited tap takeovers across town.",
"style_affinities": [
"American IPA",
"Double IPA",
"New England IPA",
"Black IPA"
]
},
{
"name": "Malt & Tradition Loyalist",
"description": "Prefers clean, balanced lagers brewed the way their region has brewed them for generations, and is quick to defend a well-made Helles.",
"style_affinities": ["Munich Helles", "Vienna Lager", "Märzen", "Festbier"]
},
{
"name": "Dark & Roasty Devotee",
"description": "Orders the darkest beer on the board year-round, drawn to roasted malt, coffee, and chocolate notes over carbonated lightness.",
"style_affinities": [
"Imperial Stout",
"Robust Porter",
"Baltic Porter",
"Oatmeal Stout"
]
},
{
"name": "Sour & Wild Forager",
"description": "Seeks out barrel-aged, spontaneously fermented, and funky beers, and will travel for a fresh bottling of something wild.",
"style_affinities": ["Lambic", "Gueuze", "Flanders Red Ale", "Wild Ale"]
},
{
"name": "Session Sipper",
"description": "Values an easy-drinking pint that holds up over a long afternoon at the taproom more than raw intensity or novelty.",
"style_affinities": [
"Session IPA",
"Ordinary Bitter",
"Cream Ale",
"Blonde Ale"
]
},
{
"name": "Belgian Abbey Pilgrim",
"description": "Collects abbey and trappist releases, drawn to the dried-fruit esters and warming strength of classic Belgian ales.",
"style_affinities": [
"Belgian Tripel",
"Belgian Dubbel",
"Belgian Quadrupel",
"Trappist Single"
]
},
{
"name": "Wheat & Citrus Casual",
"description": "Reaches for something light, cloudy, and refreshing, usually with a citrus or coriander note, especially on a patio in summer.",
"style_affinities": [
"Hefeweizen",
"Witbier",
"American Wheat Beer",
"Berliner Weisse"
]
},
{
"name": "Big & Boozy Connoisseur",
"description": "Sips slowly from a snifter, seeking out high-ABV releases meant for aging and savoring rather than quick refreshment.",
"style_affinities": [
"English Barleywine",
"Wee Heavy",
"Doppelbock",
"Old Ale"
]
}
]

View File

@@ -0,0 +1,122 @@
# FULL SYSTEM PROMPT
You are a craft beer community profile writer. You write short, first-person
biographies for fictional users of a brewery-discovery app, grounded in the
exact name, locale, and persona provided. You do not invent the user's name --
it is given to you and must not be altered, translated, or abbreviated beyond
forming a username.
You will receive the inputs like this:
## NAME:
[First name] [Last name]
## GENDER:
[Gender associated with the given first name]
## CITY:
[City Name]
## COUNTRY:
[Country Name]
## PERSONA:
[Persona archetype name]
## PERSONA DESCRIPTION:
[What this persona cares about and how they talk about beer]
## STYLE AFFINITIES:
[Comma-separated beer styles this persona favors]
## CRITICAL OUTPUT FORMAT (READ CAREFULLY):
ABSOLUTELY NO MARKDOWN FORMATTING. Do NOT wrap your response in json or ```
blocks.
Do not add markdown, code fences, or postscript around the final JSON object. Do
not say "Here is the JSON" or "Enjoy!".
The JSON must contain exactly three keys ("username", "bio", "activity_weight")
in that order. Do not rename or add any other keys.
ESCAPE ALL QUOTES inside the bio field using \", or use single quotes (' ')
instead.
DO NOT use actual line breaks (\n) inside any string. Keep the bio as one
continuous string.
The bio must be between 25 and 60 words, written in first person, and must read
as something this specific person would write about themselves -- not a generic
profile blurb.
`activity_weight` is a number between 0 and 1 (inclusive) representing how often
this persona checks in at breweries relative to other users. Persona archetypes
implying frequent, habitual brewery visits should skew higher; casual or
occasional-visitor personas should skew lower. Do not default to 0.5 -- vary it
meaningfully based on the persona description.
Expected JSON format:
```json
{
"username": "a handle derived from the given name",
"bio": "The first-person bio goes here.",
"activity_weight": 0.62
}
```
## CONTENT RULES AND CONSTRAINTS:
### THE USERNAME:
Derive the username from the given first and/or last name (e.g. lowercase,
abbreviated, or combined with a short word or number tied to the persona's style
affinities). Do not invent an unrelated handle. Do not include spaces.
### THE BIO:
Ground the bio in the supplied persona description and style affinities --
mention at least one specific beer style from the style affinities list, in a
way that sounds like a personal preference rather than a list. Reference the
city or country naturally if it fits, but do not force it.
### VOICE & PERSPECTIVE:
Write in the first person ("I"/"my"). The tone should match the persona: an
enthusiastic explorer sounds different from a relaxed, easy-drinking regular. Do
not write in second or third person.
### THE BLOCKLIST (FORBIDDEN CONCEPTS):
You absolutely cannot use the following words and phrases. Make sure your final
output doesn't have any of these:
- "hidden gem"
- "passion"
- "authentic"
- "craft beer enthusiast"
- "beer connoisseur"
- "journey"
#### FORBIDDEN WRITING PATTERNS
The following patterns are common AI writing pitfalls and must not appear in the
bio:
- Negative parallelism constructions: "It's not X, it's Y"
- Inflated significance phrases: "stands as a testament," "plays a vital role,"
"deeply rooted"
- Superficial trailing analyses: sentences ending in -ing words that add opinion
without content
- Promotional travel-copy tone: "breathtaking," "must-visit," "vibrant"
- Overused conjunctive transitions used as sentence openers: "Moreover,"
"Furthermore," "In addition"
- Rule of three: do not consistently organise ideas or examples in triplets

8
tooling/pipeline/run.sh Normal file
View File

@@ -0,0 +1,8 @@
./biergarten-pipeline \
--model ../models/google_gemma-4-E4B-it-Q6_K.gguf \
--temperature 1.0 \
--top-p 0.95 \
--top-k 64 \
--n-ctx 8192 \
--location-count 15 \
--prompt-dir ../prompts

View File

@@ -10,11 +10,10 @@ dockerStartCmd: []
isPublic: false
isServerless: false
env:
BIERGARTEN_MODE: live
BIERGARTEN_MODEL_PATH: /workspace/models/google_gemma-4-E4B-it-Q6_K.gguf
BIERGARTEN_PROMPT_DIR: /workspace/app/build/prompts
BIERGARTEN_OUTPUT_DIR: /workspace/output
BIERGARTEN_LOG_PATH: /workspace/logs/pipeline.log
BIERGARTEN_GL_LAYERS: "40"
BIERGARTEN_TEMPERATURE: "1.0"
BIERGARTEN_TOP_P: "0.95"
BIERGARTEN_TOP_K: "64"

View File

@@ -10,205 +10,212 @@
std::optional<ApplicationOptions> ParseArguments(
const int argc, char** argv, std::shared_ptr<ILogger> logger) {
prog_opts::options_description desc("Pipeline Options");
auto opt = desc.add_options();
prog_opts::options_description desc("Pipeline Options");
auto opt = desc.add_options();
opt("help,h", "Produce help message");
opt("help,h", "Produce help message");
// Defaults sourced from SamplingOptions{} so the CLI and LlamaGenerator
// share a single source of truth — changing the struct updates both.
auto add_sampling_options = [&]() -> void {
const SamplingOptions sampling_defaults{};
opt("temperature",
prog_opts::value<float>()->default_value(sampling_defaults.temperature),
"Sampling temperature (higher = more random)");
opt("top-p",
prog_opts::value<float>()->default_value(sampling_defaults.top_p),
"Nucleus sampling top-p in (0,1] (higher = more random)");
opt("top-k",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.top_k),
"Top-k sampling parameter (higher = more candidate tokens)");
opt("n-ctx",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.n_ctx),
"Context window size in tokens");
opt("seed", prog_opts::value<int>()->default_value(sampling_defaults.seed),
"Sampler seed: -1 for random, otherwise non-negative integer");
opt("n-gpu-layers", prog_opts::value<int>()->default_value(0),
"Number of layers to offload to GPU");
};
auto add_sampling_options = [&]() -> void {
const SamplingOptions sampling_defaults{};
opt("temperature",
prog_opts::value<float>()->default_value(
sampling_defaults.temperature),
"Sampling temperature (higher = more random)");
opt("top-p",
prog_opts::value<float>()->default_value(sampling_defaults.top_p),
"Nucleus sampling top-p in (0,1] (higher = more random)");
opt("top-k",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.top_k),
"Top-k sampling parameter (higher = more candidate tokens)");
opt("n-ctx",
prog_opts::value<uint32_t>()->default_value(sampling_defaults.n_ctx),
"Context window size in tokens");
opt("seed",
prog_opts::value<int>()->default_value(sampling_defaults.seed),
"Sampler seed: -1 for random, otherwise non-negative integer");
opt("n-gpu-layers", prog_opts::value<int>()->default_value(0),
"Number of layers to offload to GPU");
};
// --mocked and --model are mutually exclusive; validation is enforced below
// rather than at registration to produce a clear diagnostic message.
auto add_generator_options = [&]() -> void {
opt("mocked", prog_opts::bool_switch(),
"Use mocked generator for brewery/user data");
opt("model,m", prog_opts::value<std::string>()->default_value(""),
"Path to LLM model (gguf)");
};
// --mocked and --model are mutually exclusive; validation is enforced below
// rather than at registration to produce a clear diagnostic message.
auto add_generator_options = [&]() -> void {
opt("mocked", prog_opts::bool_switch(),
"Use mocked generator for brewery/user data");
opt("model,m", prog_opts::value<std::string>()->default_value(""),
"Path to LLM model (gguf)");
};
auto add_pipeline_options = [&]() -> void {
opt("output,o", prog_opts::value<std::string>()->default_value("output"),
"Directory for generated artifacts");
opt("log-path",
prog_opts::value<std::string>()->default_value("pipeline.log"),
"Path for application logs");
opt("prompt-dir", prog_opts::value<std::string>()->default_value(""),
"Directory containing named prompt files (e.g. BREWERY_GENERATION.md)."
" Required when not using --mocked.");
opt("location-count", prog_opts::value<uint32_t>()->default_value(10));
};
auto add_pipeline_options = [&]() -> void {
opt("output,o", prog_opts::value<std::string>()->default_value("output"),
"Directory for generated artifacts");
opt("log-path",
prog_opts::value<std::string>()->default_value("pipeline.log"),
"Path for application logs");
opt("prompt-dir", prog_opts::value<std::string>()->default_value(""),
"Directory containing named prompt files (e.g. "
"BREWERY_GENERATION.md)."
" Required when not using --mocked.");
opt("location-count", prog_opts::value<uint32_t>()->default_value(10));
};
add_sampling_options();
add_generator_options();
add_pipeline_options();
add_sampling_options();
add_generator_options();
add_pipeline_options();
// No flags provided — treat as a help request rather than an error.
if (argc == 1) {
const std::string title = "Biergarten Pipeline";
const std::string usage = ([&] {
std::stringstream usage_stream;
usage_stream << "\nUsage: biergarten-pipeline [options]\n\n" << desc;
return usage_stream.str();
})();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = title});
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = usage});
}
return std::nullopt;
}
try {
prog_opts::variables_map var_map;
prog_opts::store(prog_opts::parse_command_line(argc, argv, desc), var_map);
prog_opts::notify(var_map);
if (var_map.contains("help")) {
std::stringstream help_stream;
help_stream << "\n" << desc;
// No flags provided — treat as a help request rather than an error.
if (argc == 1) {
const std::string title = "Biergarten Pipeline";
const std::string usage = ([&] {
std::stringstream usage_stream;
usage_stream << "\nUsage: biergarten-pipeline [options]\n\n" << desc;
return usage_stream.str();
})();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = help_stream.str()});
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = title});
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = usage});
}
return std::nullopt;
}
}
ApplicationOptions options;
try {
prog_opts::variables_map var_map;
prog_opts::store(prog_opts::parse_command_line(argc, argv, desc),
var_map);
prog_opts::notify(var_map);
options.pipeline.output_path = var_map["output"].as<std::string>();
options.pipeline.log_path = var_map["log-path"].as<std::string>();
options.pipeline.prompt_dir = var_map["prompt-dir"].as<std::string>();
options.pipeline.location_count = var_map["location-count"].as<uint32_t>();
if (var_map.contains("help")) {
std::stringstream help_stream;
help_stream << "\n" << desc;
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = help_stream.str()});
}
return std::nullopt;
}
const bool use_mocked = var_map["mocked"].as<bool>();
const std::string model_path = var_map["model"].as<std::string>();
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
ApplicationOptions options;
// Enforce mutual exclusivity before any further configuration is applied.
if (use_mocked && !model_path.empty()) {
options.pipeline.output_path = var_map["output"].as<std::string>();
options.pipeline.log_path = var_map["log-path"].as<std::string>();
options.pipeline.prompt_dir = var_map["prompt-dir"].as<std::string>();
options.pipeline.location_count =
var_map["location-count"].as<uint32_t>();
const bool use_mocked = var_map["mocked"].as<bool>();
const std::string model_path = var_map["model"].as<std::string>();
const int n_gpu_layers = var_map["n-gpu-layers"].as<int>();
// Enforce mutual exclusivity before any further configuration is
// applied.
if (use_mocked && !model_path.empty()) {
const std::string msg =
"Invalid arguments: --mocked and --model are mutually "
"exclusive";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
return std::nullopt;
}
if (!use_mocked && model_path.empty()) {
const std::string msg =
"Invalid arguments: either --mocked or --model must be "
"specified";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
return std::nullopt;
}
// Prompt directory is only meaningful for live inference — the mock
// generator has no use for it and should not require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using "
"--mocked";
if (logger) {
logger->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
return std::nullopt;
}
options.generator.use_mocked = use_mocked;
options.generator.model_path = model_path;
// Only populate sampling config when the user explicitly overrides at
// least one value. Leaving it as std::nullopt lets LlamaGenerator fall
// back to its own SamplingOptions{} defaults, keeping the two paths
// consistent without redundant copies.
const bool user_provided_sampling =
!var_map["temperature"].defaulted() ||
!var_map["top-p"].defaulted() || !var_map["top-k"].defaulted() ||
!var_map["n-ctx"].defaulted() || !var_map["seed"].defaulted() ||
!var_map["n_gpu_layers"].defaulted();
if (user_provided_sampling) {
// Warn but do not fail — the run is still valid, the flags are just
// silently irrelevant when no model is loaded.
if (use_mocked) {
const std::string msg =
"Sampling parameters are ignored when using --mocked";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Warn,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
} else {
SamplingOptions sampling;
sampling.temperature = var_map["temperature"].as<float>();
sampling.top_p = var_map["top-p"].as<float>();
sampling.top_k = var_map["top-k"].as<uint32_t>();
sampling.n_ctx = var_map["n-ctx"].as<uint32_t>();
sampling.seed = var_map["seed"].as<int>();
sampling.n_gpu_layers = var_map["n-gpu-layers"].as<int>();
options.generator.sampling = sampling;
}
}
return options;
} catch (const std::exception& exception) {
const std::string msg =
"Invalid arguments: --mocked and --model are mutually exclusive";
std::string("Failed to parse command-line arguments: ") +
exception.what();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
}
if (!use_mocked && model_path.empty()) {
} catch (...) {
const std::string msg =
"Invalid arguments: either --mocked or --model must be specified";
"Failed to parse command-line arguments: unknown error";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
}
// Prompt directory is only meaningful for live inference — the mock
// generator has no use for it and should not require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg =
"Invalid arguments: --prompt-dir is required when not using --mocked";
if (logger) {
logger->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
return std::nullopt;
}
options.generator.use_mocked = use_mocked;
options.generator.model_path = model_path;
// options.generator.n_gpu_layers = n_gpu_layers;
// Only populate sampling config when the user explicitly overrides at
// least one value. Leaving it as std::nullopt lets LlamaGenerator fall
// back to its own SamplingOptions{} defaults, keeping the two paths
// consistent without redundant copies.
const bool user_provided_sampling =
!var_map["temperature"].defaulted() || !var_map["top-p"].defaulted() ||
!var_map["top-k"].defaulted() || !var_map["n-ctx"].defaulted() ||
!var_map["seed"].defaulted() || !var_map["n_gpu_layers"].defaulted();
if (user_provided_sampling) {
// Warn but do not fail — the run is still valid, the flags are just
// silently irrelevant when no model is loaded.
if (use_mocked) {
const std::string msg =
"Sampling parameters are ignored when using --mocked";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Warn,
.phase = PipelinePhase::Startup,
.message = msg});
} else {
std::cerr << msg << std::endl;
}
} else {
SamplingOptions sampling;
sampling.temperature = var_map["temperature"].as<float>();
sampling.top_p = var_map["top-p"].as<float>();
sampling.top_k = var_map["top-k"].as<uint32_t>();
sampling.n_ctx = var_map["n-ctx"].as<uint32_t>();
sampling.seed = var_map["seed"].as<int>();
sampling.n_gpu_layers = var_map["n-gpu-layers"].as<int>();
options.generator.sampling = sampling;
}
}
return options;
} catch (const std::exception& exception) {
const std::string msg =
std::string("Failed to parse command-line arguments: ") +
exception.what();
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
} catch (...) {
const std::string msg =
"Failed to parse command-line arguments: unknown error";
if (logger) {
logger->Log(LogDTO{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = msg});
}
return std::nullopt;
}
}
}

View File

@@ -1,6 +1,6 @@
/**
* @file biergarten_pipeline_orchestrator/biergarten_pipeline_orchestrator.cc
* @brief BiergartenDataGenerator constructor implementation.
* @brief BiergartenPipelineOrchestrator constructor implementation.
*/
#include "biergarten_pipeline_orchestrator.h"
@@ -12,9 +12,13 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<IEnrichmentService> context_service,
std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter,
const ApplicationOptions &app_options)
std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options)
: logger_(std::move(logger)),
context_service_(std::move(context_service)),
generator_(std::move(generator)),
exporter_(std::move(exporter)),
application_options_(app_options) {}
curated_data_service_(std::move(curated_data_service)),
postal_code_service_(std::move(postal_code_service)),
application_options_(application_options) {}

View File

@@ -1,68 +1,94 @@
/**
* @file biergarten_pipeline_orchestrator/generate_breweries.cc
* @brief BiergartenDataGenerator::GenerateBreweries() implementation.
* @brief BiergartenPipelineOrchestrator::GenerateBreweries() implementation.
*/
#include <chrono>
#include <format>
#include <optional>
#include "biergarten_pipeline_orchestrator.h"
#include "services/logging/logger.h"
void BiergartenPipelineOrchestrator::GenerateBreweries(
std::span<const EnrichedCity> cities) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = "=== SAMPLE BREWERY GENERATION ==="});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = "=== SAMPLE BREWERY GENERATION ==="});
generated_breweries_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
for (const auto& [location, region_context] : cities) {
try {
const BreweryResult brewery =
generator_->GenerateBrewery(location, region_context);
const GeneratedBrewery gen{.location = location, .brewery = brewery};
generated_breweries_.push_back(gen);
generated_breweries_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
const auto generate_record =
[this, &skipped_count](
const EnrichedCity& enriched_city) -> std::optional<BreweryRecord> {
try {
exporter_->ProcessRecord(gen);
} catch (const std::exception& export_exception) {
++export_failed_count;
const BreweryResult brewery =
generator_->GenerateBrewery(enriched_city);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(enriched_city.location);
return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code},
.brewery = brewery};
} catch (const std::exception& e) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message =
std::format("[Pipeline] Generated brewery for '{}' ({}) but SQLite export failed: {}",
location.city, location.country, export_exception.what())});
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): brewery "
"generation failed: {}",
enriched_city.location.city,
enriched_city.location.country, e.what())});
return std::nullopt;
}
} catch (const std::exception& e) {
++skipped_count;
};
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): brewery generation failed: {}",
location.city, location.country, e.what())});
}
}
if (skipped_count > 0) {
logger_->Log({.level = LogLevel::Warn,
const auto export_record =
[this, &export_failed_count](const BreweryRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to generation errors",
skipped_count)});
}
"[Pipeline] Generated brewery for '{}' ({}) "
"but SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};
if (export_failed_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format(
"[Pipeline] Failed to export {} generated brewery/breweries to SQLite",
export_failed_count)});
}
for (const EnrichedCity& enriched_city : cities) {
const std::optional<BreweryRecord> record =
generate_record(enriched_city);
if (!record.has_value()) {
continue;
}
generated_breweries_.push_back(*record);
export_record(*record);
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to generation errors",
skipped_count)});
}
if (export_failed_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} generated "
"brewery/breweries to SQLite",
export_failed_count)});
}
}

View File

@@ -0,0 +1,224 @@
/**
* @file biergarten_pipeline_orchestrator/generate_users.cc
* @brief BiergartenPipelineOrchestrator::GenerateUsers() implementation.
*/
#include <cctype>
#include <chrono>
#include <format>
#include <iterator>
#include <optional>
#include <random>
#include <stdexcept>
#include <string>
#include <string_view>
#include <unordered_set>
#include "biergarten_pipeline_orchestrator.h"
#include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h"
namespace {
std::string Sanitize(std::string_view value) {
std::string out;
out.reserve(value.size());
for (const char character : value) {
if (std::isalnum(static_cast<unsigned char>(character)) != 0) {
out.push_back(static_cast<char>(
std::tolower(static_cast<unsigned char>(character))));
}
}
return out;
}
std::string BuildEmail(const Name& name,
std::unordered_set<std::string>& used_local_parts) {
const std::string base = std::format("{}.{}", Sanitize(name.first_name),
Sanitize(name.last_name));
std::string local_part = base;
uint32_t suffix = 1;
while (used_local_parts.contains(local_part)) {
local_part = std::format("{}{}", base, suffix);
++suffix;
}
used_local_parts.insert(local_part);
return std::format("{}@thebiergarten.app", local_part);
}
std::string GenerateDateOfBirth(std::mt19937& rng) {
using namespace std::chrono;
constexpr int kMinAge = 19;
constexpr int kMaxAge = 48;
constexpr int kMaxDayOffset = 364;
std::uniform_int_distribution<int> age_dist(kMinAge, kMaxAge);
std::uniform_int_distribution<int> day_offset_dist(0, kMaxDayOffset);
const year_month_day today{floor<days>(system_clock::now())};
const year_month_day birth_year_anchor{today.year() - years{age_dist(rng)},
today.month(), today.day()};
const sys_days birth_date =
sys_days{birth_year_anchor} - days{day_offset_dist(rng)};
const year_month_day birth_ymd{birth_date};
return std::format("{:04}-{:02}-{:02}", static_cast<int>(birth_ymd.year()),
static_cast<unsigned>(birth_ymd.month()),
static_cast<unsigned>(birth_ymd.day()));
}
std::optional<Name> SampleName(
const ForenamesByCountryMap& forenames_by_country,
const SurnamesByCountryMap& surnames_by_country,
const std::string& iso3166_1, std::mt19937& rng) {
const auto forenames_it = forenames_by_country.find(iso3166_1);
const auto surnames_it = surnames_by_country.find(iso3166_1);
if (forenames_it == forenames_by_country.end() ||
surnames_it == surnames_by_country.end() ||
forenames_it->second.empty() || surnames_it->second.empty()) {
return std::nullopt;
}
const ForenameList& forenames = forenames_it->second;
const SurnameList& surnames = surnames_it->second;
std::uniform_int_distribution<size_t> forename_dist(0, forenames.size() - 1);
std::uniform_int_distribution<size_t> surname_dist(0, surnames.size() - 1);
auto forename_it = forenames.begin();
std::advance(forename_it, forename_dist(rng));
auto surname_it = surnames.begin();
std::advance(surname_it, surname_dist(rng));
return Name{.first_name = forename_it->name,
.last_name = *surname_it,
.gender = forename_it->gender};
}
} // namespace
void BiergartenPipelineOrchestrator::GenerateUsers(
std::span<const EnrichedCity> cities) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = "=== SAMPLE USER GENERATION ==="});
const PersonasList& personas = curated_data_service_->LoadPersonas();
if (personas.empty()) {
throw std::runtime_error(
"No personas available in personas.json for user generation");
}
const ForenamesByCountryMap& forenames_by_country =
curated_data_service_->LoadForenamesByCountry();
const SurnamesByCountryMap& surnames_by_country =
curated_data_service_->LoadSurnamesByCountry();
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<size_t> persona_dist(0, personas.size() - 1);
generated_users_.clear();
size_t skipped_count = 0;
size_t export_failed_count = 0;
std::unordered_set<std::string> used_email_local_parts;
const auto generate_record =
[this, &rng, &skipped_count, &used_email_local_parts](
const EnrichedCity& city, const UserPersona& persona,
const Name& sampled_name) -> std::optional<UserRecord> {
try {
const UserResult user =
generator_->GenerateUser(city, persona, sampled_name);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(city.location);
return UserRecord{
.address =
UserAddress{.city = city.location, .postal_code = postal_code},
.user = user,
.email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng),
};
} catch (const std::exception& e) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("[Pipeline] Skipping city '{}' ({}): "
"user generation failed: {}",
city.location.city, city.location.country,
e.what())});
return std::nullopt;
}
};
const auto export_record = [this,
&export_failed_count](const UserRecord& record) {
try {
exporter_->ProcessRecord(record);
} catch (const std::exception& export_exception) {
++export_failed_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Generated user for '{}' ({}) but "
"SQLite export failed: {}",
record.address.city.city, record.address.city.country,
export_exception.what())});
}
};
for (const auto& city : cities) {
const std::optional<Name> sampled_name =
SampleName(forenames_by_country, surnames_by_country,
city.location.iso3166_1, rng);
if (!sampled_name.has_value()) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): no names "
"available for country '{}'",
city.location.city, city.location.country,
city.location.iso3166_1)});
continue;
}
const UserPersona& persona = personas[persona_dist(rng)];
const std::optional<UserRecord> record =
generate_record(city, persona, *sampled_name);
if (!record.has_value()) {
continue;
}
generated_users_.push_back(*record);
export_record(*record);
}
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipped {} city/cities during user generation",
skipped_count)});
}
if (export_failed_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Teardown,
.message = std::format("[Pipeline] Failed to export {} "
"generated user/users to SQLite",
export_failed_count)});
}
}

View File

@@ -1,37 +1,59 @@
/**
* @file biergarten_pipeline_orchestrator/log_results.cc
* @brief BiergartenDataGenerator::LogResults() implementation.
* @brief BiergartenPipelineOrchestrator::LogResults() implementation.
*/
#include <boost/json/array.hpp>
#include <chrono>
#include <format>
#include "../../includes/json_handling/pretty_print.h"
#include "biergarten_pipeline_orchestrator.h"
#include "json_handling/pretty_print.h"
#include "services/logging/logger.h"
void BiergartenPipelineOrchestrator::LogResults() const {
boost::json::array output;
boost::json::array brewery_output;
for (const auto& [location, brewery] : generated_breweries_) {
output.push_back(boost::json::object{
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"location", boost::json::object{
{"city", location.city},
{"country", location.country},
{"state_province", location.state_province},
{"iso3166_2", location.iso3166_2},
{"latitude", location.latitude},
{"longitude", location.longitude},
}}});
}
for (const auto& [address, brewery] : generated_breweries_) {
const City& city = address.city;
brewery_output.push_back(boost::json::object{
{"name_en", brewery.name_en},
{"description_en", brewery.description_en},
{"name_local", brewery.name_local},
{"description_local", brewery.description_local},
{"address", boost::json::object{
{"city", city.city},
{"country", city.country},
{"state_province", city.state_province},
{"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
}}});
}
std::ostringstream oss;
PrettyPrint(oss, output);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = oss.str()});
std::ostringstream brewery_oss;
PrettyPrint(brewery_oss, brewery_output);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = brewery_oss.str()});
boost::json::array user_output;
for (const auto& generated_user : generated_users_) {
user_output.push_back(boost::json::object{
{"first_name", generated_user.user.first_name},
{"last_name", generated_user.user.last_name},
{"gender", generated_user.user.gender},
{"username", generated_user.user.username},
{"bio", generated_user.user.bio},
{"activity_weight", generated_user.user.activity_weight},
{"email", generated_user.email},
{"date_of_birth", generated_user.date_of_birth},
});
}
std::ostringstream user_oss;
PrettyPrint(user_oss, user_output);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = user_oss.str()});
}

View File

@@ -1,51 +1,49 @@
/**
* @file biergarten_pipeline_orchestrator/query_cities_with_countries.cc
* @brief BiergartenDataGenerator::QueryCitiesWithCountries() implementation.
* @brief BiergartenPipelineOrchestrator::QueryCitiesWithCountries()
* implementation.
*/
#include <algorithm>
#include <chrono>
#include <filesystem>
#include <format>
#include <iterator>
#include <random>
#include "biergarten_pipeline_orchestrator.h"
#include "json_handling/json_loader.h"
#include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h"
std::vector<Location>
BiergartenPipelineOrchestrator::QueryCitiesWithCountries() {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
std::vector<City> BiergartenPipelineOrchestrator::QueryLocations() {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "=== GEOGRAPHIC DATA OVERVIEW ==="});
const std::filesystem::path locations_path = "locations.json";
const std::vector<City>& all_locations =
curated_data_service_->LoadLocations();
auto all_locations = JsonLoader::LoadLocations(locations_path, logger_);
const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count),
all_locations.size());
const size_t sample_count = std::min(
static_cast<size_t>(application_options_.pipeline.location_count),
all_locations.size());
const auto sample_count_signed =
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count);
const auto sample_count_signed =
static_cast<std::iter_difference_t<decltype(all_locations.cbegin())>>(
sample_count);
std::vector<City> sampled_locations;
sampled_locations.reserve(sample_count);
std::vector<Location> sampled_locations;
sampled_locations.reserve(sample_count);
std::random_device random_generator;
std::ranges::sample(all_locations, std::back_inserter(sampled_locations),
sample_count_signed, random_generator);
std::random_device random_generator;
std::ranges::sample(all_locations, std::back_inserter(sampled_locations),
sample_count_signed, random_generator);
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Locations available: {}",
all_locations.size())});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Sampled locations: {}",
sampled_locations.size())});
return sampled_locations;
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Locations available: {}",
all_locations.size())});
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(" Sampled locations: {}",
sampled_locations.size())});
return sampled_locations;
}

View File

@@ -1,6 +1,6 @@
/**
* @file biergarten_pipeline_orchestrator/run.cc
* @brief BiergartenDataGenerator::Run() implementation.
* @brief BiergartenPipelineOrchestrator::Run() implementation.
*/
#include <chrono>
@@ -11,53 +11,52 @@
#include "services/logging/logger.h"
bool BiergartenPipelineOrchestrator::Run() {
try {
exporter_->Initialize();
try {
exporter_->Initialize();
std::vector<Location> cities = QueryCitiesWithCountries();
std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size());
std::vector<City> cities = QueryLocations();
std::vector<EnrichedCity> enriched;
enriched.reserve(cities.size());
size_t skipped_count = 0;
for (auto& city : cities) {
try {
std::string region_context = context_service_->GetLocationContext(city);
// logger_->Log(LogLevel::Debug, PipelinePhase::UserGeneration,
// "[Pipeline] Context for '" + city.city + "' (" +
// city.iso3166_2 + ") gathered:\n" + region_context);
size_t skipped_count = 0;
for (auto& city : cities) {
try {
std::string region_context =
context_service_->GetLocationContext(city);
enriched.push_back(
EnrichedCity{.location = std::move(city),
.region_context = std::move(region_context)});
} catch (const std::exception& exception) {
++skipped_count;
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): context lookup failed: {}",
city.city, city.country, exception.what())});
enriched.push_back(
EnrichedCity{.location = std::move(city),
.region_context = std::move(region_context)});
} catch (const std::exception& exception) {
++skipped_count;
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"[Pipeline] Skipping city '{}' ({}): context "
"lookup failed: {}",
city.city, city.country, exception.what())});
}
}
}
if (skipped_count > 0) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
if (skipped_count > 0) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format("[Pipeline] Skipped {} city/cities due "
"to context lookup errors",
skipped_count)});
}
this->GenerateUsers(enriched);
this->GenerateBreweries(enriched);
exporter_->Finalize();
this->LogResults();
return true;
} catch (const std::exception& e) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = std::format(
"[Pipeline] Skipped {} city/cities due to context lookup errors",
skipped_count)});
}
this->GenerateBreweries(enriched);
exporter_->Finalize();
this->LogResults();
return true;
} catch (const std::exception& e) {
logger_->Log(
{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message =
std::format("Pipeline execution failed with error: {}", e.what())});
return false;
}
"Pipeline execution failed with error: {}", e.what())});
return false;
}
}

View File

@@ -12,155 +12,150 @@
#include <string_view>
#include <vector>
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
static std::string FormatLocalLanguageCodes(
const std::vector<std::string>& codes) {
if (codes.empty()) {
return "Not provided";
}
if (codes.empty()) {
return "Not provided";
}
std::string formatted;
for (const std::string& code : codes) {
if (!formatted.empty()) {
formatted += ", ";
}
formatted += code;
}
std::string formatted;
for (const std::string& code : codes) {
if (!formatted.empty()) {
formatted += ", ";
}
formatted += code;
}
return formatted;
return formatted;
}
// GBNF grammar for structured brewery JSON output.
// @TODO move to a separate gbnf file if it grows in complexity or is shared
// across modules.
static constexpr std::string_view kBreweryJsonGrammar = R"json_brewery(
root ::= thought-block "{" ws "\"name_en\"" ws ":" ws string ws "," ws "\"description_en\"" ws ":" ws string ws "," ws "\"name_local\"" ws ":" ws string ws "," ws "\"description_local\"" ws ":" ws string ws "}" ws
thought-block ::= [^{]*
ws ::= [ \t\n\r]*
string ::= "\"" char+ "\""
char ::= [^"\\\x7F\x00-\x1F] | [\\] escape
escape ::= ["\\/bfnrt] | "u" hex hex hex hex
hex ::= [0-9a-fA-F]
)json_brewery";
static constexpr int kBreweryInitialMaxTokens = 2800;
BreweryResult LlamaGenerator::GenerateBrewery(
const Location& location, const std::string& region_context) {
/**
* Preprocess and truncate region context to manageable size
*/
const std::string safe_region_context = PrepareRegionContext(region_context);
const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const std::string& region_context = enriched_city.region_context;
/**
* Preprocess and truncate region context to manageable size
*/
const std::string safe_region_context = PrepareRegionContext(region_context);
const std::string local_language_codes =
FormatLocalLanguageCodes(location.local_languages);
const std::string local_language_codes =
FormatLocalLanguageCodes(location.local_languages);
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
/**
* Load brewery system prompt via the injected prompt directory.
* The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside
* the configured --prompt-dir. Throws on missing or empty file.
*/
const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION");
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
/**
* Load brewery system prompt via the injected prompt directory.
* The key "BREWERY_GENERATION" resolves to BREWERY_GENERATION.md inside
* the configured --prompt-dir. Throws on missing or empty file.
*/
const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION");
std::string user_prompt = std::format(
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
"CONTEXT:\n{}",
location.city, location.country, local_language_codes,
safe_region_context);
std::string user_prompt = std::format(
"## CITY:\n{}\n\n## COUNTRY:\n{}\n\n## LOCAL LANGUAGE CODES:\n{}\n\n## "
"CONTEXT:\n{}",
location.city, location.country, local_language_codes,
safe_region_context);
/**
* Store location context for retry prompts (without repeating full context)
*/
const std::string retry_location =
std::format("Location: {}{}\nLocal language codes: {}", location.city,
country_suffix, local_language_codes);
/**
* Store location context for retry prompts (without repeating full context)
*/
const std::string retry_location =
std::format("City: {}{}\nLocal language codes: {}", location.city,
country_suffix, local_language_codes);
/**
* RETRY LOOP with validation and error correction
* Attempts to generate valid brewery data up to 3 times, with feedback-based
* refinement
*/
constexpr int max_attempts = 3;
std::string raw;
std::string last_error;
/**
* RETRY LOOP with validation and error correction
* Attempts to generate valid brewery data up to 3 times, with
* feedback-based refinement
*/
constexpr int max_attempts = 3;
std::string raw;
std::string last_error;
// Token budget: too small risks truncating valid JSON mid-string.
// Start conservatively but allow adaptive increases on truncation.
int max_tokens = kBreweryInitialMaxTokens;
// Limit output length to keep it concise and focused
for (int attempt = 0; attempt < max_attempts; ++attempt) {
// Generate brewery data from LLM
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kBreweryJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
// Validate output: parse JSON and check required fields
BreweryResult brewery;
const std::optional<std::string> validation_error =
ValidateBreweryJson(raw, brewery);
if (!validation_error.has_value()) {
// Success: return parsed brewery data
// Token budget: too small risks truncating valid JSON mid-string.
int max_tokens = kBreweryInitialMaxTokens;
// Limit output length to keep it concise and focused
for (int attempt = 0; attempt < max_attempts; ++attempt) {
// Generate brewery data from LLM
raw = this->Infer(system_prompt, user_prompt, max_tokens,
kBreweryJsonGrammar);
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format("LlamaGenerator: successfully generated brewery data on attempt {}",
attempt + 1)});
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
return brewery;
}
// Validate output: parse JSON and check required fields
// Validation failed: log error and prepare corrective feedback
BreweryResult brewery;
const std::optional<std::string> validation_error =
ValidateBreweryJson(raw, brewery);
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message =
std::format("LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
if (!validation_error.has_value()) {
// Success: return parsed brewery data
// Update prompt with error details to guide LLM toward correct output.
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON with "
"exactly these keys, in this exact order: {{\"name_en\": \"<English "
"brewery name>\", \"description_en\": \"<English single-paragraph "
"description>\", \"name_local\": \"<local-language brewery name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\nKeep the JSON strings concise enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: successfully generated "
"brewery data on attempt {}",
attempt + 1)});
}
// All retry attempts exhausted: log failure and throw exception
if (logger_) {
logger_->Log(
{.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed brewery response");
return brewery;
}
// Validation failed: log error and prepare corrective feedback
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
// Update prompt with error details to guide LLM toward correct output.
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"name_en\": "
"\"<English "
"brewery name>\", \"description_en\": \"<English single-paragraph "
"description>\", \"name_local\": \"<local-language brewery "
"name>\", "
"\"description_local\": \"<local-language single-paragraph "
"description>\"}}.\nDo not include markdown, comments, extra keys, "
"or "
"literal placeholder values.\n\nKeep the JSON strings concise "
"enough "
"to fit within the token budget.\n\n{}",
*validation_error, retry_location);
}
// All retry attempts exhausted: log failure and throw exception
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: malformed brewery "
"response after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed brewery response");
}

View File

@@ -1,24 +1,117 @@
/**
* @file data_generation/llama/generate_user.cc
* @brief Generates locale-aware user profiles with strict two-line formatting,
* retry handling, and output sanitization for downstream parsing.
* @brief Builds persona/name-grounded user prompts, performs retry-based
* inference, and validates structured JSON output for user records.
*/
#include <format>
#include <optional>
#include <stdexcept>
#include <string>
#include <string_view>
#include "data_generation/json_grammars.h"
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
// TODO: Implement locale-aware user profile generation.
// Current implementation returns a hardcoded test value and ignores the
// locale parameter. Future implementation should:
// 1. Load a USER_GENERATION.md prompt template with locale context
// 2. Perform LLM inference with locale-specific username/bio generation
// 3. Parse and validate JSON output with retry handling (similar to brewery)
// 4. Return locale-aware username and biography
UserResult LlamaGenerator::GenerateUser(const std::string& locale) {
return {.username = "test_user",
.bio = std::format("This is a test user profile from {}.", locale)};
static constexpr int kUserInitialMaxTokens = 1200;
UserResult LlamaGenerator::GenerateUser(const EnrichedCity& city,
const UserPersona& persona,
const Name& name) {
std::string style_affinities;
for (const std::string& style : persona.style_affinities) {
if (!style_affinities.empty()) {
style_affinities += ", ";
}
style_affinities += style;
}
const std::string system_prompt = prompt_directory_->Load("USER_GENERATION");
std::string user_prompt = std::format(
"## NAME:\n\n{} {}\n\n"
"## GENDER:\n\n{}\n\n"
"## CITY:\n\n{}\n\n"
"## COUNTRY:\n\n{}\n\n"
"## PERSONA:\n\n{}\n\n"
"## PERSONA DESCRIPTION:\n\n{}\n\n"
"## STYLE AFFINITIES:\n\n{}",
name.first_name, name.last_name, name.gender, city.location.city,
city.location.country, persona.name, persona.description,
style_affinities);
const std::string retry_context = std::format(
"Name: {} {}\nCity: {}, {}\nPersona: {}", name.first_name,
name.last_name, city.location.city, city.location.country, persona.name);
constexpr int max_attempts = 3;
std::string raw;
std::string last_error;
int max_tokens = kUserInitialMaxTokens;
for (int attempt = 0; attempt < max_attempts; ++attempt) {
raw =
this->Infer(system_prompt, user_prompt, max_tokens, kUserJsonGrammar);
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: raw output (attempt {}): {}",
attempt + 1, raw)});
}
UserResult user;
const std::optional<std::string> validation_error =
ValidateUserJson(raw, user);
if (!validation_error.has_value()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("LlamaGenerator: successfully "
"generated user data on attempt {}",
attempt + 1)});
}
user.first_name = name.first_name;
user.last_name = name.last_name;
user.gender = name.gender;
return user;
}
last_error = *validation_error;
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user JSON (attempt {}): {}",
attempt + 1, *validation_error)});
}
user_prompt = std::format(
"Your previous response was invalid. Error: {}\nReturn the thought "
"process before the JSON if needed, then return ONLY valid JSON "
"with "
"exactly these keys, in this exact order: {{\"username\": "
"\"<handle "
"derived from the given name>\", \"bio\": \"<first-person bio "
"grounded in the persona>\", \"activity_weight\": <number between "
"0 "
"and 1>}}.\nDo not include markdown, comments, extra keys, or "
"literal placeholder values.\n\n{}",
*validation_error, retry_context);
}
if (logger_) {
logger_->Log({.level = LogLevel::Error,
.phase = PipelinePhase::UserGeneration,
.message = std::format(
"LlamaGenerator: malformed user response "
"after {} attempts: {}",
max_attempts, last_error.empty() ? raw : last_error)});
}
throw std::runtime_error("LlamaGenerator: malformed user response");
}

View File

@@ -4,31 +4,34 @@
* parsing, token decoding, and JSON validation helpers for Llama modules.
*/
#include <llama.h>
#include <algorithm>
#include <array>
#include <boost/json.hpp>
#include <cctype>
#include <optional>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "data_generation/llama_generator_helpers.h"
#include "llama.h"
namespace {
/**
* String trimming: removes leading and trailing whitespace
*/
std::string Trim(std::string_view value) {
constexpr std::string_view whitespace = " \t\n\r\f\v";
const size_t first_index = value.find_first_not_of(whitespace);
if (first_index == std::string_view::npos) {
return {};
}
constexpr std::string_view whitespace = " \t\n\r\f\v";
const size_t first_index = value.find_first_not_of(whitespace);
if (first_index == std::string_view::npos) {
return {};
}
const size_t last_index = value.find_last_not_of(whitespace);
return std::string(value.substr(first_index, last_index - first_index + 1));
const size_t last_index = value.find_last_not_of(whitespace);
return std::string(value.substr(first_index, last_index - first_index + 1));
}
/**
@@ -36,26 +39,26 @@ std::string Trim(std::string_view value) {
* spaces
*/
std::string CondenseWhitespace(std::string_view text) {
std::string out;
out.reserve(text.size());
std::string out;
out.reserve(text.size());
bool pending_space = false;
for (const char chr : text) {
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
if (!out.empty()) {
pending_space = true;
bool pending_space = false;
for (const char chr : text) {
if (std::isspace(static_cast<unsigned char>(chr)) != 0) {
if (!out.empty()) {
pending_space = true;
}
continue;
}
continue;
}
if (pending_space) {
out.push_back(' ');
pending_space = false;
}
out.push_back(chr);
}
if (pending_space) {
out.push_back(' ');
pending_space = false;
}
out.push_back(chr);
}
return out;
return out;
}
// Guard against truncating in the first half of the string.
@@ -66,30 +69,30 @@ constexpr size_t kTruncationGuardDivisor = 2;
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
std::string_view key, std::string& out,
std::string* error_out) {
const boost::json::value* field = obj.if_contains(key);
if (field == nullptr || !field->is_string()) {
return false;
}
const boost::json::value* field = obj.if_contains(key);
if (field == nullptr || !field->is_string()) {
return false;
}
const auto& string_value = field->as_string();
out = Trim(std::string_view(string_value.data(), string_value.size()));
return !out.empty();
const auto& string_value = field->as_string();
out = Trim(std::string_view(string_value.data(), string_value.size()));
return !out.empty();
}
bool HasSchemaPlaceholder(const std::array<std::string*, 4>& values) {
for (const std::string* value : values) {
std::string lowered = *value;
std::ranges::transform(lowered, lowered.begin(),
[](const unsigned char character) {
return static_cast<char>(std::tolower(character));
});
bool HasSchemaPlaceholder(std::span<std::string* const> values) {
for (const std::string* value : values) {
std::string lowered = *value;
std::ranges::transform(
lowered, lowered.begin(), [](const unsigned char character) {
return static_cast<char>(std::tolower(character));
});
if (lowered == "string") {
return true;
}
}
if (lowered == "string") {
return true;
}
}
return false;
return false;
}
} // namespace
@@ -99,111 +102,167 @@ bool HasSchemaPlaceholder(const std::array<std::string*, 4>& values) {
*/
std::string PrepareRegionContext(std::string_view region_context,
const size_t max_chars) {
std::string normalized = CondenseWhitespace(region_context);
if (normalized.size() <= max_chars) {
return normalized;
}
std::string normalized = CondenseWhitespace(region_context);
if (normalized.size() <= max_chars) {
return normalized;
}
normalized.resize(max_chars);
const size_t last_space = normalized.find_last_of(' ');
if (last_space != std::string::npos &&
last_space > max_chars / kTruncationGuardDivisor) {
normalized.resize(last_space);
}
normalized.resize(max_chars);
const size_t last_space = normalized.find_last_of(' ');
if (last_space != std::string::npos &&
last_space > max_chars / kTruncationGuardDivisor) {
normalized.resize(last_space);
}
normalized += "...";
return normalized;
normalized += "...";
return normalized;
}
void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
std::string& output) {
constexpr size_t initial_buffer_size = 256;
constexpr size_t initial_buffer_size = 256;
std::array<char, initial_buffer_size> buffer{};
std::array<char, initial_buffer_size> buffer{};
// serialize the sampled token into UTF-8 bytes
// serialize the sampled token into UTF-8 bytes
auto buffer_too_small = [](int32_t result) -> bool { return result < 0; };
auto buffer_too_small = [](int32_t result) -> bool { return result < 0; };
int32_t bytes =
llama_token_to_piece(vocab, token, buffer.data(), buffer.size(), 0, true);
int32_t bytes = llama_token_to_piece(vocab, token, buffer.data(),
buffer.size(), 0, true);
if (!buffer_too_small(bytes)) {
// Append the decoded bytes from the stack buffer.
output.append(buffer.data(), static_cast<size_t>(bytes));
return;
}
if (!buffer_too_small(bytes)) {
// Append the decoded bytes from the stack buffer.
output.append(buffer.data(), static_cast<size_t>(bytes));
return;
}
const int32_t required_size = -bytes;
std::vector<char> dynamic_buffer(static_cast<size_t>(required_size));
const int32_t required_size = -bytes;
std::vector<char> dynamic_buffer(static_cast<size_t>(required_size));
// Retry token decoding against the larger heap buffer.
bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(),
static_cast<int32_t>(dynamic_buffer.size()), 0,
true);
// Retry token decoding against the larger heap buffer.
bytes = llama_token_to_piece(vocab, token, dynamic_buffer.data(),
static_cast<int32_t>(dynamic_buffer.size()), 0,
true);
if (!buffer_too_small(bytes)) {
output.append(dynamic_buffer.data(), static_cast<size_t>(bytes));
return;
}
if (!buffer_too_small(bytes)) {
output.append(dynamic_buffer.data(), static_cast<size_t>(bytes));
return;
}
throw std::runtime_error(
"LlamaGenerator: failed to decode sampled token piece");
throw std::runtime_error(
"LlamaGenerator: failed to decode sampled token piece");
}
std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 4) {
return "JSON object must contain exactly four keys";
}
const auto& obj = json_value.get_object();
if (obj.size() != 4) {
return "JSON object must contain exactly four keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
&validation_error)) {
return validation_error;
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "name_en", brewery_out.name_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_en",
brewery_out.description_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_en",
brewery_out.description_en,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "name_local", brewery_out.name_local,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(
obj, "name_local", brewery_out.name_local, &validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_local",
brewery_out.description_local,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "description_local",
brewery_out.description_local,
&validation_error)) {
return validation_error;
}
const std::array schema_placeholders = {
&brewery_out.name_en, &brewery_out.description_en,
&brewery_out.name_local, &brewery_out.description_local};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
const std::array schema_placeholders = {
&brewery_out.name_en, &brewery_out.description_en,
&brewery_out.name_local, &brewery_out.description_local};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
return std::nullopt;
}
std::optional<std::string> ValidateUserJson(const std::string& raw,
UserResult& user_out) {
boost::system::error_code error_code;
const std::string_view raw_view(raw);
const size_t opening_brace = raw_view.find('{');
if (opening_brace == std::string_view::npos) {
return "JSON parse error: missing opening brace '{'";
}
const std::string_view json_payload = raw_view.substr(opening_brace);
boost::json::value json_value = boost::json::parse(json_payload, error_code);
if (error_code) {
return "JSON parse error: " + error_code.message();
}
if (!json_value.is_object()) {
return "JSON root must be an object";
}
const auto& obj = json_value.get_object();
if (obj.size() != 3) {
return "JSON object must contain exactly three keys";
}
std::string validation_error;
if (!ReadRequiredTrimmedStringField(obj, "username", user_out.username,
&validation_error)) {
return validation_error;
}
if (!ReadRequiredTrimmedStringField(obj, "bio", user_out.bio,
&validation_error)) {
return validation_error;
}
const boost::json::value* activity_weight_field =
obj.if_contains("activity_weight");
if (activity_weight_field == nullptr ||
!activity_weight_field->is_number()) {
return "Missing or invalid numeric field: activity_weight";
}
const double activity_weight = activity_weight_field->to_number<double>();
if (activity_weight < 0.0 || activity_weight > 1.0) {
return "activity_weight must be between 0 and 1";
}
user_out.activity_weight = static_cast<float>(activity_weight);
const std::array schema_placeholders = {&user_out.username, &user_out.bio};
if (HasSchemaPlaceholder(schema_placeholders)) {
return "JSON appears to be a schema placeholder, not content";
}
return std::nullopt;
}

View File

@@ -5,6 +5,8 @@
* output tokens back to text for system+user chat prompts.
*/
#include <llama.h>
#include <algorithm>
#include <chrono>
#include <format>
@@ -16,7 +18,6 @@
#include "data_generation/llama_generator.h"
#include "data_generation/llama_generator_helpers.h"
#include "llama.h"
static constexpr size_t kPromptTokenSlack = 8;
// Minimum tokens to keep when using top-p sampling. Ensures at least one
@@ -29,48 +30,49 @@ using SamplerHandle =
std::unique_ptr<llama_sampler, decltype(&llama_sampler_free)>;
struct SamplerConfig {
float temperature;
uint32_t top_k;
float top_p;
uint32_t seed;
float temperature;
uint32_t top_k;
float top_p;
uint32_t seed;
};
SamplerHandle MakeSamplerChain(const llama_vocab* vocab,
const SamplerConfig& config,
std::string_view grammar) {
const llama_sampler_chain_params sampler_params =
llama_sampler_chain_default_params();
const llama_sampler_chain_params sampler_params =
llama_sampler_chain_default_params();
SamplerHandle chain(llama_sampler_chain_init(sampler_params),
llama_sampler_free);
if (!chain) {
throw std::runtime_error("LlamaGenerator: failed to initialize sampler");
}
SamplerHandle chain(llama_sampler_chain_init(sampler_params),
llama_sampler_free);
if (!chain) {
throw std::runtime_error("LlamaGenerator: failed to initialize sampler");
}
auto add_sampler = [&](llama_sampler* sampler, const char* error_message) {
if (sampler == nullptr) {
throw std::runtime_error(error_message);
}
auto add_sampler = [&](llama_sampler* sampler, const char* error_message) {
if (sampler == nullptr) {
throw std::runtime_error(error_message);
}
llama_sampler_chain_add(chain.get(), sampler);
};
llama_sampler_chain_add(chain.get(), sampler);
};
if (!grammar.empty()) {
const std::string grammar_text(grammar);
add_sampler(llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"),
"LlamaGenerator: failed to initialize grammar sampler");
}
if (!grammar.empty()) {
const std::string grammar_text(grammar);
add_sampler(
llama_sampler_init_grammar(vocab, grammar_text.c_str(), "root"),
"LlamaGenerator: failed to initialize grammar sampler");
}
add_sampler(llama_sampler_init_temp(config.temperature),
"LlamaGenerator: failed to initialize temperature sampler");
add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)),
"LlamaGenerator: failed to initialize top-k sampler");
add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep),
"LlamaGenerator: failed to initialize top-p sampler");
add_sampler(llama_sampler_init_dist(config.seed),
"LlamaGenerator: failed to initialize distribution sampler");
add_sampler(llama_sampler_init_temp(config.temperature),
"LlamaGenerator: failed to initialize temperature sampler");
add_sampler(llama_sampler_init_top_k(static_cast<int32_t>(config.top_k)),
"LlamaGenerator: failed to initialize top-k sampler");
add_sampler(llama_sampler_init_top_p(config.top_p, kTopPMinKeep),
"LlamaGenerator: failed to initialize top-p sampler");
add_sampler(llama_sampler_init_dist(config.seed),
"LlamaGenerator: failed to initialize distribution sampler");
return chain;
return chain;
}
} // namespace
@@ -79,163 +81,164 @@ std::string LlamaGenerator::Infer(const std::string& system_prompt,
const std::string& prompt,
const int max_tokens,
std::string_view grammar) {
return InferFormatted(prompt_formatter_->Format(system_prompt, prompt),
max_tokens, grammar);
return InferFormatted(prompt_formatter_->Format(system_prompt, prompt),
max_tokens, grammar);
}
std::string LlamaGenerator::InferFormatted(const std::string& formatted_prompt,
const int max_tokens,
std::string_view grammar) {
/**
* Validate that model and context are loaded
*/
if (!model_ || !context_) {
throw std::runtime_error("LlamaGenerator: model not loaded");
}
/**
* Validate that model and context are loaded
*/
if (!model_ || !context_) {
throw std::runtime_error("LlamaGenerator: model not loaded");
}
/**
* Get vocabulary for tokenization and token-to-text conversion
*/
const llama_vocab* vocab = llama_model_get_vocab(model_.get());
if (vocab == nullptr) {
throw std::runtime_error("LlamaGenerator: vocab unavailable");
}
/**
* Get vocabulary for tokenization and token-to-text conversion
*/
const llama_vocab* vocab = llama_model_get_vocab(model_.get());
if (vocab == nullptr) {
throw std::runtime_error("LlamaGenerator: vocab unavailable");
}
const SamplerConfig sampler_config{
.temperature = sampling_temperature_,
.top_k = sampling_top_k_,
.top_p = sampling_top_p_,
.seed = static_cast<uint32_t>(rng_()),
};
const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar);
const SamplerConfig sampler_config{
.temperature = sampling_temperature_,
.top_k = sampling_top_k_,
.top_p = sampling_top_p_,
.seed = static_cast<uint32_t>(rng_()),
};
const auto sampler = MakeSamplerChain(vocab, sampler_config, grammar);
/**
* Clear KV cache to ensure clean inference state (no residual context)
*/
llama_memory_clear(llama_get_memory(context_.get()), true);
/**
* Clear KV cache to ensure clean inference state (no residual context)
*/
llama_memory_clear(llama_get_memory(context_.get()), true);
/**
* TOKENIZATION PHASE
* Convert text prompt into token IDs (integers) that the model understands
*/
std::vector<llama_token> prompt_tokens(formatted_prompt.size() +
kPromptTokenSlack);
/**
* TOKENIZATION PHASE
* Convert text prompt into token IDs (integers) that the model understands
*/
std::vector<llama_token> prompt_tokens(formatted_prompt.size() +
kPromptTokenSlack);
int32_t token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
int32_t token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
/**
* If buffer too small, negative return indicates required size
*/
if (token_count < 0) {
prompt_tokens.resize(static_cast<size_t>(-token_count));
/**
* If buffer too small, negative return indicates required size
*/
if (token_count < 0) {
prompt_tokens.resize(static_cast<size_t>(-token_count));
token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
}
token_count = llama_tokenize(
vocab, formatted_prompt.c_str(),
static_cast<int32_t>(formatted_prompt.size()), prompt_tokens.data(),
static_cast<int32_t>(prompt_tokens.size()), true, true);
}
if (token_count < 0) {
throw std::runtime_error("LlamaGenerator: prompt tokenization failed");
}
if (token_count < 0) {
throw std::runtime_error("LlamaGenerator: prompt tokenization failed");
}
/**
* CONTEXT SIZE VALIDATION
* Validate and compute effective token budgets based on context window
* constraints
*/
const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get()));
const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get()));
if (n_ctx <= 1 || n_batch <= 0) {
throw std::runtime_error("LlamaGenerator: invalid context or batch size");
}
/**
* CONTEXT SIZE VALIDATION
* Validate and compute effective token budgets based on context window
* constraints
*/
const auto n_ctx = static_cast<int32_t>(llama_n_ctx(context_.get()));
const auto n_batch = static_cast<int32_t>(llama_n_batch(context_.get()));
if (n_ctx <= 1 || n_batch <= 0) {
throw std::runtime_error("LlamaGenerator: invalid context or batch size");
}
/**
* Clamp generation limit to available context window, reserve space for
* output
*/
const int32_t effective_max_tokens =
std::max(1, std::min(max_tokens, n_ctx - 1));
/**
* Clamp generation limit to available context window, reserve space for
* output
*/
const int32_t effective_max_tokens =
std::max(1, std::min(max_tokens, n_ctx - 1));
/**
* Prompt can use remaining context after reserving space for generation
*/
int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens);
prompt_budget = std::max<int32_t>(1, prompt_budget);
/**
* Prompt can use remaining context after reserving space for generation
*/
int32_t prompt_budget = std::min(n_batch, n_ctx - effective_max_tokens);
prompt_budget = std::max<int32_t>(1, prompt_budget);
/**
* Truncate prompt if necessary to fit within constraints
*/
prompt_tokens.resize(static_cast<size_t>(token_count));
if (token_count > prompt_budget) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: prompt too long ({} tokens), "
"truncating to {} tokens to fit n_batch/n_ctx limits",
token_count, prompt_budget)});
}
prompt_tokens.resize(static_cast<size_t>(prompt_budget));
token_count = prompt_budget;
}
/**
* Truncate prompt if necessary to fit within constraints
*/
prompt_tokens.resize(static_cast<size_t>(token_count));
if (token_count > prompt_budget) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::BreweryAndBeerGeneration,
.message = std::format(
"LlamaGenerator: prompt too long ({} tokens), "
"truncating to {} tokens to fit n_batch/n_ctx limits",
token_count, prompt_budget)});
}
prompt_tokens.resize(static_cast<size_t>(prompt_budget));
token_count = prompt_budget;
}
/**
* PROMPT PROCESSING PHASE
* Create a batch containing all prompt tokens and feed through the model
* This computes internal representations and fills the KV cache
*/
const llama_batch prompt_batch = llama_batch_get_one(
prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size()));
if (llama_decode(context_.get(), prompt_batch) != 0) {
throw std::runtime_error("LlamaGenerator: prompt decode failed");
}
/**
* PROMPT PROCESSING PHASE
* Create a batch containing all prompt tokens and feed through the model
* This computes internal representations and fills the KV cache
*/
const llama_batch prompt_batch = llama_batch_get_one(
prompt_tokens.data(), static_cast<int32_t>(prompt_tokens.size()));
if (llama_decode(context_.get(), prompt_batch) != 0) {
throw std::runtime_error("LlamaGenerator: prompt decode failed");
}
/**
* TOKEN GENERATION LOOP
* Iteratively generate tokens one at a time until max_tokens or
* end-of-sequence
*/
std::vector<llama_token> generated_tokens;
generated_tokens.reserve(static_cast<size_t>(effective_max_tokens));
/**
* TOKEN GENERATION LOOP
* Iteratively generate tokens one at a time until max_tokens or
* end-of-sequence
*/
std::vector<llama_token> generated_tokens;
generated_tokens.reserve(static_cast<size_t>(effective_max_tokens));
for (int i = 0; i < effective_max_tokens; ++i) {
/**
* Sample next token using configured sampler chain and model logits
* Index -1 means use the last output position from previous batch
*/
const llama_token next =
llama_sampler_sample(sampler.get(), context_.get(), -1);
/**
* Stop if model predicts end-of-generation token (EOS/EOT)
*/
if (llama_vocab_is_eog(vocab, next)) {
break;
}
generated_tokens.push_back(next);
/**
* Feed the sampled token back into model for next iteration
* (autoregressive)
*/
llama_token decode_token = next;
const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1);
if (llama_decode(context_.get(), one_token_batch) != 0) {
throw std::runtime_error(
"LlamaGenerator: decode failed during generation");
}
}
for (int i = 0; i < effective_max_tokens; ++i) {
/**
* Sample next token using configured sampler chain and model logits
* Index -1 means use the last output position from previous batch
*/
const llama_token next =
llama_sampler_sample(sampler.get(), context_.get(), -1);
/**
* Stop if model predicts end-of-generation token (EOS/EOT)
*/
if (llama_vocab_is_eog(vocab, next)) {
break;
}
generated_tokens.push_back(next);
/**
* Feed the sampled token back into model for next iteration
* (autoregressive)
*/
llama_token decode_token = next;
const llama_batch one_token_batch = llama_batch_get_one(&decode_token, 1);
if (llama_decode(context_.get(), one_token_batch) != 0) {
throw std::runtime_error(
"LlamaGenerator: decode failed during generation");
}
}
/**
* DETOKENIZATION PHASE
* Convert generated token IDs back to text using vocabulary
*/
std::string output;
for (const llama_token token : generated_tokens) {
AppendTokenPiece(vocab, token, output);
}
/**
* DETOKENIZATION PHASE
* Convert generated token IDs back to text using vocabulary
*/
std::string output;
for (const llama_token token : generated_tokens) {
AppendTokenPiece(vocab, token, output);
}
return output;
return output;
}

View File

@@ -5,6 +5,8 @@
#include "data_generation/llama_generator.h"
#include <llama.h>
#include <filesystem>
#include <memory>
#include <random>
@@ -12,22 +14,21 @@
#include <string>
#include "data_model/models.h"
#include "llama.h"
static constexpr uint32_t kMaxContextSize = 32768U;
void LlamaGenerator::ModelDeleter::operator()(
llama_model* model) const noexcept {
if (model != nullptr) {
llama_model_free(model);
}
if (model != nullptr) {
llama_model_free(model);
}
}
void LlamaGenerator::ContextDeleter::operator()(
llama_context* context) const noexcept {
if (context != nullptr) {
llama_free(context);
}
if (context != nullptr) {
llama_free(context);
}
}
LlamaGenerator::LlamaGenerator(
@@ -39,61 +40,61 @@ LlamaGenerator::LlamaGenerator(
logger_(std::move(logger)),
prompt_formatter_(std::move(prompt_formatter)),
prompt_directory_(std::move(prompt_directory)) {
if (model_path.empty()) {
throw std::runtime_error("LlamaGenerator: model path must not be empty");
}
if (model_path.empty()) {
throw std::runtime_error("LlamaGenerator: model path must not be empty");
}
if (!prompt_formatter_) {
throw std::runtime_error(
"LlamaGenerator: prompt formatter dependency must not be null");
}
if (!prompt_formatter_) {
throw std::runtime_error(
"LlamaGenerator: prompt formatter dependency must not be null");
}
if (!prompt_directory_) {
throw std::runtime_error(
"LlamaGenerator: prompt directory dependency must not be null");
}
if (!prompt_directory_) {
throw std::runtime_error(
"LlamaGenerator: prompt directory dependency must not be null");
}
const auto sampling = options.generator.sampling.value_or(SamplingOptions{});
const auto sampling = options.generator.sampling.value_or(SamplingOptions{});
if (sampling.temperature < 0.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling temperature must be >= 0");
}
if (sampling.temperature < 0.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling temperature must be >= 0");
}
if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling top-p must be in (0, 1]");
}
if (sampling.top_p <= 0.0F || sampling.top_p > 1.0F) {
throw std::runtime_error(
"LlamaGenerator: sampling top-p must be in (0, 1]");
}
if (sampling.top_k == 0U) {
throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0");
}
if (sampling.top_k == 0U) {
throw std::runtime_error("LlamaGenerator: sampling top-k must be > 0");
}
if (sampling.seed < -1) {
throw std::runtime_error(
"LlamaGenerator: seed must be >= 0, or -1 for random");
}
if (sampling.seed < -1) {
throw std::runtime_error(
"LlamaGenerator: seed must be >= 0, or -1 for random");
}
if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) {
throw std::runtime_error(
"LlamaGenerator: context size must be in range [1, 32768]");
}
if (sampling.n_ctx == 0 || sampling.n_ctx > kMaxContextSize) {
throw std::runtime_error(
"LlamaGenerator: context size must be in range [1, 32768]");
}
sampling_temperature_ = sampling.temperature;
sampling_top_p_ = sampling.top_p;
sampling_top_k_ = sampling.top_k;
sampling_temperature_ = sampling.temperature;
sampling_top_p_ = sampling.top_p;
sampling_top_k_ = sampling.top_k;
if (sampling.seed == -1) {
std::random_device random_device;
rng_.seed(random_device());
} else {
rng_.seed(static_cast<uint32_t>(sampling.seed));
}
if (sampling.seed == -1) {
std::random_device random_device;
rng_.seed(random_device());
} else {
rng_.seed(static_cast<uint32_t>(sampling.seed));
}
n_ctx_ = sampling.n_ctx;
n_gpu_layers_ = sampling.n_gpu_layers;
n_ctx_ = sampling.n_ctx;
n_gpu_layers_ = sampling.n_gpu_layers;
this->Load(model_path);
this->Load(model_path);
}
LlamaGenerator::~LlamaGenerator() = default;

View File

@@ -4,6 +4,9 @@
* context, and resets prior resources during model initialization.
*/
#include <ggml-backend.h>
#include <llama.h>
#include <algorithm>
#include <chrono>
#include <stdexcept>
@@ -11,50 +14,48 @@
#include <utility>
#include "data_generation/llama_generator.h"
#include "ggml-backend.h"
#include "llama.h"
// Maximum batch size for decode operations. Capping the batch prevents
// excessive memory allocation while maintaining inference performance.
static constexpr uint32_t kMaxBatchSize = 5000U;
void LlamaGenerator::Load(const std::string& model_path) {
context_.reset();
model_.reset();
context_.reset();
model_.reset();
// Specifically load dynamic ggml backends (like CUDA) that are provided
// externally before attempting to load a model.
ggml_backend_load_all();
// Specifically load dynamic ggml backends (like CUDA) that are provided
// externally before attempting to load a model.
ggml_backend_load_all();
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = n_gpu_layers_;
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = n_gpu_layers_;
ModelHandle loaded_model(
llama_model_load_from_file(model_path.c_str(), model_params));
ModelHandle loaded_model(
llama_model_load_from_file(model_path.c_str(), model_params));
if (!loaded_model) {
throw std::runtime_error(
"LlamaGenerator: failed to load model from path: " + model_path);
}
if (!loaded_model) {
throw std::runtime_error(
"LlamaGenerator: failed to load model from path: " + model_path);
}
llama_context_params context_params = llama_context_default_params();
context_params.n_ctx = n_ctx_;
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
llama_context_params context_params = llama_context_default_params();
context_params.n_ctx = n_ctx_;
context_params.n_batch = std::min(n_ctx_, kMaxBatchSize);
ContextHandle loaded_context(
llama_init_from_model(loaded_model.get(), context_params));
ContextHandle loaded_context(
llama_init_from_model(loaded_model.get(), context_params));
if (!loaded_context) {
throw std::runtime_error("LlamaGenerator: failed to create context");
}
if (!loaded_context) {
throw std::runtime_error("LlamaGenerator: failed to create context");
}
model_ = std::move(loaded_model);
context_ = std::move(loaded_context);
model_ = std::move(loaded_model);
context_ = std::move(loaded_context);
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format("[LlamaGenerator] Loaded model: {} ",
model_path)});
}
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format("[LlamaGenerator] Loaded model: {} ",
model_path)});
}
}

View File

@@ -8,9 +8,19 @@
#include "data_generation/mock_generator.h"
size_t MockGenerator::DeterministicHash(const Location& location) {
size_t seed = 0;
boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country);
return seed;
size_t MockGenerator::DeterministicHash(const City& location) {
size_t seed = 0;
boost::hash_combine(seed, location.city);
boost::hash_combine(seed, location.country);
return seed;
}
size_t MockGenerator::DeterministicHash(const City& location,
const UserPersona& persona,
const Name& name) {
size_t seed = DeterministicHash(location);
boost::hash_combine(seed, persona.name);
boost::hash_combine(seed, name.first_name);
boost::hash_combine(seed, name.last_name);
return seed;
}

View File

@@ -11,34 +11,35 @@
#include "data_generation/mock_generator.h"
BreweryResult MockGenerator::GenerateBrewery(
const Location& location, const std::string& /*region_context*/) {
const size_t hash = DeterministicHash(location);
const EnrichedCity& enriched_city) {
const City& location = enriched_city.location;
const size_t hash = DeterministicHash(location);
const std::string_view adjective =
kBreweryAdjectives.at(hash % kBreweryAdjectives.size());
const std::string_view noun =
kBreweryNouns.at(hash / kNounHashStride % kBreweryNouns.size());
const std::string_view base_description = kBreweryDescriptions.at(
(hash / kDescriptionHashStride) % kBreweryDescriptions.size());
const std::string_view adjective =
kBreweryAdjectives.at(hash % kBreweryAdjectives.size());
const std::string_view noun =
kBreweryNouns.at(hash / kNounHashStride % kBreweryNouns.size());
const std::string_view base_description = kBreweryDescriptions.at(
(hash / kDescriptionHashStride) % kBreweryDescriptions.size());
const std::string name =
std::format("{} {} {}", location.city, adjective, noun);
const std::string name =
std::format("{} {} {}", location.city, adjective, noun);
const std::string state_suffix =
location.state_province.empty()
? std::string{}
: std::format(", {}", location.state_province);
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
const std::string description =
std::format("{} Located in {}{}{}.", base_description, location.city,
state_suffix, country_suffix);
const std::string state_suffix =
location.state_province.empty()
? std::string{}
: std::format(", {}", location.state_province);
const std::string country_suffix =
location.country.empty() ? std::string{}
: std::format(", {}", location.country);
const std::string description =
std::format("{} Located in {}{}{}.", base_description, location.city,
state_suffix, country_suffix);
return {
.name_en = name,
.description_en = description,
.name_local = name,
.description_local = description,
};
return {
.name_en = name,
.description_en = description,
.name_local = name,
.description_local = description,
};
}

View File

@@ -1,22 +1,31 @@
/**
* @file data_generation/mock/generate_user.cc
* @brief Generates deterministic mock user profiles by hashing locale values
* into predefined username and bio collections.
* @brief Generates deterministic mock user profiles by hashing city, persona,
* and sampled name into predefined username and bio collections.
*/
#include <functional>
#include <string>
#include <string_view>
#include "data_generation/mock_generator.h"
UserResult MockGenerator::GenerateUser(const std::string& locale) {
const size_t hash = std::hash<std::string>{}(locale);
UserResult MockGenerator::GenerateUser(const EnrichedCity& city,
const UserPersona& persona,
const Name& name) {
const size_t hash = DeterministicHash(city.location, persona, name);
UserResult result;
const std::string_view username = kUsernames[hash % kUsernames.size()];
const std::string_view bio = kBios[hash / kBioHashStride % kBios.size()];
result.username = username;
result.bio = bio;
return result;
const std::string_view username = kUsernames[hash % kUsernames.size()];
const std::string_view bio = kBios[hash / kBioHashStride % kBios.size()];
const float activity_weight =
static_cast<float>(hash / kActivityWeightHashStride %
kActivityWeightHashRange) /
static_cast<float>(kActivityWeightHashRange);
return {
.first_name = name.first_name,
.last_name = name.last_name,
.gender = name.gender,
.username = std::string(username),
.bio = std::string(bio),
.activity_weight = activity_weight,
};
}

View File

@@ -8,25 +8,25 @@ static constexpr std::string_view kWhitespace = " \t\n\r\f\v";
// Strips leading and trailing whitespace to ensure clean prompt injection.
static std::string_view Trim(std::string_view value) {
const size_t first_index = value.find_first_not_of(kWhitespace);
const size_t first_index = value.find_first_not_of(kWhitespace);
const bool is_all_whitespace = (first_index == std::string_view::npos);
if (is_all_whitespace) {
return "";
}
const bool is_all_whitespace = (first_index == std::string_view::npos);
if (is_all_whitespace) {
return "";
}
const size_t last_index = value.find_last_not_of(kWhitespace);
return value.substr(first_index, last_index - first_index + 1);
const size_t last_index = value.find_last_not_of(kWhitespace);
return value.substr(first_index, last_index - first_index + 1);
}
std::string Gemma4JinjaPromptFormatter::Format(
std::string_view system_prompt, std::string_view user_prompt) const {
std::string_view trimmed_system = Trim(system_prompt);
std::string_view trimmed_user = Trim(user_prompt);
std::string_view trimmed_system = Trim(system_prompt);
std::string_view trimmed_user = Trim(user_prompt);
return std::format(
"<|turn|>system\n<|think|>\n{}\n<|turn|>\n"
"<|turn|>user\n{}\n<|turn|>\n"
"<|turn|>model\n<|channel>thought\n",
trimmed_system, trimmed_user);
return std::format(
"<|turn|>system\n<|think|>\n{}\n<|turn|>\n"
"<|turn|>user\n{}\n<|turn|>\n"
"<|turn|>model\n<|channel>thought\n",
trimmed_system, trimmed_user);
}

View File

@@ -1,110 +0,0 @@
/**
* @file json_handling/json_loader.cc
* @brief Parses curated location JSON input into strongly typed Location
* records with strict field validation and descriptive error reporting.
*/
#include "json_handling/json_loader.h"
#include <format>
#include "services/logging/logger.h"
#include <iostream>
#include <boost/json.hpp>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
static std::string ReadRequiredString(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_string()) {
throw std::runtime_error(
std::format("Missing or invalid string field: {}", key));
}
const std::string_view text = value->as_string();
return std::string(text);
}
static double ReadRequiredNumber(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_number()) {
throw std::runtime_error(
std::format("Missing or invalid numeric field: {}", key));
}
return value->to_number<double>();
}
static std::vector<std::string> ReadRequiredStringArray(
const boost::json::object& object, const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_array()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
const auto& array = value->as_array();
std::vector<std::string> items;
items.reserve(array.size());
for (const auto& item : array) {
if (!item.is_string()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
items.emplace_back(item.as_string());
}
return items;
}
std::vector<Location> JsonLoader::LoadLocations(
const std::filesystem::path& filepath, std::shared_ptr<ILogger> logger) {
std::ifstream input(filepath);
if (!input.is_open()) {
throw std::runtime_error("Failed to open locations file: " +
filepath.string());
}
std::stringstream buffer;
buffer << input.rdbuf();
const std::string content = buffer.str();
boost::system::error_code error;
boost::json::value root = boost::json::parse(content, error);
if (error) {
throw std::runtime_error("Failed to parse locations JSON: " +
error.message());
}
if (!root.is_array()) {
throw std::runtime_error(
"Invalid locations JSON: root element must be an array");
}
std::vector<Location> locations;
const auto& items = root.as_array();
locations.reserve(items.size());
for (const auto& item : items) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid locations JSON: each entry must be an object");
}
const auto& object = item.as_object();
locations.push_back(Location{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.latitude = ReadRequiredNumber(object, "latitude"),
.longitude = ReadRequiredNumber(object, "longitude"),
});
}
return locations;
}

View File

@@ -18,201 +18,237 @@
#include <string>
#include <thread>
#include "biergarten_pipeline_orchestrator.h"
#include "concurrency/bounded_channel.h"
#include "data_generation/llama_generator.h"
#include "data_generation/mock_generator.h"
#include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
#include "data_model/models.h"
#include "llama_backend_state.h"
#include "services/database/export_service.h"
#include "services/database/sqlite_export_service.h"
#include "services/datetime/timer.h"
#include "services/enrichment/enrichment_service.h"
#include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h"
#include "services/logging/log_dispatcher.h"
#include "services/logging/log_entry.h"
#include "services/logging/log_producer.h"
#include "services/logging/logger.h"
#include "services/prompting/prompt_directory.h"
#include "web_client/http_web_client.h"
#include "biergarten_pipeline.h"
namespace di = boost::di;
static constexpr size_t kLogMaxCount = 512;
int main(const int argc, char** argv) {
spdlog::set_level(spdlog::level::debug);
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
spdlog::set_level(spdlog::level::debug);
spdlog::set_pattern("│ %Y-%m-%d %H:%M:%S.%e │ %^%-7l%$ │ %v");
BoundedChannel<LogEntry> log_channel(kLogMaxCount);
auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel);
auto log_dispatcher = //
std::make_unique<LogDispatcher>(log_channel);
std::shared_ptr<ILogger> log_producer =
std::make_shared<LogProducer>(log_channel);
std::thread log_thread([&log_dispatcher] { log_dispatcher->Run(); });
auto shutdown = [&](const int exit_code) {
log_channel.Close();
log_thread.join();
return exit_code;
};
std::thread log_thread([&log_dispatcher] { log_dispatcher->Run(); });
try {
Timer timer;
auto shutdown = [&](const int exit_code) {
log_channel.Close();
log_thread.join();
return exit_code;
};
try {
Timer timer;
#ifndef BIERGARTEN_MOCK_ONLY
const LlamaBackendState llama_backend_state;
const LlamaBackendState llama_backend_state;
#endif
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "STARTING PIPELINE"});
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "STARTING PIPELINE"});
const std::optional<ApplicationOptions> parsed_options =
ParseArguments(argc, argv, log_producer);
const std::optional<ApplicationOptions> parsed_options =
ParseArguments(argc, argv, log_producer);
if (!parsed_options.has_value()) {
return shutdown(EXIT_FAILURE);
}
const auto options = *parsed_options;
const std::string model_path = options.generator.model_path.string();
const auto sampling =
options.generator.sampling.value_or(SamplingOptions{});
std::unique_ptr<IPromptDirectory> prompt_directory;
if (!options.generator.use_mocked) {
try {
prompt_directory = std::make_unique<PromptDirectory>(
options.pipeline.prompt_dir, log_producer);
} catch (const std::exception& dir_error) {
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = std::format("Invalid --prompt-dir: {}",
dir_error.what())});
return shutdown(EXIT_FAILURE);
if (!parsed_options.has_value()) {
return shutdown(EXIT_FAILURE);
}
}
const auto injector = di::make_injector(
di::bind<ILogger>().to(log_producer),
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
const auto options = *parsed_options;
const std::string model_path = options.generator.model_path.string();
const auto sampling =
options.generator.sampling.value_or(SamplingOptions{});
std::unique_ptr<IPromptDirectory> prompt_directory;
if (!options.generator.use_mocked) {
try {
prompt_directory = std::make_unique<PromptDirectory>(
options.pipeline.prompt_dir, log_producer);
} catch (const std::exception& dir_error) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>());
}),
di::bind<WebClient>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: none (mock mode)"});
}
return std::unique_ptr<WebClient>(nullptr);
}
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: HttpWebClient"});
}
return std::unique_ptr<WebClient>(
std::make_unique<HttpWebClient>(log_producer));
}),
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
{.level = LogLevel::Error,
.phase = PipelinePhase::Startup,
.message = std::format("Invalid --prompt-dir: {}",
dir_error.what())});
return shutdown(EXIT_FAILURE);
}
}
const auto injector = di::make_injector(
di::bind<ILogger>().to(log_producer),
di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPostalCodeService>().to(
[options,
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
if (options.generator.use_mocked) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: mock (curated examples)"});
return std::make_unique<MockPostalCodeService>();
}
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
"Postal code: Xeger (regex-based generation)"});
return std::make_unique<XegerPostalCodeService>();
}),
di::bind<ICuratedDataService>().to(
[options,
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: mock"});
return std::make_unique<MockCuratedDataService>();
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Curated data: JsonLoader"});
return std::make_unique<CuratedJsonDataService>(
CuratedDataFilePaths{
.locations_path = "locations.json",
.personas_path = "personas.json",
.forenames_path = "forenames-by-country.json",
.surnames_path = "surnames-by-country.json",
});
}),
di::bind<IPromptFormatter>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: mock"});
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Prompt formatter: none (mock mode)"});
}
return std::make_unique<MockEnrichmentService>();
}
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: Wikipedia"});
}
return std::make_unique<WikipediaEnrichmentService>(
inj.template create<std::unique_ptr<WebClient>>(),
log_producer);
}),
di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory,
&log_producer](const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
{
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Generator: mock"});
}
return std::make_unique<MockGenerator>();
}
{
return std::unique_ptr<IPromptFormatter>(nullptr);
}
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | temp={:.2f} "
"top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
}
return std::make_unique<LlamaGenerator>(
options, model_path, log_producer,
inj.template create<std::unique_ptr<IPromptFormatter>>(),
std::move(prompt_directory));
}));
.phase = PipelinePhase::Startup,
.message =
"Prompt formatter: Gemma4JinjaPromptFormatter"});
}
return std::unique_ptr<IPromptFormatter>(
std::make_unique<Gemma4JinjaPromptFormatter>());
}),
di::bind<WebClient>().to([options, log_producer] {
if (options.generator.use_mocked) {
{
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: none (mock mode)"});
}
return std::unique_ptr<WebClient>(nullptr);
}
const auto orchestrator =
injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Web client: HttpWebClient"});
return std::unique_ptr<WebClient>(
std::make_unique<HttpWebClient>(log_producer));
}),
di::bind<IEnrichmentService>().to(
[options, &log_producer](
const auto& inj) -> std::unique_ptr<IEnrichmentService> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: mock"});
return std::make_unique<MockEnrichmentService>();
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Enrichment: Wikipedia"});
return std::make_unique<WikipediaEnrichmentService>(
inj.template create<std::unique_ptr<WebClient>>(),
log_producer);
}),
di::bind<DataGenerator>().to(
[&options, &model_path, &sampling, &prompt_directory,
&log_producer](
const auto& inj) -> std::unique_ptr<DataGenerator> {
if (options.generator.use_mocked) {
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Generator: mock"});
return std::make_unique<MockGenerator>();
}
#ifdef BIERGARTEN_MOCK_ONLY
// Unreachable: ParseArguments requires --mocked when
// BIERGARTEN_MOCK_ONLY is compiled in, so LlamaGenerator is
// never constructed and is not linked into this binary.
throw std::runtime_error(
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
"build");
#else
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"Generator: LlamaGenerator | model={} | "
"temp={:.2f} top_p={:.2f} top_k={} n_ctx={} seed={}",
model_path, sampling.temperature, sampling.top_p,
sampling.top_k, sampling.n_ctx, sampling.seed)});
return std::make_unique<LlamaGenerator>(
options, model_path, log_producer,
inj.template create<std::unique_ptr<IPromptFormatter>>(),
std::move(prompt_directory));
#endif
}));
const auto orchestrator =
injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
if (!orchestrator->Run()) {
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = "Pipeline execution failed"});
return shutdown(EXIT_FAILURE);
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = std::format("Pipeline complete in {} ms",
timer.Elapsed())});
return shutdown(EXIT_SUCCESS);
} catch (const std::exception& exception) {
const LogDTO log_entry{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = exception.what()};
if (log_producer) {
log_producer->Log(log_entry);
} else {
std::cerr << log_entry.message << std::endl;
}
if (!orchestrator->Run()) {
log_producer->Log({.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = "Pipeline execution failed"});
return shutdown(EXIT_FAILURE);
}
log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Teardown,
.message = std::format("Pipeline complete in {} ms",
timer.Elapsed())});
return shutdown(EXIT_SUCCESS);
} catch (const std::exception& exception) {
const LogDTO log_entry{.level = LogLevel::Error,
.phase = PipelinePhase::Teardown,
.message = exception.what()};
if (log_producer) {
log_producer->Log(log_entry);
} else {
std::cerr << log_entry.message << std::endl;
}
return shutdown(EXIT_FAILURE);
}
}
}

View File

@@ -0,0 +1,270 @@
/**
* @file json_handling/json_loader.cc
* @brief Parses curated location JSON input into strongly typed City
* records with strict field validation and descriptive error reporting.
*/
#include "services/curated_data/curated_json_data_service.h"
#include <boost/json.hpp>
#include <format>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "services/logging/logger.h"
static std::string ReadRequiredString(const boost::json::object& object,
const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_string()) {
throw std::runtime_error(
std::format("Missing or invalid string field: {}", key));
}
const std::string_view text = value->as_string();
return std::string(text);
}
static std::vector<std::string> ReadRequiredStringArray(
const boost::json::object& object, const char* key) {
const boost::json::value* value = object.if_contains(key);
if (value == nullptr || !value->is_array()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
const auto& array = value->as_array();
std::vector<std::string> items;
items.reserve(array.size());
for (const auto& item : array) {
if (!item.is_string()) {
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
items.emplace_back(item.as_string());
}
return items;
}
static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
const boost::json::value* value = object.if_contains("postal_code");
if (value == nullptr || !value->is_object()) {
throw std::runtime_error("Missing or invalid object field: postal_code");
}
const auto& postal_code = value->as_object();
return PostalCodeSpec{
.country_format_regex =
ReadRequiredString(postal_code, "country_format_regex"),
.city_regexes = ReadRequiredStringArray(postal_code, "city_regex"),
.examples = ReadRequiredStringArray(postal_code, "examples"),
};
}
namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
const char* what) {
std::ifstream input(filepath);
if (!input.is_open()) {
throw std::runtime_error(
std::format("Failed to open {} file: {}", what, filepath.string()));
}
std::stringstream buffer;
buffer << input.rdbuf();
boost::system::error_code error;
boost::json::value root = boost::json::parse(buffer.str(), error);
if (error) {
throw std::runtime_error(
std::format("Failed to parse {} JSON: {}", what, error.message()));
}
return root;
}
/**
* @brief Returns the first element of a string array field, falling back to
* `fallback_key` if `key` is missing/empty (some source entries only have a
* "localized" form and no "romanized" form).
*/
std::string ReadFirstOfStringArray(const boost::json::object& object,
const char* key, const char* fallback_key) {
for (const char* candidate_key : {key, fallback_key}) {
const boost::json::value* value = object.if_contains(candidate_key);
if (value == nullptr || !value->is_array()) {
continue;
}
const auto& array = value->as_array();
if (!array.empty() && array.front().is_string()) {
return std::string(array.front().as_string());
}
}
throw std::runtime_error(
std::format("Missing or invalid string array field: {}", key));
}
} // namespace
CuratedJsonDataService::CuratedJsonDataService(CuratedDataFilePaths filepaths)
: filepaths_(std::move(filepaths)) {}
const LocationsList& CuratedJsonDataService::LoadLocations() {
if (!cache_.locations.empty()) {
return cache_.locations;
}
const boost::json::value root =
ParseJsonFile(filepaths_.locations_path, "locations");
if (!root.is_array()) {
throw std::runtime_error(
"Invalid locations JSON: root element must be an array");
}
LocationsList locations;
const auto& items = root.as_array();
locations.reserve(items.size());
for (const auto& item : items) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid locations JSON: each entry must be an object");
}
const auto& object = item.as_object();
locations.push_back(City{
.city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.local_languages = ReadRequiredStringArray(object, "local_languages"),
.postal_code = ReadPostalCodeSpec(object),
});
}
cache_.locations = std::move(locations);
return cache_.locations;
}
const PersonasList& CuratedJsonDataService::LoadPersonas() {
if (!cache_.personas.empty()) {
return cache_.personas;
}
const boost::json::value root =
ParseJsonFile(filepaths_.personas_path, "personas");
if (!root.is_array()) {
throw std::runtime_error(
"Invalid personas JSON: root element must be an array");
}
PersonasList personas;
const auto& items = root.as_array();
personas.reserve(items.size());
for (const auto& item : items) {
if (!item.is_object()) {
throw std::runtime_error(
"Invalid personas JSON: each entry must be an object");
}
const auto& object = item.as_object();
personas.push_back(UserPersona{
.name = ReadRequiredString(object, "name"),
.description = ReadRequiredString(object, "description"),
.style_affinities =
ReadRequiredStringArray(object, "style_affinities"),
});
}
cache_.personas = std::move(personas);
return cache_.personas;
}
const ForenamesByCountryMap& CuratedJsonDataService::LoadForenamesByCountry() {
if (!cache_.forenames_by_country.empty()) {
return cache_.forenames_by_country;
}
const boost::json::value root =
ParseJsonFile(filepaths_.forenames_path, "forenames-by-country");
if (!root.is_object()) {
throw std::runtime_error(
"Invalid forenames-by-country JSON: root element must be an object "
"keyed by ISO 3166-1 country code");
}
ForenamesByCountryMap forenames_by_country;
for (const auto& [country_code, regions] : root.as_object()) {
if (!regions.is_array()) {
continue;
}
ForenameList entries;
for (const auto& region : regions.as_array()) {
if (!region.is_object()) {
continue;
}
const boost::json::value* names =
region.as_object().if_contains("names");
if (names == nullptr || !names->is_array()) {
continue;
}
for (const auto& name_value : names->as_array()) {
if (!name_value.is_object()) {
continue;
}
const auto& name_object = name_value.as_object();
entries.emplace_back(ForenameEntry{
.name = ReadFirstOfStringArray(name_object, "romanized",
"localized"),
.gender = ReadRequiredString(name_object, "gender"),
});
}
}
forenames_by_country.emplace(country_code, std::move(entries));
}
cache_.forenames_by_country = std::move(forenames_by_country);
return cache_.forenames_by_country;
}
const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() {
if (!cache_.surnames_by_country.empty()) {
return cache_.surnames_by_country;
}
const boost::json::value root =
ParseJsonFile(filepaths_.surnames_path, "surnames-by-country");
if (!root.is_object()) {
throw std::runtime_error(
"Invalid surnames-by-country JSON: root element must be an object "
"keyed by ISO 3166-1 country code");
}
SurnamesByCountryMap surnames_by_country;
for (const auto& [country_code, name_entries] : root.as_object()) {
if (!name_entries.is_array()) continue;
SurnameList surnames;
for (const auto& name_value : name_entries.as_array()) {
if (!name_value.is_object()) {
continue;
}
surnames.emplace_back(ReadFirstOfStringArray(
name_value.as_object(), "romanized", "localized"));
}
surnames_by_country.emplace(country_code, std::move(surnames));
}
cache_.surnames_by_country = std::move(surnames_by_country);
return cache_.surnames_by_country;
}

View File

@@ -0,0 +1,106 @@
/**
* @file services/curated_data/mock_curated_data_service.cc
* @brief Fixed in-memory location, persona, and name dataset for mock mode.
*/
#include "services/curated_data/mock_curated_data_service.h"
MockCuratedDataService::MockCuratedDataService()
: locations_{
City{.city = "Portland",
.state_province = "Oregon",
.iso3166_2 = "US-OR",
.country = "United States",
.iso3166_1 = "US",
.local_languages = {"en"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^972\d{2}$)"},
.examples = {"97201", "97202", "97209"}}},
City{.city = "Munich",
.state_province = "Bavaria",
.iso3166_2 = "DE-BY",
.country = "Germany",
.iso3166_1 = "DE",
.local_languages = {"de"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^8\d{4}$)"},
.examples = {"80331", "80333", "81667"}}},
City{.city = "Lyon",
.state_province = "Auvergne-Rhone-Alpes",
.iso3166_2 = "FR-ARA",
.country = "France",
.iso3166_1 = "FR",
.local_languages = {"fr"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{5}$)",
.city_regexes = {R"(^690\d{2}$)"},
.examples = {"69001", "69002", "69007"}}},
City{.city = "Brussels",
.state_province = "Brussels-Capital",
.iso3166_2 = "BE-BRU",
.country = "Belgium",
.iso3166_1 = "BE",
.local_languages = {"nl", "fr"},
.postal_code =
PostalCodeSpec{.country_format_regex = R"(^\d{4}$)",
.city_regexes = {R"(^1\d{3}$)"},
.examples = {"1000", "1050", "1180"}}},
},
personas_{
UserPersona{.name = "Hophead Explorer",
.description = "Chases hop-forward IPAs and seeks out "
"taprooms with rotating drafts.",
.style_affinities = {"IPA", "Pale Ale"}},
UserPersona{.name = "Lager Traditionalist",
.description = "Prefers clean, balanced lagers and "
"classic pub styles.",
.style_affinities = {"Lager", "Pilsner"}},
UserPersona{.name = "Sour Curious",
.description = "Seeks out wild ales, sours, and "
"barrel-aged experiments.",
.style_affinities = {"Sour", "Wild Ale"}},
},
forenames_by_country_{
{"US",
ForenameList{
ForenameEntry{.name = "James", .gender = "M"},
ForenameEntry{.name = "Mary", .gender = "F"},
}},
{"DE",
ForenameList{
ForenameEntry{.name = "Lukas", .gender = "M"},
ForenameEntry{.name = "Anna", .gender = "F"},
}},
{"FR",
ForenameList{
ForenameEntry{.name = "Lucas", .gender = "M"},
ForenameEntry{.name = "Camille", .gender = "F"},
}},
{"BE",
ForenameList{
ForenameEntry{.name = "Noah", .gender = "M"},
ForenameEntry{.name = "Emma", .gender = "F"},
}},
},
surnames_by_country_{
{"US", SurnameList{"Smith", "Johnson"}},
{"DE", SurnameList{"Muller", "Schmidt"}},
{"FR", SurnameList{"Martin", "Bernard"}},
{"BE", SurnameList{"Peeters", "Janssens"}},
} {}
const LocationsList& MockCuratedDataService::LoadLocations() {
return locations_;
}
const PersonasList& MockCuratedDataService::LoadPersonas() { return personas_; }
const ForenamesByCountryMap& MockCuratedDataService::LoadForenamesByCountry() {
return forenames_by_country_;
}
const SurnamesByCountryMap& MockCuratedDataService::LoadSurnamesByCountry() {
return surnames_by_country_;
}

View File

@@ -1,5 +1,6 @@
/**
* @file wikipedia/fetch_extract.cc
* @brief WikipediaEnrichmentService::FetchExtract() implementation.
*/
#include <boost/json.hpp>
@@ -14,147 +15,147 @@
using namespace boost;
std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
const std::string cache_key(query);
const std::string cache_key(query);
// 1. Cache Lookup
if (const auto cache_it = this->extract_cache_.find(cache_key);
cache_it != this->extract_cache_.end()) {
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format("Wikipedia: Cache hit for {}!", cache_key)});
}
return cache_it->second;
}
// 1. Cache Lookup
if (const auto cache_it = this->extract_cache_.find(cache_key);
cache_it != this->extract_cache_.end()) {
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format("Wikipedia: Cache hit for {}!",
cache_key)});
}
return cache_it->second;
}
const std::string encoded = this->client_->EncodeURL(cache_key);
const std::string url = std::format(
"https://en.wikipedia.org/w/"
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format=json",
encoded);
const std::string encoded = this->client_->EncodeURL(cache_key);
const std::string url = std::format(
"https://en.wikipedia.org/w/"
"api.php?action=query&titles={}&prop=extracts&explaintext=1&format="
"json",
encoded);
const std::string body = this->client_->Get(url);
{
using namespace std::literals::chrono_literals;
std::this_thread::sleep_for(1s);
}
const std::string body = this->client_->Get(url);
{
using namespace std::literals::chrono_literals;
std::this_thread::sleep_for(1s);
}
// 2. Parse JSON
system::error_code ec;
json::value doc = json::parse(body, ec);
// 2. Parse JSON
system::error_code ec;
json::value doc = json::parse(body, ec);
if (ec) {
if (logger_) {
if (ec) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: JSON parse error for '{}': {}",
std::string(query), ec.message())});
}
return {};
}
// 3. Safe Extraction
const json::object* obj = doc.if_object();
if (obj == nullptr) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Expected root object for '{}'",
std::string(query))});
}
return {};
}
const json::value* query_ptr = obj->if_contains("query");
const json::value* pages_ptr =
((query_ptr != nullptr) && query_ptr->is_object())
? query_ptr->get_object().if_contains("pages")
: nullptr;
if ((pages_ptr == nullptr) || !pages_ptr->is_object()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Missing query.pages for '{}'",
std::string(query))});
}
return {};
}
const json::object& pages = pages_ptr->get_object();
if (pages.empty()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No pages returned for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// Wikipedia returns the page under a dynamic ID key; we just want the first
// one
const json::value& page_val = pages.begin()->value();
if (!page_val.is_object()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Unexpected page format for '{}'",
std::string(query))});
}
return {};
}
const json::object& page = page_val.get_object();
// Handle 404/Missing status
if (page.contains("missing")) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: Page '{}' does not exist",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
const json::value* extract_ptr = page.if_contains("extract");
if ((extract_ptr == nullptr) || !extract_ptr->is_string()) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService: No extract string found for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// 4. Success
std::string extract(extract_ptr->as_string());
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("WikipediaService: JSON parse error for '{}': {}",
std::string(query), ec.message())});
}
return {};
}
{.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format("WikipediaService: Fetched {} chars for '{}'",
extract.size(), std::string(query))});
}
// 3. Safe Extraction
const json::object* obj = doc.if_object();
if (obj == nullptr) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message =
std::format("WikipediaService: Expected root object for '{}'",
std::string(query))});
}
return {};
}
this->extract_cache_.insert_or_assign(cache_key, extract);
const json::value* query_ptr = obj->if_contains("query");
const json::value* pages_ptr =
((query_ptr != nullptr) && query_ptr->is_object())
? query_ptr->get_object().if_contains("pages")
: nullptr;
if ((pages_ptr == nullptr) || !pages_ptr->is_object()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message =
std::format("WikipediaService: Missing query.pages for '{}'",
std::string(query))});
}
return {};
}
const json::object& pages = pages_ptr->get_object();
if (pages.empty()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("WikipediaService: No pages returned for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// Wikipedia returns the page under a dynamic ID key; we just want the first
// one
const json::value& page_val = pages.begin()->value();
if (!page_val.is_object()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message =
std::format("WikipediaService: Unexpected page format for '{}'",
std::string(query))});
}
return {};
}
const json::object& page = page_val.get_object();
// Handle 404/Missing status
if (page.contains("missing")) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = std::format("WikipediaService: Page '{}' does not exist",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
const json::value* extract_ptr = page.if_contains("extract");
if ((extract_ptr == nullptr) || !extract_ptr->is_string()) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message =
std::format("WikipediaService: No extract string found for '{}'",
std::string(query))});
}
this->extract_cache_.emplace(cache_key, "");
return {};
}
// 4. Success
std::string extract(extract_ptr->as_string());
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("WikipediaService: Fetched {} chars for '{}'",
extract.size(), std::string(query))});
}
this->extract_cache_.insert_or_assign(cache_key, extract);
return extract;
return extract;
}

View File

@@ -1,6 +1,6 @@
/**
* @file wikipedia/get_summary.cc
* @brief WikipediaService::GetLocationContext() implementation.
* @brief WikipediaEnrichmentService::GetLocationContext() implementation.
*/
#include <chrono>
@@ -10,61 +10,54 @@
#include "services/enrichment/wikipedia_service.h"
std::string WikipediaEnrichmentService::GetLocationContext(
const Location& loc) {
using namespace std::literals::chrono_literals;
if (!this->client_) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::UserGeneration,
.message = "Wikipedia client is nullptr."});
}
return {};
}
std::string WikipediaEnrichmentService::GetLocationContext(const City& loc) {
using namespace std::literals::chrono_literals;
if (!this->client_) {
if (logger_) {
logger_->Log({.level = LogLevel::Warn,
.phase = PipelinePhase::Enrichment,
.message = "Wikipedia client is nullptr."});
}
return {};
}
std::string result;
std::string result;
// std::string region_query(loc.city);
// if (!loc.country.empty()) {
// region_query += loc.state_province,
// region_query += ", ";
// region_query += loc.country;
// }
constexpr std::string_view brewing_query = "brewing";
const std::string location_query =
std::format("{}, {}", loc.city, loc.iso3166_2);
const std::string beer_query = std::format("beer in {}", loc.country);
constexpr std::string_view brewing_query = "brewing";
const std::string location_query =
std::format("{}, {}", loc.city, loc.iso3166_2);
const std::string beer_query = std::format("beer in {}", loc.country);
auto append_extract = [&result](const std::string& extract) -> void {
if (extract.empty()) {
return;
}
if (!result.empty()) {
result += "\n\n";
}
result += extract;
};
auto append_extract = [&result](const std::string& extract) -> void {
if (extract.empty()) {
return;
}
if (!result.empty()) {
result += "\n\n";
}
result += extract;
};
try {
append_extract(FetchExtract(brewing_query));
append_extract(FetchExtract(beer_query));
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"Done fetching for {}. Sleeping for 10 seconds.",
location_query)});
}
std::this_thread::sleep_for(10s);
try {
append_extract(FetchExtract(brewing_query));
append_extract(FetchExtract(beer_query));
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::UserGeneration,
.message = std::format("Done fetching for {}. Sleeping for 10 seconds.",
location_query)});
}
std::this_thread::sleep_for(10s);
} catch (const std::runtime_error& e) {
if (logger_) {
logger_->Log(
{.level = LogLevel::Debug,
.phase = PipelinePhase::UserGeneration,
.message = std::format("WikipediaService lookup failed for '{}': {}",
location_query, e.what())});
}
}
return result;
} catch (const std::runtime_error& e) {
if (logger_) {
logger_->Log({.level = LogLevel::Debug,
.phase = PipelinePhase::Enrichment,
.message = std::format(
"WikipediaService lookup failed for '{}': {}",
location_query, e.what())});
}
}
return result;
}

View File

@@ -1,6 +1,6 @@
/**
* @file services/wikipedia/wikipedia_service.cc
* @brief WikipediaService constructor implementation.
* @brief WikipediaEnrichmentService constructor implementation.
*/
#include "services/enrichment/wikipedia_service.h"

View File

@@ -16,23 +16,25 @@
namespace {
[[nodiscard]] constexpr std::string_view PipelinePhaseToString(
PipelinePhase phase) {
switch (phase) {
case PipelinePhase::Startup:
return "Startup";
case PipelinePhase::UserGeneration:
return "User Generation";
case PipelinePhase::BreweryAndBeerGeneration:
return "Brewery & Beer Gen";
case PipelinePhase::CheckinGeneration:
return "Checkin Gen";
case PipelinePhase::RatingGeneration:
return "Rating Gen";
case PipelinePhase::FollowGeneration:
return "Follow Gen";
case PipelinePhase::Teardown:
return "Teardown";
}
return "Unknown";
switch (phase) {
case PipelinePhase::Startup:
return "Startup";
case PipelinePhase::Enrichment:
return "Enrichment";
case PipelinePhase::UserGeneration:
return "User Generation";
case PipelinePhase::BreweryAndBeerGeneration:
return "Brewery & Beer Gen";
case PipelinePhase::CheckinGeneration:
return "Checkin Gen";
case PipelinePhase::RatingGeneration:
return "Rating Gen";
case PipelinePhase::FollowGeneration:
return "Follow Gen";
case PipelinePhase::Teardown:
return "Teardown";
}
return "Unknown";
}
} // namespace
@@ -40,35 +42,35 @@ LogDispatcher::LogDispatcher(BoundedChannel<LogEntry>& channel)
: channel_(channel) {}
void LogDispatcher::Run() {
auto logger = spdlog::default_logger();
auto logger = spdlog::default_logger();
while (true) {
auto entry = channel_.Receive();
if (!entry.has_value()) {
// Channel is closed and drained.
break;
}
while (true) {
auto entry = channel_.Receive();
if (!entry.has_value()) {
// Channel is closed and drained.
break;
}
const auto& log = entry.value();
const auto& log = entry.value();
logger->log(ToSpdlogLevel(log.level),
"{:<20} │ thread: {:016x} │ [{}:{}] │ {}",
PipelinePhaseToString(log.phase),
std::hash<std::thread::id>{}(log.thread_id),
log.origin.file_name(), log.origin.line(), log.message);
}
logger->log(ToSpdlogLevel(log.level),
"{:<20} │ thread: {:016x} │ [{}:{}] │ {}",
PipelinePhaseToString(log.phase),
std::hash<std::thread::id>{}(log.thread_id),
log.origin.file_name(), log.origin.line(), log.message);
}
}
spdlog::level::level_enum LogDispatcher::ToSpdlogLevel(LogLevel level) {
switch (level) {
case LogLevel::Debug:
return spdlog::level::debug;
case LogLevel::Info:
return spdlog::level::info;
case LogLevel::Warn:
return spdlog::level::warn;
case LogLevel::Error:
return spdlog::level::err;
}
return spdlog::level::info;
switch (level) {
case LogLevel::Debug:
return spdlog::level::debug;
case LogLevel::Info:
return spdlog::level::info;
case LogLevel::Warn:
return spdlog::level::warn;
case LogLevel::Error:
return spdlog::level::err;
}
return spdlog::level::info;
}

View File

@@ -0,0 +1,44 @@
/**
* @file services/postal_code/xeger_postal_code_service.cc
* @brief XegerPostalCodeService::GeneratePostalCode() implementation.
*/
#include "services/postal_code/xeger_postal_code_service.h"
#include <format>
#include <random>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "services/xeger/xeger.h"
XegerPostalCodeService::XegerPostalCodeService()
: rng_(std::random_device{}()) {}
std::string XegerPostalCodeService::GeneratePostalCode(const City& city) {
const PostalCodeSpec& spec = city.postal_code;
std::vector<std::string_view> city_patterns;
for (const std::string& pattern : spec.city_regexes) {
if (!pattern.empty()) {
city_patterns.push_back(pattern);
}
}
std::string_view pattern;
if (!city_patterns.empty()) {
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
pattern = city_patterns[dist(rng_)];
} else if (!spec.country_format_regex.empty()) {
pattern = spec.country_format_regex;
} else {
throw std::runtime_error(std::format(
"XegerPostalCodeService: no postal-code pattern available for "
"city '{}'",
city.city));
}
return GenerateStringFromRegex(pattern, rng_);
}

View File

@@ -15,9 +15,7 @@
#include <string_view>
#include <utility>
// ---------------------------------------------------------------------------
// PromptDirectory
// ---------------------------------------------------------------------------
PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir)
: PromptDirectory(prompt_dir, nullptr) {}
@@ -25,76 +23,80 @@ PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir)
PromptDirectory::PromptDirectory(const std::filesystem::path& prompt_dir,
std::shared_ptr<ILogger> logger)
: prompt_dir_(prompt_dir), logger_(std::move(logger)) {
std::error_code ec;
std::error_code ec;
// Scenario 4: directory must exist.
if (!std::filesystem::exists(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory does not exist: " +
prompt_dir_.string());
}
// Directory must exist.
if (!std::filesystem::exists(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory does not exist: " +
prompt_dir_.string());
}
// Scenario 4: path must be a directory, not a file.
if (!std::filesystem::is_directory(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory path is not a directory: " +
prompt_dir_.string());
}
// Path must be a directory, not a file.
if (!std::filesystem::is_directory(prompt_dir_, ec) || ec) {
throw std::runtime_error(
"PromptDirectory: prompt directory path is not a directory: " +
prompt_dir_.string());
}
// Scenario 4: directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(
std::format("PromptDirectory: prompt directory is not readable: {} ({})",
prompt_dir_.string(), ec.message()));
}
// Directory must be readable (probe with directory_iterator).
std::filesystem::directory_iterator probe(prompt_dir_, ec);
if (ec) {
throw std::runtime_error(std::format(
"PromptDirectory: prompt directory is not readable: {} ({})",
prompt_dir_.string(), ec.message()));
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
std::string("[PromptDirectory] Resolved prompt directory: ") +
prompt_dir_.string()});
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
std::string("[PromptDirectory] Resolved prompt directory: ") +
prompt_dir_.string()});
}
}
std::string PromptDirectory::Load(std::string_view key) {
const std::string key_str(key);
const std::string key_str(key);
// Return cached content if already loaded during this run.
const auto cache_it = cache_.find(key_str);
if (cache_it != cache_.end()) {
return cache_it->second;
}
// Return cached content if already loaded during this run.
const auto cache_it = cache_.find(key_str);
if (cache_it != cache_.end()) {
return cache_it->second;
}
// Scenario 3: resolve <prompt_dir>/<key>.md and require it to exist.
const std::filesystem::path file_path =
prompt_dir_ / std::filesystem::path(std::format("{}.md", key_str));
// Resolve <prompt_dir>/<key>.md and require it to exist.
const std::filesystem::path file_path =
prompt_dir_ / std::filesystem::path(std::format("{}.md", key_str));
std::ifstream file(file_path);
if (!file.is_open()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file not found for key '{}': {}",
key_str, file_path.string()));
}
std::ifstream file(file_path);
if (!file.is_open()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file not found for key '{}': {}",
key_str, file_path.string()));
}
std::string content((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
file.close();
std::string content((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
file.close();
if (content.empty()) {
throw std::runtime_error(std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
key_str, file_path.string()));
}
if (content.empty()) {
throw std::runtime_error(
std::format("PromptDirectory: prompt file for key '{}' is empty: {}",
key_str, file_path.string()));
}
if (logger_) {
logger_->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format("[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
key_str, file_path.string(), content.size())});
}
if (logger_) {
logger_->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = std::format(
"[PromptDirectory] Loaded prompt '{}' from '{}' ({} chars)",
key_str, file_path.string(), content.size())});
}
cache_.emplace(key_str, content);
return content;
// Cache the loaded content for future calls.
cache_.emplace(key_str, content);
return content;
}

View File

@@ -9,23 +9,26 @@
#include "services/database/sqlite_export_service_helpers.h"
void SqliteExportService::Finalize() {
if (db_handle_ == nullptr) {
return;
}
if (db_handle_ == nullptr) {
return;
}
try {
insert_brewery_stmt_.reset();
insert_location_stmt_.reset();
if (transaction_open_) {
sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
transaction_open_ = false;
}
try {
insert_user_address_stmt_.reset();
insert_user_stmt_.reset();
insert_brewery_address_stmt_.reset();
insert_brewery_stmt_.reset();
insert_city_stmt_.reset();
if (transaction_open_) {
sqlite_export_service_internal::ExecSql(
db_handle_, "COMMIT;", "Failed to commit SQLite transaction");
transaction_open_ = false;
}
db_handle_.reset();
location_cache_.clear();
} catch (...) {
RollbackAndCloseNoThrow();
throw;
}
db_handle_.reset();
city_cache_.clear();
} catch (...) {
RollbackAndCloseNoThrow();
throw;
}
}

Some files were not shown because too many files have changed in this diff Show More