update pipeline docs

docs/pipeline/diagrams/current/activity.puml

@@ -18,23 +18,40 @@ skinparam ActivityBarColor #628A5B
 skinparam SwimlaneBorderColor #547461
 skinparam SwimlaneBorderThickness 0.3
 
-title The Biergarten Data Pipeline (Streaming Architecture)
+title The Biergarten Data Pipeline (Current — Synchronous Data Path)
 
 |#F2F6F0|main.cc|
 start
+:Create BoundedChannel<LogEntry> log_channel;
+:Spawn log dispatcher thread\n(LogDispatcher::Run());
+note right
+  The only concurrency in the current pipeline:
+  a dedicated thread drains log_channel and
+  forwards entries to spdlog for the entire run.
+  Joined during shutdown after log_channel closes.
+end note
 :ParseArguments(argc, argv);
 if (Are arguments valid?) then (no)
-  :spdlog::error usage info;
+  :Log error / usage info;
   stop
 else (yes)
 endif
 
 :Init OpenSSL global state & LlamaBackendState;
 :di::make_injector(...);
-:injector.create<std::unique_ptr<BiergartenDataGenerator>>();
-:BiergartenDataGenerator::Run();
+note right
+  Binds ILogger, IEnrichmentService, DataGenerator,
+  IExportService, IPromptFormatter, WebClient based
+  on --mocked vs. model-backed mode.
+end note
+:injector.create<std::unique_ptr<BiergartenPipelineOrchestrator>>();
+:BiergartenPipelineOrchestrator::Run();
+note right
+  Run() itself is synchronous — sampling, enrichment,
+  generation, and export all happen on this thread.
+end note
 
-|#EAF0E8|BiergartenDataGenerator|
+|#EAF0E8|BiergartenPipelineOrchestrator|
 :Initialize SQLite export;
 
 |#E0EAE0|SqliteExportService|
@@ -48,23 +65,35 @@ note right
   Begins Transaction
 end note
 
-|#EAF0E8|BiergartenDataGenerator|
+|#EAF0E8|BiergartenPipelineOrchestrator|
 :QueryCitiesWithCountries();
 
 |#E2EBDC|JsonLoader|
 :JsonLoader::LoadLocations("locations.json");
-:std::ranges::sample(all_locations, 50);
+:std::ranges::sample(all_locations, --location-count);
+note right
+  --location-count defaults to 10.
+end note
 
-|#EAF0E8|BiergartenDataGenerator|
+|#EAF0E8|BiergartenPipelineOrchestrator|
 while (For each sampled Location?) is (Remaining cities)
-  |#DCE8D8|WikipediaService|
+  |#DCE8D8|IEnrichmentService|
   :GetLocationContext(loc);
-  :FetchExtracts(City, Country, Beer);
-  |#EAF0E8|BiergartenDataGenerator|
-  :Store EnrichedCity{Location, region_context};
+  note right
+    WikipediaEnrichmentService fetches a generic
+    "brewing" extract and a "beer in {country}" extract
+    (not a city/region-specific page). MockEnrichmentService
+    (--mocked) returns an empty string instead.
+  end note
+  |#EAF0E8|BiergartenPipelineOrchestrator|
+  if (Lookup failed?) then (yes)
+    :Log warning, skip city;
+  else (no)
+    :Store EnrichedCity{Location, region_context};
+  endif
 endwhile (Done)
 
-|#EAF0E8|BiergartenDataGenerator|
+|#EAF0E8|BiergartenPipelineOrchestrator|
 :GenerateBreweries(enriched_cities);
 
 |#E5EDE1|DataGenerator|
@@ -73,7 +102,10 @@ while (For each EnrichedCity?) is (Remaining cities)
     :DeterministicHash & Format;
   else (LlamaGenerator)
     :PrepareRegionContext;
-    :LoadBrewerySystemPrompt("prompts/system.md");
+    :prompt_directory_->Load("BREWERY_GENERATION");
+    note right
+      Resolves to BREWERY_GENERATION.md inside --prompt-dir.
+    end note
     repeat
       :Infer(system_prompt, user_prompt, max_tokens, kBreweryJsonGrammar);
       :ValidateBreweryJson(raw, brewery);
@@ -81,11 +113,16 @@ while (For each EnrichedCity?) is (Remaining cities)
         break
       else (no)
         :Attempt++;
+        :Append validation error to retry prompt;
       endif
     repeat while (Attempt < 3?) is (yes)
+    note right
+      max_tokens is fixed across retries —
+      it is not raised on truncation.
+    end note
   endif
 
-  |#EAF0E8|BiergartenDataGenerator|
+  |#EAF0E8|BiergartenPipelineOrchestrator|
   if (Generation successful?) then (yes)
     |#E0EAE0|SqliteExportService|
     :ProcessRecord(GeneratedBrewery);
@@ -97,8 +134,8 @@ while (For each EnrichedCity?) is (Remaining cities)
     :Insert Brewery (FK: location_id);
 
     if (Exception caught during insert?) then (yes)
-      |#EAF0E8|BiergartenDataGenerator|
-      :spdlog::warn "Failed to stream record to SQLite export";
+      |#EAF0E8|BiergartenPipelineOrchestrator|
+      :Log warning "SQLite export failed";
       note right
         Data loss is prevented per-record.
         The pipeline continues running.
@@ -106,7 +143,7 @@ while (For each EnrichedCity?) is (Remaining cities)
     else (no)
     endif
   else (no)
-    :spdlog::warn "Generation failed, skipping...";
+    :Log warning "Generation failed, skipping...";
   endif
   |#E5EDE1|DataGenerator|
 endwhile (Done)
@@ -119,6 +156,8 @@ note right
 end note
 
 |#F2F6F0|main.cc|
+:Close log_channel;
+:Join log dispatcher thread;
 :Return 0;
 stop
 

docs/pipeline/diagrams/current/class.puml

@@ -25,11 +25,12 @@ skinparam note {
 
 title The Biergarten Data Pipeline - Class Diagram
 
-class BiergartenDataGenerator {
+class BiergartenPipelineOrchestrator {
   - logger_ : std::shared_ptr<ILogger>
   - context_service_ : std::unique_ptr<IEnrichmentService>
   - generator_ : std::unique_ptr<DataGenerator>
   - exporter_ : std::unique_ptr<IExportService>
+  - application_options_ : ApplicationOptions
   - generated_breweries_ : std::vector<GeneratedBrewery>
   + Run() : bool
   - QueryCitiesWithCountries() : std::vector<Location>
@@ -54,21 +55,29 @@ class PipelinePhase <<enumeration>> {
   Teardown
 }
 
-struct LogEntry {
-  + timestamp : std::chrono::system_clock::time_point
+struct LogDTO {
+  + level : LogLevel
+  + phase : PipelinePhase
+  + message : std::string
+}
+
+struct LogEntry {
+  + timestamp : std::chrono::system_clock::time_point
+  + origin : std::source_location
+  + thread_id : std::thread::id
   + level : LogLevel
   + phase : PipelinePhase
   + message : std::string
-  + worker : std::optional<std::string>
 }
 
 interface ILogger <<interface>> {
-  + Log(entry : const LogEntry&) : void
+  + Log(payload : LogDTO) : void
+  - {abstract} DoLog(entry : LogEntry) : void
 }
 
 class LogProducer {
   - channel_ : BoundedChannel<LogEntry>&
-  + Log(entry : const LogEntry&) : void
+  - DoLog(entry : LogEntry) : void
 }
 
 class LogDispatcher {
@@ -81,7 +90,11 @@ interface IEnrichmentService <<interface>> {
   + GetLocationContext(loc : const Location&) : std::string
 }
 
-class WikipediaService {
+class MockEnrichmentService {
+  + GetLocationContext(loc : const Location&) : std::string
+}
+
+class WikipediaEnrichmentService {
   - client_ : std::unique_ptr<WebClient>
   - extract_cache_ : std::unordered_map<std::string, std::string>
   + GetLocationContext(loc : const Location&) : std::string
@@ -113,13 +126,13 @@ class LlamaGenerator {
   - model_ : ModelHandle
   - context_ : ContextHandle
   - prompt_formatter_ : std::unique_ptr<IPromptFormatter>
+  - prompt_directory_ : std::unique_ptr<IPromptDirectory>
   - rng_ : std::mt19937
   + GenerateBrewery(...) : BreweryResult
   + GenerateUser(...) : UserResult
   - Load(model_path : const std::string&) : void
   - Infer(...) : std::string
   - InferFormatted(...) : std::string
-  - LoadBrewerySystemPrompt(...) : std::string
 }
 
 interface IPromptFormatter <<interface>> {
@@ -130,6 +143,16 @@ class Gemma4JinjaPromptFormatter {
   + Format(system_prompt : std::string_view, user_prompt : std::string_view) : std::string
 }
 
+interface IPromptDirectory <<interface>> {
+  + Load(key : std::string_view) : std::string
+}
+
+class PromptDirectory {
+  - prompt_dir_ : std::filesystem::path
+  - cache_ : std::unordered_map<std::string, std::string>
+  + Load(key : std::string_view) : std::string
+}
+
 class JsonLoader {
   + {static} LoadLocations(filepath : const std::filesystem::path&) : std::vector<Location>
 }
@@ -164,19 +187,22 @@ class SystemDateTimeProvider {
 }
 
 ' Structural Relationships / Dependency Injection
-BiergartenDataGenerator *-- ILogger : owns
-BiergartenDataGenerator *-- IEnrichmentService : owns
-BiergartenDataGenerator *-- DataGenerator : owns
-BiergartenDataGenerator *-- IExportService : owns
+BiergartenPipelineOrchestrator *-- ILogger : owns
+BiergartenPipelineOrchestrator *-- IEnrichmentService : owns
+BiergartenPipelineOrchestrator *-- DataGenerator : owns
+BiergartenPipelineOrchestrator *-- IExportService : owns
 
 LogEntry *-- LogLevel
 LogEntry *-- PipelinePhase
+LogDTO *-- LogLevel
+LogDTO *-- PipelinePhase
 ILogger <|.. LogProducer : implements
 LogProducer ..> LogEntry : emits
 LogDispatcher ..> LogEntry : consumes
 
-IEnrichmentService <|.. WikipediaService : implements
-WikipediaService *-- WebClient : owns
+IEnrichmentService <|.. WikipediaEnrichmentService : implements
+IEnrichmentService <|.. MockEnrichmentService : implements
+WikipediaEnrichmentService *-- WebClient : owns
 
 WebClient <|.. HttpWebClient : implements
 
@@ -184,10 +210,12 @@ DataGenerator <|.. MockGenerator : implements
 DataGenerator <|.. LlamaGenerator : implements
 
 LlamaGenerator *-- IPromptFormatter : uses
+LlamaGenerator *-- IPromptDirectory : uses
 
 IPromptFormatter <|.. Gemma4JinjaPromptFormatter : implements
+IPromptDirectory <|.. PromptDirectory : implements
 
-BiergartenDataGenerator ..> JsonLoader : uses
+BiergartenPipelineOrchestrator ..> JsonLoader : uses
 
 IExportService <|.. SqliteExportService : implements
 SqliteExportService *-- IDateTimeProvider : owns