Pipeline: Add LLM and mocked user generation to the pipeline (#230)

tooling/pipeline/includes/services/curated_data/curated_data_service.h

@@ -0,0 +1,50 @@
+#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
+#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_
+
+/**
+ * @file services/curated_data/curated_data_service.h
+ * @brief Abstraction for loading curated location, persona, and name data.
+ */
+
+#include <filesystem>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "data_model/models.h"
+
+using ForenameList = std::vector<ForenameEntry>;
+using SurnameList = std::vector<std::string>;
+using LocationsList = std::vector<Location>;
+using PersonasList = std::vector<UserPersona>;
+using ForenamesByCountryMap = std::unordered_map<std::string, ForenameList>;
+using SurnamesByCountryMap = std::unordered_map<std::string, SurnameList>;
+
+/**
+ * @brief Interface for services that load curated data used to seed
+ * brewery/user generation.
+ */
+class ICuratedDataService {
+ public:
+  ICuratedDataService() = default;
+  virtual ~ICuratedDataService() = default;
+
+  ICuratedDataService(const ICuratedDataService&) = delete;
+  ICuratedDataService& operator=(const ICuratedDataService&) = delete;
+  ICuratedDataService(ICuratedDataService&&) = delete;
+  ICuratedDataService& operator=(ICuratedDataService&&) = delete;
+
+  /**
+   * @brief Loads all curated location records.
+   */
+  virtual const LocationsList& LoadLocations() = 0;
+
+  /**
+   * @brief Loads all curated persona records.
+   */
+  virtual const PersonasList& LoadPersonas() = 0;
+  virtual const ForenamesByCountryMap& LoadForenamesByCountry() = 0;
+  virtual const SurnamesByCountryMap& LoadSurnamesByCountry() = 0;
+};
+
+#endif  // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_CURATED_DATA_SERVICE_H_

tooling/pipeline/includes/services/curated_data/curated_json_data_service.h

@@ -0,0 +1,63 @@
+#ifndef BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
+#define BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_
+
+/**
+ * @file json_handling/json_loader.h
+ * @brief JSON-backed implementation of ICuratedDataService.
+ */
+
+#include <filesystem>
+
+#include "data_model/models.h"
+#include "services/curated_data/curated_data_service.h"
+
+/**
+ * @brief File locations for the curated JSON fixtures consumed by
+ * CuratedJsonDataService.
+ */
+struct CuratedDataFilePaths {
+  std::filesystem::path locations_path;
+  std::filesystem::path personas_path;
+  std::filesystem::path forenames_path;
+  std::filesystem::path surnames_path;
+};
+
+/**
+ * @brief Loads curated location, persona, and name data from JSON files.
+ */
+class CuratedJsonDataService final : public ICuratedDataService {
+  struct cache {
+    LocationsList locations;
+    PersonasList personas;
+    ForenamesByCountryMap forenames_by_country;
+    SurnamesByCountryMap surnames_by_country;
+
+    cache() = default;
+    ~cache() = default;
+  };
+
+  CuratedDataFilePaths filepaths_;
+  cache cache_;
+
+ public:
+  explicit CuratedJsonDataService(CuratedDataFilePaths filepaths);
+
+  /**
+   * @brief Parses a JSON array file and returns all location records.
+   */
+  const LocationsList& LoadLocations() override;
+
+  /**
+   * @brief Parses a JSON array file and returns all persona records.
+   */
+  const PersonasList& LoadPersonas() override;
+
+  const ForenamesByCountryMap& LoadForenamesByCountry() override;
+
+  /**
+   * @brief Parses a JSON file and returns all the forenames per country.
+   */
+  const SurnamesByCountryMap& LoadSurnamesByCountry() override;
+};
+
+#endif  // BIERGARTEN_PIPELINE_INCLUDES_JSON_HANDLING_JSON_LOADER_H_

tooling/pipeline/includes/services/curated_data/mock_curated_data_service.h

@@ -0,0 +1,38 @@
+#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
+#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_
+
+/**
+ * @file services/curated_data/mock_curated_data_service.h
+ * @brief In-memory ICuratedDataService backed by a small fixed dataset, used
+ * when file-backed curated data is disabled (mock mode).
+ */
+
+#include <filesystem>
+
+#include "data_model/models.h"
+#include "services/curated_data/curated_data_service.h"
+
+/**
+ * @brief Curated data service returning a small fixed in-memory dataset in
+ * place of the JSON fixture files used by JsonLoader.
+ */
+class MockCuratedDataService final : public ICuratedDataService {
+ public:
+  MockCuratedDataService();
+
+  const LocationsList& LoadLocations() override;
+
+  const PersonasList& LoadPersonas() override;
+
+  const ForenamesByCountryMap& LoadForenamesByCountry() override;
+
+  const SurnamesByCountryMap& LoadSurnamesByCountry() override;
+
+ private:
+  LocationsList locations_;
+  PersonasList personas_;
+  ForenamesByCountryMap forenames_by_country_;
+  SurnamesByCountryMap surnames_by_country_;
+};
+
+#endif  // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_CURATED_DATA_MOCK_CURATED_DATA_SERVICE_H_

tooling/pipeline/includes/services/database/export_service.h

@@ -16,8 +16,6 @@
 class IExportService {
  public:
   IExportService() = default;
-
-  /// @brief Virtual destructor for polymorphic cleanup.
   virtual ~IExportService() = default;
 
   IExportService(const IExportService&) = delete;
@@ -25,7 +23,9 @@ class IExportService {
   IExportService(IExportService&&) = delete;
   IExportService& operator=(IExportService&&) = delete;
 
-  /// @brief Prepares the export destination for a new run.
+  /**
+   * @brief Prepares the export destination for a new run.
+   */
   virtual void Initialize() = 0;
 
   /**
@@ -33,9 +33,18 @@ class IExportService {
    *
    * @param brewery Generated brewery payload to store.
    */
-  virtual uint64_t ProcessRecord(const GeneratedBrewery& brewery) = 0;
+  virtual uint64_t ProcessRecord(const BreweryRecord& brewery) = 0;
 
-  /// @brief Finalizes the export destination.
+  /**
+   * @brief Persists one generated user record.
+   *
+   * @param user Generated user payload to store.
+   */
+  virtual uint64_t ProcessRecord(const UserRecord& user) = 0;
+
+  /**
+   * @brief Finalizes the export destination.
+   */
   virtual void Finalize() = 0;
 };
 

tooling/pipeline/includes/services/database/sqlite_connection_helpers.h

@@ -12,7 +12,7 @@
 #include <string>
 #include <string_view>
 
-#include "sqlite_handle_types.h"
+#include "services/database/sqlite_handle_types.h"
 
 namespace sqlite_export_service_internal {
 

tooling/pipeline/includes/services/database/sqlite_export_service.h

@@ -12,9 +12,9 @@
 #include <unordered_map>
 
 #include "data_model/models.h"
-#include "../datetime/date_time_provider.h"
-#include "export_service.h"
-#include "sqlite_export_service_helpers.h"
+#include "services/database/export_service.h"
+#include "services/database/sqlite_export_service_helpers.h"
+#include "services/datetime/date_time_provider.h"
 
 /**
  * @brief Persists generated brewery records into a fresh SQLite database.
@@ -30,7 +30,8 @@ class SqliteExportService final : public IExportService {
   SqliteExportService& operator=(SqliteExportService&&) = delete;
 
   void Initialize() override;
-  uint64_t ProcessRecord(const GeneratedBrewery& brewery) override;
+  uint64_t ProcessRecord(const BreweryRecord& brewery) override;
+  uint64_t ProcessRecord(const UserRecord& user) override;
   void Finalize() override;
 
  private:
@@ -46,6 +47,15 @@ class SqliteExportService final : public IExportService {
   [[nodiscard]] std::filesystem::path BuildDatabasePath() const;
   [[nodiscard]] static std::string BuildLocationKey(const Location& location);
 
+  /**
+   * @brief Returns the row id for @p location, inserting it first if it has
+   * not already been seen during this run.
+   *
+   * Shared by both ProcessRecord() overloads so breweries and users
+   * referencing the same location resolve to the same row.
+   */
+  [[nodiscard]] sqlite3_int64 ResolveLocationId(const Location& location);
+
   std::unique_ptr<IDateTimeProvider> date_time_provider_;
   std::filesystem::path output_path_;
   std::string run_timestamp_utc_;
@@ -53,6 +63,7 @@ class SqliteExportService final : public IExportService {
   SqliteDatabaseHandle db_handle_;
   SqliteStatementHandle insert_location_stmt_;
   SqliteStatementHandle insert_brewery_stmt_;
+  SqliteStatementHandle insert_user_stmt_;
   bool transaction_open_ = false;
   std::unordered_map<std::string, sqlite3_int64> location_cache_;
 };

tooling/pipeline/includes/services/database/sqlite_export_service_helpers.h

@@ -3,8 +3,8 @@
 
 /* Umbrella header for backward compatibility. */
 
-#include "sqlite_connection_helpers.h"
-#include "sqlite_handle_types.h"
-#include "sqlite_statement_helpers.h"
+#include "services/database/sqlite_connection_helpers.h"
+#include "services/database/sqlite_handle_types.h"
+#include "services/database/sqlite_statement_helpers.h"
 
 #endif  // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_DATABASE_SQLITE_EXPORT_SERVICE_HELPERS_H_

tooling/pipeline/includes/services/database/sqlite_handle_types.h

@@ -24,8 +24,10 @@ using SqliteDatabaseHandle = std::unique_ptr<sqlite3, SqliteDatabaseDeleter>;
 using SqliteStatementHandle =
     std::unique_ptr<sqlite3_stmt, SqliteStatementDeleter>;
 
+// Represents a parameter that is bound to a prepared SQLite statement.
+// N.B. indices are 1 based.
 template <typename T>
-struct BindParam {
+struct BoundParam {
   int index;
   T value;
   std::string_view action;

tooling/pipeline/includes/services/database/sqlite_statement_helpers.h

@@ -13,7 +13,7 @@
 #include <string_view>
 #include <vector>
 
-#include "sqlite_handle_types.h"
+#include "services/database/sqlite_handle_types.h"
 
 namespace sqlite_export_service_internal {
 
@@ -50,6 +50,26 @@ CREATE INDEX IF NOT EXISTS idx_breweries_location_id ON breweries(location_id);
 
 )sql";
 
+inline constexpr std::string_view kCreateUsersTableSql = R"sql(
+
+CREATE TABLE IF NOT EXISTS users (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  location_id INTEGER NOT NULL,
+  first_name TEXT NOT NULL,
+  last_name TEXT NOT NULL,
+  gender TEXT NOT NULL,
+  username TEXT NOT NULL,
+  bio TEXT NOT NULL,
+  activity_weight REAL NOT NULL,
+  email TEXT NOT NULL UNIQUE,
+  date_of_birth TEXT NOT NULL,
+  FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_users_location_id ON users(location_id);
+
+)sql";
+
 inline constexpr std::string_view kInsertLocationSql = R"sql(
 INSERT INTO locations (
   city,
@@ -73,20 +93,52 @@ INSERT INTO breweries (
 ) VALUES (?, ?, ?, ?, ?);
 )sql";
 
-inline constexpr int kLocationCityBindIndex = 1;
-inline constexpr int kLocationStateProvinceBindIndex = 2;
-inline constexpr int kLocationIso31662BindIndex = 3;
-inline constexpr int kLocationCountryBindIndex = 4;
-inline constexpr int kLocationIso31661BindIndex = 5;
-inline constexpr int kLocationLanguagesBindIndex = 6;
-inline constexpr int kLocationLatitudeBindIndex = 7;
-inline constexpr int kLocationLongitudeBindIndex = 8;
+inline constexpr std::string_view kInsertUserSql = R"sql(
+INSERT INTO users (
+  location_id,
+  first_name,
+  last_name,
+  gender,
+  username,
+  bio,
+  activity_weight,
+  email,
+  date_of_birth
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
+)sql";
 
-inline constexpr int kBreweryLocationIdBindIndex = 1;
-inline constexpr int kBreweryEnglishNameBindIndex = 2;
-inline constexpr int kBreweryEnglishDescriptionBindIndex = 3;
-inline constexpr int kBreweryLocalNameBindIndex = 4;
-inline constexpr int kBreweryLocalDescriptionBindIndex = 5;
+// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
+// placeholder order in the SQL above.
+enum LocationBindIndex {
+  kLocationCityBindIndex = 1,
+  kLocationStateProvinceBindIndex,
+  kLocationIso31662BindIndex,
+  kLocationCountryBindIndex,
+  kLocationIso31661BindIndex,
+  kLocationLanguagesBindIndex,
+  kLocationLatitudeBindIndex,
+  kLocationLongitudeBindIndex,
+};
+
+enum BreweryBindIndex {
+  kBreweryLocationIdBindIndex = 1,
+  kBreweryEnglishNameBindIndex,
+  kBreweryEnglishDescriptionBindIndex,
+  kBreweryLocalNameBindIndex,
+  kBreweryLocalDescriptionBindIndex,
+};
+
+enum UserBindIndex {
+  kUserLocationIdBindIndex = 1,
+  kUserFirstNameBindIndex,
+  kUserLastNameBindIndex,
+  kUserGenderBindIndex,
+  kUserUsernameBindIndex,
+  kUserBioBindIndex,
+  kUserActivityWeightBindIndex,
+  kUserEmailBindIndex,
+  kUserDateOfBirthBindIndex,
+};
 
 SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
                                        std::string_view sql,
@@ -95,13 +147,13 @@ SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
 void ResetStatement(SqliteStatementHandle& statement);
 
 void Bind(const SqliteStatementHandle& statement,
-          const BindParam<std::string_view>& param);
+          const BoundParam<std::string_view>& param);
 
 void Bind(const SqliteStatementHandle& statement,
-          const BindParam<double>& param);
+          const BoundParam<double>& param);
 
 void Bind(const SqliteStatementHandle& statement,
-          const BindParam<sqlite3_int64>& param);
+          const BoundParam<sqlite3_int64>& param);
 
 void StepStatement(const SqliteDatabaseHandle& db_handle,
                    const SqliteStatementHandle& statement,

tooling/pipeline/includes/services/datetime/date_time_provider.h

@@ -18,7 +18,6 @@
  */
 class IDateTimeProvider {
  public:
-  /// @brief Virtual destructor for polymorphic cleanup.
   virtual ~IDateTimeProvider() = default;
 
   IDateTimeProvider() = default;

tooling/pipeline/includes/services/enrichment/enrichment_service.h

@@ -15,7 +15,6 @@
  */
 class IEnrichmentService {
  public:
-  /// @brief Virtual destructor for polymorphic cleanup.
   virtual ~IEnrichmentService() = default;
 
   /**

tooling/pipeline/includes/services/enrichment/mock_enrichment.h

@@ -1,13 +1,18 @@
-//
-// Created by aaronpo on 13/05/2026.
-//
-
 #ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
 #define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ENRICHMENT_MOCK_ENRICHMENT_H_
+
+/**
+ * @file services/enrichment/mock_enrichment.h
+ * @brief No-op IEnrichmentService used when network enrichment is disabled.
+ */
+
 #include <string>
 
-#include "enrichment_service.h"
+#include "services/enrichment/enrichment_service.h"
 
+/**
+ * @brief Enrichment service that returns no context for any location.
+ */
 class MockEnrichmentService final : public IEnrichmentService {
  public:
   std::string GetLocationContext(const Location& /*loc*/) override {

tooling/pipeline/includes/services/enrichment/wikipedia_service.h

@@ -11,25 +11,37 @@
 #include <string_view>
 #include <unordered_map>
 
-#include "enrichment_service.h"
+#include "services/enrichment/enrichment_service.h"
 #include "services/logging/logger.h"
 #include "web_client/web_client.h"
 
-/// @brief Provides Wikipedia summary lookups backed by cached raw extracts.
+/**
+ * @brief Provides Wikipedia summary lookups backed by cached raw extracts.
+ */
 class WikipediaEnrichmentService final : public IEnrichmentService {
  public:
-  /// @brief Creates a new Wikipedia service with the provided web client.
+  /**
+   * @brief Creates a new Wikipedia service with the provided web client.
+   */
   explicit WikipediaEnrichmentService(std::unique_ptr<WebClient> client,
                                       std::shared_ptr<ILogger> logger);
 
-  /// @brief Returns the Wikipedia-derived context for a location.
+  /**
+   * @brief Returns the Wikipedia-derived context for a location.
+   */
   [[nodiscard]] std::string GetLocationContext(const Location& loc) override;
 
  private:
   std::string FetchExtract(std::string_view query);
   std::unique_ptr<WebClient> client_;
   std::shared_ptr<ILogger> logger_;
-  /// @brief Canonical cache for raw Wikipedia query extracts.
+  /**
+   * @brief Cache for raw Wikipedia query extracts, keyed by query string.
+   *
+   * GetLocationContext() always queries "brewing" and reuses "beer in
+   * {country}" for every city in that country, so caching avoids refetching
+   * the same extract across locations in a run.
+   */
   std::unordered_map<std::string, std::string> extract_cache_;
 };
 

tooling/pipeline/includes/services/logging/log_entry.h

@@ -20,10 +20,22 @@
  * @brief Severity levels supported by the logging infra.
  */
 enum class LogLevel {
-  Debug,  ///< Development/debugging information.
-  Info,   ///< General informational messages.
-  Warn,   ///< Warning conditions.
-  Error,  ///< Error conditions.
+  /**
+   * @brief Development/debugging information.
+   */
+  Debug,
+  /**
+   * @brief General informational messages.
+   */
+  Info,
+  /**
+   * @brief Warning conditions.
+   */
+  Warn,
+  /**
+   * @brief Error conditions.
+   */
+  Error,
 };
 
 /**
@@ -34,18 +46,44 @@ enum class LogLevel {
  * pipeline that emitted it.
  */
 enum class PipelinePhase {
-  Startup,                   ///< Initialization and validation.
-  UserGeneration,            ///< User profile generation.
-  BreweryAndBeerGeneration,  ///< Brewery and beer data generation.
-  CheckinGeneration,         ///< Checkin (visit) record generation.
-  RatingGeneration,          ///< Rating and review generation.
-  FollowGeneration,          ///< Follow relationship generation.
-  Teardown,                  ///< Finalization and cleanup.
+  /**
+   * @brief Initialization and validation.
+   */
+  Startup,
+  /**
+   * @brief Location/context enrichment (e.g. Wikipedia lookups).
+   */
+  Enrichment,
+  /**
+   * @brief User profile generation.
+   */
+  UserGeneration,
+  /**
+   * @brief Brewery and beer data generation.
+   */
+  BreweryAndBeerGeneration,
+  /**
+   * @brief Checkin (visit) record generation.
+   */
+  CheckinGeneration,
+  /**
+   * @brief Rating and review generation.
+   */
+  RatingGeneration,
+  /**
+   * @brief Follow relationship generation.
+   */
+  FollowGeneration,
+  /**
+   * @brief Finalization and cleanup.
+   */
+  Teardown,
 };
 
 /**
  * @struct LogDTO
- * @brief User-provided subset of log fields. Used to capture call-site info transparently.
+ * @brief User-provided subset of log fields. Used to capture call-site info
+ * transparently.
  */
 struct LogDTO {
   LogLevel level;
@@ -64,25 +102,35 @@ struct LogDTO {
  * before the entry is dispatched.
  */
 struct LogEntry {
-  /// @brief Timestamp when the entry was created.
+  /**
+   * @brief Timestamp when the entry was created.
+   */
   std::chrono::system_clock::time_point timestamp{};
 
-  /// @brief Source location where the log call was made.
+  /**
+   * @brief Source location where the log call was made.
+   */
   std::source_location origin{};
 
-  /// @brief Thread responsible for emitting the log.
+  /**
+   * @brief Thread responsible for emitting the log.
+   */
   std::thread::id thread_id{};
 
-
-  /// @brief Severity level of this entry.
+  /**
+   * @brief Severity level of this entry.
+   */
   LogLevel level;
 
-  /// @brief Pipeline phase associated with the entry.
+  /**
+   * @brief Pipeline phase associated with the entry.
+   */
   PipelinePhase phase;
 
-  /// @brief Log message text.
+  /**
+   * @brief Log message text.
+   */
   std::string message;
-
 };
 
 #endif  // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_ENTRY_H_

tooling/pipeline/includes/services/logging/log_producer.h

@@ -6,8 +6,8 @@
  * a bounded channel for later processing by the dispatcher.
  */
 
-#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
-#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
+#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
+#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_
 
 #include <string_view>
 
@@ -50,4 +50,4 @@ class LogProducer final : public ILogger {
   BoundedChannel<LogEntry>& channel_;
 };
 
-#endif  // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_CHANNEL_LOGGER_H_
+#endif  // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_LOGGING_LOG_PRODUCER_H_

tooling/pipeline/includes/services/logging/logger.h

@@ -39,7 +39,8 @@ class ILogger {
    */
   void Log(LogDTO payload,
            std::source_location origin = std::source_location::current(),
-           std::chrono::system_clock::time_point timestamp = std::chrono::system_clock::now(),
+           std::chrono::system_clock::time_point timestamp =
+               std::chrono::system_clock::now(),
            std::thread::id thread_id = std::this_thread::get_id()) {
     LogEntry entry;
     entry.timestamp = timestamp;