Update generate pipeline to add OpenAI api as an additional backend (for comparison testing), add brewery coordinates (#244)

* add brewery coordinates

* Add openai mode

* Add coordinates to schema

* Fix C++ review findings in pipeline: k-prefixed enum values, include ordering, populated validation error messages, conventional const notation, and endl replaced with newline

tooling/pipeline/includes/biergarten_pipeline.h

@@ -15,12 +15,15 @@
 
 // --- data_generation ---
 #include "data_generation/data_generator.h"
+#include "data_generation/generation_json_validation.h"
 #include "data_generation/json_grammars.h"
 #ifndef BIERGARTEN_MOCK_ONLY
 #include "data_generation/llama_generator.h"
 #include "data_generation/llama_generator_helpers.h"
 #endif
 #include "data_generation/mock_generator.h"
+#include "data_generation/openai_generator.h"
+#include "data_generation/openai_json_schemas.h"
 #include "data_generation/prompt_formatting/gemma4_jinja_prompt_formatter.h"
 #include "data_generation/prompt_formatting/prompt_formatter.h"
 

tooling/pipeline/includes/data_generation/generation_json_validation.h

@@ -0,0 +1,52 @@
+#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_
+#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_
+
+/**
+ * @file data_generation/generation_json_validation.h
+ * @brief Generator-agnostic helpers for preparing prompt context and
+ * validating structured JSON output. Shared across all DataGenerator
+ * implementations (Llama, OpenAI, ...).
+ */
+
+#include <cstddef>
+#include <optional>
+#include <string>
+#include <string_view>
+
+#include "data_model/generated_models.h"
+
+/**
+ * @brief Normalizes and truncates regional context.
+ *
+ * @param region_context Input regional context text.
+ * @param max_chars Maximum output length.
+ * @return Processed region context.
+ */
+std::string PrepareRegionContext(std::string_view region_context,
+                                 size_t max_chars = 2000);
+
+/**
+ * @brief Validates and parses brewery JSON output.
+ *
+ * @param raw Raw model output.
+ * @param brewery_out Parsed brewery payload.
+ * @return Validation error message if invalid, or std::nullopt on success.
+ */
+std::optional<std::string> ValidateBreweryJson(const std::string& raw,
+                                               BreweryResult& brewery_out);
+
+/**
+ * @brief Validates and parses user JSON output.
+ *
+ * Only populates `username`, `bio`, and `activity_weight` -- `first_name`
+ * and `last_name` are not LLM-authored and are set separately from the
+ * sampled Name.
+ *
+ * @param raw Raw model output.
+ * @param user_out Parsed user payload.
+ * @return Validation error message if invalid, or std::nullopt on success.
+ */
+std::optional<std::string> ValidateUserJson(const std::string& raw,
+                                            UserResult& user_out);
+
+#endif  // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_GENERATION_JSON_VALIDATION_H_

tooling/pipeline/includes/data_generation/llama_generator_helpers.h

@@ -3,30 +3,17 @@
 
 /**
  * @file data_generation/llama_generator_helpers.h
- * @brief Shared helper APIs used by LlamaGenerator translation units.
+ * @brief Llama.cpp-specific helper APIs used by LlamaGenerator translation
+ * units. Generator-agnostic helpers (prompt-context prep, JSON validation)
+ * live in data_generation/generation_json_validation.h instead.
  */
 
-#include <cstddef>
 #include <cstdint>
-#include <optional>
 #include <string>
-#include <string_view>
-
-#include "data_model/generated_models.h"
 
 struct llama_vocab;
 using llama_token = int32_t;
 
-/**
- * @brief Normalizes and truncates regional context.
- *
- * @param region_context Input regional context text.
- * @param max_chars Maximum output length.
- * @return Processed region context.
- */
-std::string PrepareRegionContext(std::string_view region_context,
-                                 size_t max_chars = 2000);
-
 /**
  * @brief Decodes a sampled token and appends it to output text.
  *
@@ -37,28 +24,4 @@ std::string PrepareRegionContext(std::string_view region_context,
 void AppendTokenPiece(const llama_vocab* vocab, llama_token token,
                       std::string& output);
 
-/**
- * @brief Validates and parses brewery JSON output.
- *
- * @param raw Raw model output.
- * @param brewery_out Parsed brewery payload.
- * @return Validation error message if invalid, or std::nullopt on success.
- */
-std::optional<std::string> ValidateBreweryJson(const std::string& raw,
-                                               BreweryResult& brewery_out);
-
-/**
- * @brief Validates and parses user JSON output.
- *
- * Only populates `username`, `bio`, and `activity_weight` -- `first_name`
- * and `last_name` are not LLM-authored and are set separately from the
- * sampled Name.
- *
- * @param raw Raw model output.
- * @param user_out Parsed user payload.
- * @return Validation error message if invalid, or std::nullopt on success.
- */
-std::optional<std::string> ValidateUserJson(const std::string& raw,
-                                            UserResult& user_out);
-
 #endif  // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_LLAMA_GENERATOR_HELPERS_H_

tooling/pipeline/includes/data_generation/openai_generator.h

@@ -0,0 +1,104 @@
+#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_
+#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_
+
+/**
+ * @file data_generation/openai_generator.h
+ * @brief OpenAI Chat Completions API-backed implementation of DataGenerator.
+ */
+
+#include <memory>
+#include <string>
+#include <string_view>
+
+#include "data_generation/data_generator.h"
+#include "data_model/models.h"
+#include "services/logging/logger.h"
+#include "services/prompting/prompt_directory.h"
+#include "web_client/web_client.h"
+
+/**
+ * @brief Data generator implementation backed by the OpenAI Chat
+ * Completions API.
+ *
+ * Unlike LlamaGenerator, this generator has no local model state and no
+ * chat-template formatting step -- Chat Completions takes system/user
+ * messages directly, and Structured Outputs (response_format: json_schema,
+ * strict: true) guarantee schema-valid JSON, replacing llama.cpp's GBNF
+ * grammar constraint.
+ */
+class OpenAIGenerator final : public DataGenerator {
+  public:
+   /**
+    * @brief Constructs a generator that calls the OpenAI Chat Completions
+    * API.
+    *
+    * @param api_key OpenAI API key (from the OPENAI_API_KEY environment
+    * variable -- never accepted as a CLI argument).
+    * @param model OpenAI model ID, e.g. "gpt-4o-mini".
+    * @param logger Sink for pipeline diagnostics.
+    * @param prompt_directory Directory service for loading named prompt
+    * files (the same BREWERY_GENERATION/USER_GENERATION prompts Llama
+    * uses -- they're backend-agnostic).
+    * @param web_client HTTP client used to reach the Chat Completions API.
+    */
+   OpenAIGenerator(std::string api_key, std::string model,
+                   std::shared_ptr<ILogger> logger,
+                   std::unique_ptr<IPromptDirectory> prompt_directory,
+                   std::unique_ptr<WebClient> web_client);
+
+   ~OpenAIGenerator() override = default;
+
+   OpenAIGenerator(const OpenAIGenerator&) = delete;
+   OpenAIGenerator& operator=(const OpenAIGenerator&) = delete;
+   OpenAIGenerator(OpenAIGenerator&&) = delete;
+   OpenAIGenerator& operator=(OpenAIGenerator&&) = delete;
+
+   /**
+    * @brief Generates brewery data for a specific location.
+    *
+    * @param enriched_city Enriched city the brewery is associated with.
+    * @return Generated brewery result.
+    */
+   BreweryResult GenerateBrewery(const EnrichedCity& enriched_city) override;
+
+   /**
+    * @brief Generates a user profile grounded in a sampled name and persona.
+    *
+    * @param city Enriched city the user is associated with.
+    * @param persona Persona archetype grounding the generated bio.
+    * @param name Sampled first/last name -- not LLM-invented.
+    * @return Generated user profile.
+    */
+   UserResult GenerateUser(const EnrichedCity& city, const UserPersona& persona,
+                           const Name& name) override;
+
+  private:
+   /**
+    * @brief Sends a single Chat Completions request with a Structured
+    * Outputs schema and returns the raw JSON text of the response.
+    *
+    * @param system_prompt System role message content.
+    * @param user_prompt User role message content.
+    * @param json_schema Raw JSON Schema text constraining the response
+    * shape (see data_generation/openai_json_schemas.h).
+    * @param schema_name Identifier for the schema, required by the
+    * response_format.json_schema wrapper.
+    * @param max_tokens Maximum completion tokens to generate.
+    * @return Raw JSON text produced by the model.
+    * @throws std::runtime_error on a request/HTTP failure, a safety-policy
+    * refusal, or a response with no message content.
+    */
+   std::string CallChatCompletionsApi(const std::string& system_prompt,
+                                      const std::string& user_prompt,
+                                      std::string_view json_schema,
+                                      std::string_view schema_name,
+                                      int max_tokens);
+
+   std::string api_key_;
+   std::string model_;
+   std::shared_ptr<ILogger> logger_;
+   std::unique_ptr<IPromptDirectory> prompt_directory_;
+   std::unique_ptr<WebClient> web_client_;
+};
+
+#endif  // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_GENERATOR_H_

tooling/pipeline/includes/data_generation/openai_json_schemas.h

@@ -0,0 +1,46 @@
+#ifndef BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_
+#define BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_
+
+/**
+ * @file data_generation/openai_json_schemas.h
+ * @brief JSON Schemas constraining structured output from OpenAIGenerator
+ * Chat Completions requests, via response_format.json_schema (Structured
+ * Outputs, strict mode). This is OpenAI's equivalent of the GBNF grammars
+ * in json_grammars.h -- the API guarantees the response message content is
+ * a single JSON string that validates against the schema, so there's no
+ * grammar-level "thought-block" prefix to account for.
+ *
+ * OpenAI's strict mode requires "additionalProperties": false on every
+ * object in the schema, which these already have.
+ */
+
+#include <string_view>
+
+inline constexpr std::string_view kBreweryJsonSchema = R"json_brewery(
+{
+  "type": "object",
+  "properties": {
+    "name_en": {"type": "string"},
+    "description_en": {"type": "string"},
+    "name_local": {"type": "string"},
+    "description_local": {"type": "string"}
+  },
+  "required": ["name_en", "description_en", "name_local", "description_local"],
+  "additionalProperties": false
+}
+)json_brewery";
+
+inline constexpr std::string_view kUserJsonSchema = R"json_user(
+{
+  "type": "object",
+  "properties": {
+    "username": {"type": "string"},
+    "bio": {"type": "string"},
+    "activity_weight": {"type": "number"}
+  },
+  "required": ["username", "bio", "activity_weight"],
+  "additionalProperties": false
+}
+)json_user";
+
+#endif  // BIERGARTEN_PIPELINE_INCLUDES_DATA_GENERATION_OPENAI_JSON_SCHEMAS_H_

tooling/pipeline/includes/data_model/generated_models.h

@@ -94,6 +94,8 @@ struct EnrichedCity {
 struct BreweryAddress {
    City city;
    std::string postal_code{};
+   double longitude{};
+   double latitude{};
 };
 
 /**

tooling/pipeline/includes/data_model/models.h

@@ -66,6 +66,9 @@ struct City {
     */
    std::string iso3166_1{};
 
+   double longitude{};
+   double latitude{};
+
    /**
     * @brief Local language codes in priority order.
     */
@@ -184,25 +187,47 @@ struct SamplingOptions {
    int n_gpu_layers = 0;
 };
 
+/**
+ * @brief Selects which DataGenerator backend the pipeline uses.
+ */
+enum class GeneratorMode {
+   kMock,
+   kLlama,
+   kOpenAI,
+};
+
 /**
  * @brief Configuration for the LLM generator component.
  */
 struct GeneratorOptions {
    /**
-    * @brief Path to the LLM model file (gguf format).
+    * @brief Path to the LLM model file (gguf format). Only meaningful for
+    * GeneratorMode::kLlama.
     */
    std::filesystem::path model_path;
 
    /**
-    * @brief Use mocked generator instead of actual LLM inference.
+    * @brief Which generator backend to use.
     */
-   bool use_mocked = false;
+   GeneratorMode mode = GeneratorMode::kMock;
 
    /**
-    * @brief Specific sampling parameters for this generator.
-    * If nullopt, the application should use global defaults.
+    * @brief Specific sampling parameters for this generator. Only
+    * meaningful for GeneratorMode::kLlama; if nullopt, LlamaGenerator uses
+    * its own defaults.
     */
    std::optional<SamplingOptions> sampling;
+
+   /**
+    * @brief OpenAI API key, read from the OPENAI_API_KEY environment
+    * variable. Only populated for GeneratorMode::kOpenAI.
+    */
+   std::string openai_api_key;
+
+   /**
+    * @brief OpenAI model ID used by OpenAIGenerator.
+    */
+   std::string openai_model = "gpt-4o-mini";
 };
 
 /**

tooling/pipeline/includes/services/database/sqlite_statement_helpers.h

@@ -26,6 +26,8 @@ CREATE TABLE IF NOT EXISTS cities (
   iso3166_2 TEXT NOT NULL,
   country TEXT NOT NULL,
   iso3166_1 TEXT NOT NULL,
+  longitude REAL NOT NULL,
+  latitude REAL NOT NULL,
   local_languages_json TEXT NOT NULL,
   postal_code_country_format_regex TEXT NOT NULL,
   postal_code_city_regex_json TEXT NOT NULL,
@@ -53,6 +55,8 @@ CREATE TABLE IF NOT EXISTS brewery_addresses (
   brewery_id INTEGER NOT NULL,
   city_id INTEGER NOT NULL,
   postal_code TEXT NOT NULL,
+  longitude REAL NOT NULL,
+  latitude REAL NOT NULL,
   FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
   FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
 );
@@ -105,10 +109,12 @@ INSERT INTO cities (
   iso3166_2,
   country,
   iso3166_1,
+  longitude,
+  latitude,
   local_languages_json,
   postal_code_country_format_regex,
   postal_code_city_regex_json
-) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
 )sql";
 
 inline constexpr std::string_view kInsertBrewerySql = R"sql(
@@ -124,8 +130,10 @@ inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
 INSERT INTO brewery_addresses (
   brewery_id,
   city_id,
-  postal_code
-) VALUES (?, ?, ?);
+  postal_code,
+  longitude,
+  latitude
+) VALUES (?, ?, ?, ?, ?);
 )sql";
 
 inline constexpr std::string_view kInsertUserSql = R"sql(
@@ -157,6 +165,8 @@ enum CityBindIndex {
    kCityIso31662BindIndex,
    kCityCountryBindIndex,
    kCityIso31661BindIndex,
+   kCityLongitudeBindIndex,
+   kCityLatitudeBindIndex,
    kCityLanguagesBindIndex,
    kCityPostalCodeCountryFormatRegexBindIndex,
    kCityPostalCodeCityRegexJsonBindIndex,
@@ -173,6 +183,8 @@ enum BreweryAddressBindIndex {
    kBreweryAddressBreweryIdBindIndex = 1,
    kBreweryAddressCityIdBindIndex,
    kBreweryAddressPostalCodeBindIndex,
+   kBreweryAddressLongitudeBindIndex,
+   kBreweryAddressLatitudeBindIndex,
 };
 
 enum UserBindIndex {

tooling/pipeline/includes/web_client/http_web_client.h

@@ -9,6 +9,7 @@
 #include <memory>
 #include <string>
 #include <utility>
+#include <vector>
 
 #include "services/logging/logger.h"
 #include "web_client/web_client.h"
@@ -39,6 +40,22 @@ class HttpWebClient final : public WebClient {
     */
    std::string Get(const std::string& url) override;
 
+   /**
+    * @brief Executes a blocking HTTP/HTTPS POST request against a full URL.
+    *
+    * Sends @p body with a "Content-Type: application/json" header, plus any
+    * additional headers supplied by the caller (e.g. authentication).
+    *
+    * @param url Fully-qualified URL.
+    * @param body Request body, sent as-is.
+    * @param headers Additional request headers.
+    * @return Response body on HTTP 2xx; throws std::runtime_error otherwise.
+    */
+   std::string Post(
+       const std::string& url, const std::string& body,
+       const std::vector<std::pair<std::string, std::string>>& headers)
+       override;
+
    /**
     * @brief Percent-encodes a single URI component (query parameter value or
     *        path segment). Delegates to httplib::encode_uri_component().

tooling/pipeline/includes/web_client/web_client.h

@@ -7,6 +7,8 @@
  */
 
 #include <string>
+#include <utility>
+#include <vector>
 
 /**
  * @brief Abstract web client interface.
@@ -23,6 +25,20 @@ class WebClient {
     */
    virtual std::string Get(const std::string& url) = 0;
 
+   /**
+    * @brief Executes an HTTP POST request with a JSON body.
+    *
+    * @param url Request URL.
+    * @param body Request body (sent as-is; callers are responsible for
+    * providing valid JSON when the target API expects it).
+    * @param headers Additional request headers (e.g. authentication),
+    * sent alongside a fixed "Content-Type: application/json" header.
+    * @return Response body.
+    */
+   virtual std::string Post(
+       const std::string& url, const std::string& body,
+       const std::vector<std::pair<std::string, std::string>>& headers) = 0;
+
    /**
     * @brief URL-encodes a string value.
     *