Add reverse geocoding for brewery street addresses (#245)

* Add geocoding to seed application, remove postal codes from generator

Update comments
This commit is contained in:
2026-08-23 23:53:56 -04:00
committed by Aaron Po
parent e738fe484a
commit cd5fd2d998
41 changed files with 2966 additions and 2297 deletions

2
.gitignore vendored
View File

@@ -1,5 +1,5 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
scratchpad.txt
# dependencies # dependencies
/node_modules /node_modules
/.pnp /.pnp

View File

@@ -18,7 +18,7 @@ low-resource language failures.
- [Known issues](#known-issues) - [Known issues](#known-issues)
- [Hallucinated brewing techniques](#hallucinated-brewing-techniques) - [Hallucinated brewing techniques](#hallucinated-brewing-techniques)
- [Low-resource language hallucination](#low-resource-language-hallucination) - [Low-resource language hallucination](#low-resource-language-hallucination)
- [Synthetic postal codes](#synthetic-postal-codes) - [Synthetic coordinates](#synthetic-coordinates)
--- ---
@@ -284,20 +284,17 @@ Output sample: [./french-cities.example](french-cities.example)
SQLite schema so downstream applications can filter or flag potentially SQLite schema so downstream applications can filter or flag potentially
hallucinated text by language tier. hallucinated text by language tier.
### Synthetic postal codes ### Synthetic coordinates
When the pipeline is run without `--mocked`, `XegerPostalCodeService` generates Each brewery and user address carries a longitude/latitude pair, sampled
a fresh postal code per brewery/user address by reverse-matching one of the uniformly at random within a 5 km disc centred on the city's curated
city's curated postal-code regexes from `locations.json` (falling back to the `latitude`/`longitude` from `locations.json`.
country format regex), rather than returning one of `MockPostalCodeService`'s
curated real-world examples.
The generated code is **format-conformant by construction** — it always The resulting point is **plausible by construction** — it always falls within
satisfies the regex it was generated from — but it is not looked up from any the city's vicinity — but it is not looked up from any real address registry.
real postal registry. It does not correspond to a real address, business, or It does not correspond to a real address, business, or resident, and should
resident, and should not be treated as accurate geocoding or delivery data. This not be treated as accurate geocoding data. This is consistent with the rest of
is consistent with the rest of the dataset: fixture data for a proof-of-concept, the dataset: fixture data for a proof-of-concept, not a source of truth.
not a source of truth.
--- ---

View File

@@ -32,7 +32,9 @@ else()
if(CUDAToolkit_FOUND) if(CUDAToolkit_FOUND)
message(STATUS "[biergarten] NVIDIA GPU detected — enabling CUDA acceleration.") message(STATUS "[biergarten] NVIDIA GPU detected — enabling CUDA acceleration.")
set(GGML_CUDA ON CACHE BOOL "Enable CUDA for NVIDIA GPUs" FORCE) set(GGML_CUDA ON CACHE BOOL "Enable CUDA for NVIDIA GPUs" FORCE)
set(CMAKE_CUDA_ARCHITECTURES native) if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_CUDA_ARCHITECTURES native)
endif()
elseif(hip_FOUND OR DEFINED ENV{ROCM_PATH} OR EXISTS "/opt/rocm") elseif(hip_FOUND OR DEFINED ENV{ROCM_PATH} OR EXISTS "/opt/rocm")
message(STATUS "[biergarten] AMD GPU detected — enabling HIP/ROCm acceleration.") message(STATUS "[biergarten] AMD GPU detected — enabling HIP/ROCm acceleration.")
set(GGML_HIPBLAS ON CACHE BOOL "Enable HIP for AMD GPUs" FORCE) set(GGML_HIPBLAS ON CACHE BOOL "Enable HIP for AMD GPUs" FORCE)
@@ -149,7 +151,6 @@ add_executable(${PROJECT_NAME}
target_sources(${PROJECT_NAME} PRIVATE target_sources(${PROJECT_NAME} PRIVATE
src/main.cc src/main.cc
) )
# --- json_handling --- # --- json_handling ---
target_sources(${PROJECT_NAME} PRIVATE target_sources(${PROJECT_NAME} PRIVATE
src/services/curated_data/curated_json_data_service.cc src/services/curated_data/curated_json_data_service.cc
@@ -224,11 +225,6 @@ target_sources(${PROJECT_NAME} PRIVATE
src/services/enrichment/wikipedia/get_summary.cc src/services/enrichment/wikipedia/get_summary.cc
) )
# --- services: postal_code ---
target_sources(${PROJECT_NAME} PRIVATE
src/services/postal_code/xeger_postal_code_service.cc
)
# --- services: xeger --- # --- services: xeger ---
target_sources(${PROJECT_NAME} PRIVATE target_sources(${PROJECT_NAME} PRIVATE
src/services/xeger/xeger.cc src/services/xeger/xeger.cc

View File

@@ -61,10 +61,7 @@
#include "services/enrichment/mock_enrichment.h" #include "services/enrichment/mock_enrichment.h"
#include "services/enrichment/wikipedia_service.h" #include "services/enrichment/wikipedia_service.h"
// --- services: postal_code --- // --- services: xeger ---
#include "services/postal_code/mock_postal_code_service.h"
#include "services/postal_code/postal_code_service.h"
#include "services/postal_code/xeger_postal_code_service.h"
#include "services/xeger/xeger.h" #include "services/xeger/xeger.h"
// --- services: logging --- // --- services: logging ---

View File

@@ -16,7 +16,6 @@
#include "services/database/export_service.h" #include "services/database/export_service.h"
#include "services/enrichment/enrichment_service.h" #include "services/enrichment/enrichment_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
#include "services/postal_code/postal_code_service.h"
/** /**
* @brief Main orchestrator for the Biergarten data generation pipeline. * @brief Main orchestrator for the Biergarten data generation pipeline.
@@ -35,8 +34,6 @@ class BiergartenPipelineOrchestrator {
* @param exporter Database backend for persisting generated records. * @param exporter Database backend for persisting generated records.
* @param curated_data_service Loads curated location, persona, and name * @param curated_data_service Loads curated location, persona, and name
* data used to seed generation. * data used to seed generation.
* @param postal_code_service Resolves a postal code for a city from its
* curated postal-code data.
* @param application_options CLI configuration and paths. * @param application_options CLI configuration and paths.
*/ */
BiergartenPipelineOrchestrator( BiergartenPipelineOrchestrator(
@@ -45,7 +42,6 @@ class BiergartenPipelineOrchestrator {
std::unique_ptr<DataGenerator> generator, std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options); const ApplicationOptions& application_options);
/** /**
@@ -82,7 +78,6 @@ class BiergartenPipelineOrchestrator {
*/ */
std::unique_ptr<IExportService> exporter_; std::unique_ptr<IExportService> exporter_;
std::unique_ptr<ICuratedDataService> curated_data_service_; std::unique_ptr<ICuratedDataService> curated_data_service_;
std::unique_ptr<IPostalCodeService> postal_code_service_;
ApplicationOptions application_options_; ApplicationOptions application_options_;

View File

@@ -0,0 +1,58 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_
/**
* @file biergarten_pipeline_orchestrator/coords.h
* @brief Shared coordinate jitter helper used to place generated
* brewery/user addresses near a city's centre.
*/
#include <cmath>
#include <numbers>
#include <random>
namespace biergarten_pipeline_orchestrator_internal {
struct Coords {
double longitude;
double latitude;
};
inline constexpr double kEarthRadiusKm = 6371.0;
inline double DegreesToRadians(double degrees) {
return degrees * std::numbers::pi / 180.0;
}
inline double RadiansToDegrees(double radians) {
return radians * 180.0 / std::numbers::pi;
}
/**
* @brief Samples a point uniformly at random within @p distance_km of
* @p centre.
*/
inline Coords GetRandomCoordsWithinRange(const Coords& centre,
const int distance_km,
std::mt19937& rng) {
std::uniform_real_distribution<double> angle_dist(0.0,
2.0 * std::numbers::pi);
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
const double angle_radians = angle_dist(rng);
// sqrt() keeps points uniformly distributed over the disc's area.
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
const double delta_latitude_degrees =
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
const double delta_longitude_degrees = RadiansToDegrees(
radius_km * std::sin(angle_radians) /
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
.latitude = centre.latitude + delta_latitude_degrees};
}
} // namespace biergarten_pipeline_orchestrator_internal
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_

View File

@@ -3,9 +3,9 @@
/** /**
* @file data_generation/generation_json_validation.h * @file data_generation/generation_json_validation.h
* @brief Generator-agnostic helpers for preparing prompt context and * @brief Generator-agnostic helpers, shared across all DataGenerator
* validating structured JSON output. Shared across all DataGenerator * implementations (Llama, OpenAI, ...), for preparing prompt context and
* implementations (Llama, OpenAI, ...). * validating structured JSON output.
*/ */
#include <cstddef> #include <cstddef>
@@ -36,11 +36,9 @@ std::optional<std::string> ValidateBreweryJson(const std::string& raw,
BreweryResult& brewery_out); BreweryResult& brewery_out);
/** /**
* @brief Validates and parses user JSON output. * @brief Validates and parses user JSON output, populating only
* * `username`, `bio`, and `activity_weight` (`first_name` and `last_name`
* Only populates `username`, `bio`, and `activity_weight` -- `first_name` * come from the sampled Name).
* and `last_name` are not LLM-authored and are set separately from the
* sampled Name.
* *
* @param raw Raw model output. * @param raw Raw model output.
* @param user_out Parsed user payload. * @param user_out Parsed user payload.

View File

@@ -18,13 +18,9 @@
/** /**
* @brief Data generator implementation backed by the OpenAI Chat * @brief Data generator implementation backed by the OpenAI Chat
* Completions API. * Completions API, sending system/user messages directly and using
* * Structured Outputs (response_format: json_schema, strict: true) to
* Unlike LlamaGenerator, this generator has no local model state and no * guarantee schema-valid JSON.
* chat-template formatting step -- Chat Completions takes system/user
* messages directly, and Structured Outputs (response_format: json_schema,
* strict: true) guarantee schema-valid JSON, replacing llama.cpp's GBNF
* grammar constraint.
*/ */
class OpenAIGenerator final : public DataGenerator { class OpenAIGenerator final : public DataGenerator {
public: public:

View File

@@ -4,14 +4,9 @@
/** /**
* @file data_generation/openai_json_schemas.h * @file data_generation/openai_json_schemas.h
* @brief JSON Schemas constraining structured output from OpenAIGenerator * @brief JSON Schemas constraining structured output from OpenAIGenerator
* Chat Completions requests, via response_format.json_schema (Structured * Chat Completions requests via response_format.json_schema (Structured
* Outputs, strict mode). This is OpenAI's equivalent of the GBNF grammars * Outputs, strict mode), each setting "additionalProperties": false as
* in json_grammars.h -- the API guarantees the response message content is * strict mode requires.
* a single JSON string that validates against the schema, so there's no
* grammar-level "thought-block" prefix to account for.
*
* OpenAI's strict mode requires "additionalProperties": false on every
* object in the schema, which these already have.
*/ */
#include <string_view> #include <string_view>

View File

@@ -88,23 +88,23 @@ struct EnrichedCity {
}; };
/** /**
* @brief A physical address: the city plus a concrete postal code selected * @brief A physical address: the city plus a coordinate pair sampled near
* from that city's curated postal-code examples. * the city's centre.
*/ */
struct BreweryAddress { struct BreweryAddress {
City city; City city;
std::string postal_code{};
double longitude{}; double longitude{};
double latitude{}; double latitude{};
}; };
/** /**
* @brief A physical address for a generated user: the city plus a concrete * @brief A physical address for a generated user: the city plus a
* postal code selected from that city's curated postal-code examples. * coordinate pair sampled near the city's centre.
*/ */
struct UserAddress { struct UserAddress {
City city; City city;
std::string postal_code{}; double longitude{};
double latitude{};
}; };
/** /**

View File

@@ -54,7 +54,6 @@ CREATE TABLE IF NOT EXISTS brewery_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
brewery_id INTEGER NOT NULL, brewery_id INTEGER NOT NULL,
city_id INTEGER NOT NULL, city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL,
longitude REAL NOT NULL, longitude REAL NOT NULL,
latitude REAL NOT NULL, latitude REAL NOT NULL,
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE, FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
@@ -90,7 +89,8 @@ CREATE TABLE IF NOT EXISTS user_addresses (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL, user_id INTEGER NOT NULL,
city_id INTEGER NOT NULL, city_id INTEGER NOT NULL,
postal_code TEXT NOT NULL, longitude REAL NOT NULL,
latitude REAL NOT NULL,
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE, FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
); );
@@ -130,10 +130,9 @@ inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
INSERT INTO brewery_addresses ( INSERT INTO brewery_addresses (
brewery_id, brewery_id,
city_id, city_id,
postal_code,
longitude, longitude,
latitude latitude
) VALUES (?, ?, ?, ?, ?); ) VALUES (?, ?, ?, ?);
)sql"; )sql";
inline constexpr std::string_view kInsertUserSql = R"sql( inline constexpr std::string_view kInsertUserSql = R"sql(
@@ -153,8 +152,9 @@ inline constexpr std::string_view kInsertUserAddressSql = R"sql(
INSERT INTO user_addresses ( INSERT INTO user_addresses (
user_id, user_id,
city_id, city_id,
postal_code longitude,
) VALUES (?, ?, ?); latitude
) VALUES (?, ?, ?, ?);
)sql"; )sql";
// sqlite3_bind_*() parameter indices are 1-based, matching the "?" // sqlite3_bind_*() parameter indices are 1-based, matching the "?"
@@ -182,7 +182,6 @@ enum BreweryBindIndex {
enum BreweryAddressBindIndex { enum BreweryAddressBindIndex {
kBreweryAddressBreweryIdBindIndex = 1, kBreweryAddressBreweryIdBindIndex = 1,
kBreweryAddressCityIdBindIndex, kBreweryAddressCityIdBindIndex,
kBreweryAddressPostalCodeBindIndex,
kBreweryAddressLongitudeBindIndex, kBreweryAddressLongitudeBindIndex,
kBreweryAddressLatitudeBindIndex, kBreweryAddressLatitudeBindIndex,
}; };
@@ -201,7 +200,8 @@ enum UserBindIndex {
enum UserAddressBindIndex { enum UserAddressBindIndex {
kUserAddressUserIdBindIndex = 1, kUserAddressUserIdBindIndex = 1,
kUserAddressCityIdBindIndex, kUserAddressCityIdBindIndex,
kUserAddressPostalCodeBindIndex, kUserAddressLongitudeBindIndex,
kUserAddressLatitudeBindIndex,
}; };
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle, SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,

View File

@@ -1,80 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/mock_postal_code_service.h
* @brief Data-driven postal code service that selects the first curated
* example for a city and validates it against the city's postal-code regex.
*/
#include <format>
#include <regex>
#include <stdexcept>
#include <string>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service backed entirely by the curated locations
* dataset: it returns the city's first example postal code after checking it
* against the curated postal-code regex.
*
* A precise city regex is preferred when present; otherwise the permissive
* country format regex is used as the fallback. A city with no examples, or an
* example that fails validation, throws so the caller can skip the city.
*/
class MockPostalCodeService final : public IPostalCodeService {
public:
std::string GeneratePostalCode(const City& city) override {
const PostalCodeSpec& spec = city.postal_code;
if (spec.examples.empty()) {
throw std::runtime_error(std::format(
"No postal-code examples available for city '{}'", city.city));
}
const std::string& candidate = spec.examples.front();
if (!MatchesAnyPattern(candidate, spec)) {
throw std::runtime_error(
std::format("Postal code '{}' for city '{}' does not match the "
"curated postal-code regex",
candidate, city.city));
}
return candidate;
}
private:
/**
* @brief Returns true if @p candidate matches any city regex, or the
* country format regex when no city regex matches. When the spec carries no
* usable regex at all, validation is skipped (returns true).
*/
static bool MatchesAnyPattern(const std::string& candidate,
const PostalCodeSpec& spec) {
bool has_pattern = false;
for (const std::string& pattern : spec.city_regexes) {
if (pattern.empty()) {
continue;
}
has_pattern = true;
if (std::regex_match(candidate, std::regex(pattern))) {
return true;
}
}
if (!spec.country_format_regex.empty()) {
has_pattern = true;
if (std::regex_match(candidate,
std::regex(spec.country_format_regex))) {
return true;
}
}
return !has_pattern;
}
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_

View File

@@ -1,35 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/postal_code_service.h
* @brief Abstraction for resolving a postal code for a city.
*/
#include <string>
#include "data_model/models.h"
/**
* @brief Interface for services that resolve a postal code for a city.
*/
class IPostalCodeService {
public:
IPostalCodeService() = default;
virtual ~IPostalCodeService() = default;
IPostalCodeService(const IPostalCodeService&) = delete;
IPostalCodeService& operator=(const IPostalCodeService&) = delete;
IPostalCodeService(IPostalCodeService&&) = delete;
IPostalCodeService& operator=(IPostalCodeService&&) = delete;
/**
* @brief Resolves a postal code for @p city.
*
* @param city City whose curated postal-code examples supply the result.
* @return A postal code string, or an empty string if none is available.
*/
virtual std::string GeneratePostalCode(const City& city) = 0;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_

View File

@@ -1,32 +0,0 @@
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
/**
* @file services/postal_code/xeger_postal_code_service.h
* @brief Postal code service that generates a fresh, format-conformant code
* per call from the curated regex patterns, instead of returning a fixed
* example.
*/
#include <random>
#include "services/postal_code/postal_code_service.h"
/**
* @brief Postal code service using regex-driven generation (xeger).
*
* Picks among the city's curated `city_regexes` (falls back to
* `country_format_regex`), then generates a random string matching that pattern
* via GenerateStringFromRegex().
*/
class XegerPostalCodeService final : public IPostalCodeService {
public:
XegerPostalCodeService();
std::string GeneratePostalCode(const City& city) override;
private:
std::mt19937 rng_;
};
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_

File diff suppressed because it is too large Load Diff

View File

@@ -38,9 +38,8 @@ std::optional<ApplicationOptions> ParseArguments(
"Number of layers to offload to GPU"); "Number of layers to offload to GPU");
}; };
// --mocked, --model, and --openai are mutually exclusive; validation is // --mocked, --model, and --openai are mutually exclusive, validated below
// enforced below rather than at registration to produce a clear // to produce a clear diagnostic message.
// diagnostic message.
auto add_generator_options = [&]() -> void { auto add_generator_options = [&]() -> void {
opt("mocked", prog_opts::bool_switch(), opt("mocked", prog_opts::bool_switch(),
"Use mocked generator for brewery/user data"); "Use mocked generator for brewery/user data");
@@ -154,9 +153,7 @@ std::optional<ApplicationOptions> ParseArguments(
return std::nullopt; return std::nullopt;
} }
// Prompt directory is only meaningful for live inference (Llama or // Prompt directory is required only for live inference (Llama or OpenAI).
// OpenAI) — the mock generator has no use for it and should not
// require it to be present.
if (!use_mocked && options.pipeline.prompt_dir.empty()) { if (!use_mocked && options.pipeline.prompt_dir.empty()) {
const std::string msg = const std::string msg =
"Invalid arguments: --prompt-dir is required when not using " "Invalid arguments: --prompt-dir is required when not using "

View File

@@ -13,12 +13,10 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
std::unique_ptr<DataGenerator> generator, std::unique_ptr<DataGenerator> generator,
std::unique_ptr<IExportService> exporter, std::unique_ptr<IExportService> exporter,
std::unique_ptr<ICuratedDataService> curated_data_service, std::unique_ptr<ICuratedDataService> curated_data_service,
std::unique_ptr<IPostalCodeService> postal_code_service,
const ApplicationOptions& application_options) const ApplicationOptions& application_options)
: logger_(std::move(logger)), : logger_(std::move(logger)),
context_service_(std::move(context_service)), context_service_(std::move(context_service)),
generator_(std::move(generator)), generator_(std::move(generator)),
exporter_(std::move(exporter)), exporter_(std::move(exporter)),
curated_data_service_(std::move(curated_data_service)), curated_data_service_(std::move(curated_data_service)),
postal_code_service_(std::move(postal_code_service)),
application_options_(application_options) {} application_options_(application_options) {}

View File

@@ -4,54 +4,16 @@
*/ */
#include <chrono> #include <chrono>
#include <cmath>
#include <format> #include <format>
#include <numbers>
#include <optional> #include <optional>
#include <random> #include <random>
#include "biergarten_pipeline_orchestrator.h" #include "biergarten_pipeline_orchestrator.h"
#include "biergarten_pipeline_orchestrator/coords.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
struct Coords { using biergarten_pipeline_orchestrator_internal::Coords;
double longitude; using biergarten_pipeline_orchestrator_internal::GetRandomCoordsWithinRange;
double latitude;
};
// @todo find a better home for this
namespace {
constexpr double kEarthRadiusKm = 6371.0;
double DegreesToRadians(double degrees) {
return degrees * std::numbers::pi / 180.0;
}
double RadiansToDegrees(double radians) {
return radians * 180.0 / std::numbers::pi;
}
} // namespace
static Coords GetRandomCoordsWithinRange(const Coords& centre,
const int distance_km,
std::mt19937& rng) {
std::uniform_real_distribution<double> angle_dist(0.0,
2.0 * std::numbers::pi);
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
const double angle_radians = angle_dist(rng);
// sqrt() keeps points uniformly distributed over the disc's area rather
// than clustering near the centre.
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
const double delta_latitude_degrees =
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
const double delta_longitude_degrees = RadiansToDegrees(
radius_km * std::sin(angle_radians) /
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
.latitude = centre.latitude + delta_latitude_degrees};
}
void BiergartenPipelineOrchestrator::GenerateBreweries( void BiergartenPipelineOrchestrator::GenerateBreweries(
std::span<const EnrichedCity> cities) { std::span<const EnrichedCity> cities) {
@@ -70,8 +32,6 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
try { try {
const BreweryResult brewery = const BreweryResult brewery =
generator_->GenerateBrewery(enriched_city); generator_->GenerateBrewery(enriched_city);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(enriched_city.location);
constexpr int kMaxDistanceFromCentreKm = 5; constexpr int kMaxDistanceFromCentreKm = 5;
const Coords city_centre{.longitude = enriched_city.location.longitude, const Coords city_centre{.longitude = enriched_city.location.longitude,
@@ -81,7 +41,6 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
return BreweryRecord{ return BreweryRecord{
.address = BreweryAddress{.city = enriched_city.location, .address = BreweryAddress{.city = enriched_city.location,
.postal_code = postal_code,
.longitude = brewery_coords.longitude, .longitude = brewery_coords.longitude,
.latitude = brewery_coords.latitude}, .latitude = brewery_coords.latitude},
.brewery = brewery}; .brewery = brewery};

View File

@@ -15,9 +15,13 @@
#include <unordered_set> #include <unordered_set>
#include "biergarten_pipeline_orchestrator.h" #include "biergarten_pipeline_orchestrator.h"
#include "biergarten_pipeline_orchestrator/coords.h"
#include "services/curated_data/curated_json_data_service.h" #include "services/curated_data/curated_json_data_service.h"
#include "services/logging/logger.h" #include "services/logging/logger.h"
using biergarten_pipeline_orchestrator_internal::Coords;
using biergarten_pipeline_orchestrator_internal::GetRandomCoordsWithinRange;
namespace { namespace {
std::string Sanitize(std::string_view value) { std::string Sanitize(std::string_view value) {
std::string out; std::string out;
@@ -135,12 +139,17 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
try { try {
const UserResult user = const UserResult user =
generator_->GenerateUser(city, persona, sampled_name); generator_->GenerateUser(city, persona, sampled_name);
const std::string postal_code =
postal_code_service_->GeneratePostalCode(city.location); constexpr int kMaxDistanceFromCentreKm = 5;
const Coords city_centre{.longitude = city.location.longitude,
.latitude = city.location.latitude};
const Coords user_coords = GetRandomCoordsWithinRange(
city_centre, kMaxDistanceFromCentreKm, rng);
return UserRecord{ return UserRecord{
.address = .address = UserAddress{.city = city.location,
UserAddress{.city = city.location, .postal_code = postal_code}, .longitude = user_coords.longitude,
.latitude = user_coords.latitude},
.user = user, .user = user,
.email = BuildEmail(sampled_name, used_email_local_parts), .email = BuildEmail(sampled_name, used_email_local_parts),
.date_of_birth = GenerateDateOfBirth(rng), .date_of_birth = GenerateDateOfBirth(rng),

View File

@@ -26,7 +26,6 @@ void BiergartenPipelineOrchestrator::LogResults() const {
{"country", city.country}, {"country", city.country},
{"state_province", city.state_province}, {"state_province", city.state_province},
{"iso3166_2", city.iso3166_2}, {"iso3166_2", city.iso3166_2},
{"postal_code", address.postal_code},
{"longitude", address.longitude}, {"longitude", address.longitude},
{"latitude", address.latitude}, {"latitude", address.latitude},
}}}); }}});

View File

@@ -58,9 +58,7 @@ std::string CondenseWhitespace(std::string_view text) {
return out; return out;
} }
// Guard against truncating in the first half of the string. // Keeps truncation past the string's midpoint, preserving the critical opening content.
// This preserves the critical opening content and avoids cutting critical
// context words early in the region description.
constexpr size_t kTruncationGuardDivisor = 2; constexpr size_t kTruncationGuardDivisor = 2;
bool ReadRequiredTrimmedStringField(const boost::json::object& obj, bool ReadRequiredTrimmedStringField(const boost::json::object& obj,

View File

@@ -36,10 +36,7 @@ std::string FormatLocalLanguageCodes(
constexpr int kBreweryMaxTokens = 2800; constexpr int kBreweryMaxTokens = 2800;
// Structured outputs already guarantee schema-valid JSON, so retries here // Structured outputs already guarantee schema-valid JSON, so retries here only cover transient failures or the model returning placeholder text.
// are only for transient failures or the model returning placeholder text
// -- not for malformed JSON shape, unlike LlamaGenerator's grammar-assisted
// retry loop.
constexpr int kMaxAttempts = 2; constexpr int kMaxAttempts = 2;
} // namespace } // namespace
@@ -54,9 +51,7 @@ BreweryResult OpenAIGenerator::GenerateBrewery(
FormatLocalLanguageCodes(location.local_languages); FormatLocalLanguageCodes(location.local_languages);
/** /**
* Load brewery system prompt via the injected prompt directory. The * Loads the backend-agnostic brewery system prompt via the injected prompt directory.
* same file LlamaGenerator uses -- the prompt content is backend
* agnostic.
*/ */
const std::string system_prompt = const std::string system_prompt =
prompt_directory_->Load("BREWERY_GENERATION"); prompt_directory_->Load("BREWERY_GENERATION");

View File

@@ -16,8 +16,7 @@
namespace { namespace {
constexpr int kUserMaxTokens = 1200; constexpr int kUserMaxTokens = 1200;
// Structured outputs already guarantee schema-valid JSON, so retries here // Structured outputs already guarantee schema-valid JSON, so retries here only cover transient failures or placeholder text.
// are only for transient failures or placeholder text -- not JSON shape.
constexpr int kMaxAttempts = 2; constexpr int kMaxAttempts = 2;
} // namespace } // namespace

View File

@@ -87,36 +87,16 @@ int main(const int argc, char** argv) {
di::bind<ApplicationOptions>().to(options), di::bind<ApplicationOptions>().to(options),
di::bind<std::string>().to(model_path), di::bind<std::string>().to(model_path),
di::bind<IExportService>().to<SqliteExportService>(), di::bind<IExportService>().to<SqliteExportService>(),
di::bind<IPostalCodeService>().to(
[options,
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
if (options.generator.mode == GeneratorMode::kMock) {
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message = "Postal code: mock (curated examples)"});
return std::make_unique<MockPostalCodeService>();
}
log_producer->Log(
{.level = LogLevel::Info,
.phase = PipelinePhase::Startup,
.message =
"Postal code: Xeger (regex-based generation)"});
return std::make_unique<XegerPostalCodeService>();
}),
di::bind<ICuratedDataService>().to( di::bind<ICuratedDataService>().to(
[options, [options,
&log_producer]() -> std::unique_ptr<ICuratedDataService> { &log_producer]() -> std::unique_ptr<ICuratedDataService> {
// if (options.generator.mode == GeneratorMode::kMock) { if (options.generator.mode == GeneratorMode::kMock) {
// log_producer->Log({.level = LogLevel::Info, log_producer->Log({.level = LogLevel::Info,
// .phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
// .message = "Curated data: mock"}); .message = "Curated data: mock"});
//
// return std::make_unique<MockCuratedDataService>(); return std::make_unique<MockCuratedDataService>();
// } }
log_producer->Log({.level = LogLevel::Info, log_producer->Log({.level = LogLevel::Info,
.phase = PipelinePhase::Startup, .phase = PipelinePhase::Startup,
@@ -214,9 +194,7 @@ int main(const int argc, char** argv) {
} }
#ifdef BIERGARTEN_MOCK_ONLY #ifdef BIERGARTEN_MOCK_ONLY
// Only reachable when mode == Llama: Mock and OpenAI are // Only reachable when mode == Llama, which requires llama.cpp and is excluded from a BIERGARTEN_MOCK_ONLY build.
// handled above and don't need llama.cpp, but LlamaGenerator
// itself is not linked into a BIERGARTEN_MOCK_ONLY build.
throw std::runtime_error( throw std::runtime_error(
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY " "LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
"build"); "build");

View File

@@ -77,12 +77,11 @@ static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
} }
namespace { namespace {
boost::json::value ParseJsonFile(const std::filesystem::path& filepath, boost::json::value ParseJsonFile(const std::filesystem::path& filepath) {
const char* what) {
std::ifstream input(filepath); std::ifstream input(filepath);
if (!input.is_open()) { if (!input.is_open()) {
throw std::runtime_error( throw std::runtime_error(
std::format("Failed to open {} file: {}", what, filepath.string())); std::format("Failed to open file: {}", filepath.string()));
} }
std::stringstream buffer; std::stringstream buffer;
@@ -92,7 +91,7 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
boost::json::value root = boost::json::parse(buffer.str(), error); boost::json::value root = boost::json::parse(buffer.str(), error);
if (error) { if (error) {
throw std::runtime_error( throw std::runtime_error(
std::format("Failed to parse {} JSON: {}", what, error.message())); std::format("Failed to parse JSON: {}", error.message()));
} }
return root; return root;
} }
@@ -126,34 +125,47 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
if (!cache_.locations.empty()) { if (!cache_.locations.empty()) {
return cache_.locations; return cache_.locations;
} }
boost::json::value root_val =
ParseJsonFile(filepaths_.locations_path);
const boost::json::value root = if (!root_val.is_object()) {
ParseJsonFile(filepaths_.locations_path, "locations");
if (!root.is_array()) {
throw std::runtime_error( throw std::runtime_error(
"Invalid locations JSON: root element must be an array"); "Invalid locations JSON: root element must be an object");
} }
LocationsList locations; boost::json::object root_obj = root_val.as_object();
const auto& items = root.as_array();
locations.reserve(items.size());
for (const auto& item : items) { if (!root_obj.contains("cities"))
{
throw std::runtime_error(
"Invalid locations JSON: root object must contain a 'cities' key");
}
if (!root_obj["cities"].is_array())
{
throw std::runtime_error(
"Invalid locations JSON: 'cities' key must be an array");
}
LocationsList locations;
const boost::json::array& cities = root_obj["cities"].as_array();
locations.reserve(cities.size());
for (const boost::json::value& item : cities) {
if (!item.is_object()) { if (!item.is_object()) {
throw std::runtime_error( throw std::runtime_error(
"Invalid locations JSON: each entry must be an object"); "Invalid locations JSON: each entry must be an object");
} }
const auto& object = item.as_object(); const boost::json::object& object = item.as_object();
locations.push_back(City{ locations.push_back(City{
.city = ReadRequiredString(object, "city"), .city = ReadRequiredString(object, "city"),
.state_province = ReadRequiredString(object, "state_province"), .state_province = ReadRequiredString(object, "state_province"),
.iso3166_2 = ReadRequiredString(object, "iso3166_2"), .iso3166_2 = ReadRequiredString(object, "iso3166_2"),
.country = ReadRequiredString(object, "country"), .country = ReadRequiredString(object, "country"),
.iso3166_1 = ReadRequiredString(object, "iso3166_1"), .iso3166_1 = ReadRequiredString(object, "iso3166_1"),
.latitude = ReadRequiredDouble(object, "latitude"),
.longitude = ReadRequiredDouble(object, "longitude"), .longitude = ReadRequiredDouble(object, "longitude"),
.latitude = ReadRequiredDouble(object, "latitude"),
.local_languages = ReadRequiredStringArray(object, "local_languages"), .local_languages = ReadRequiredStringArray(object, "local_languages"),
.postal_code = ReadPostalCodeSpec(object), .postal_code = ReadPostalCodeSpec(object),
}); });
@@ -168,7 +180,7 @@ const PersonasList& CuratedJsonDataService::LoadPersonas() {
} }
const boost::json::value root = const boost::json::value root =
ParseJsonFile(filepaths_.personas_path, "personas"); ParseJsonFile(filepaths_.personas_path);
if (!root.is_array()) { if (!root.is_array()) {
throw std::runtime_error( throw std::runtime_error(
@@ -176,10 +188,10 @@ const PersonasList& CuratedJsonDataService::LoadPersonas() {
} }
PersonasList personas; PersonasList personas;
const auto& items = root.as_array(); const boost::json::array& items = root.as_array();
personas.reserve(items.size()); personas.reserve(items.size());
for (const auto& item : items) { for (const boost::json::value& item : items) {
if (!item.is_object()) { if (!item.is_object()) {
throw std::runtime_error( throw std::runtime_error(
"Invalid personas JSON: each entry must be an object"); "Invalid personas JSON: each entry must be an object");
@@ -204,7 +216,7 @@ const ForenamesByCountryMap& CuratedJsonDataService::LoadForenamesByCountry() {
} }
const boost::json::value root = const boost::json::value root =
ParseJsonFile(filepaths_.forenames_path, "forenames-by-country"); ParseJsonFile(filepaths_.forenames_path);
if (!root.is_object()) { if (!root.is_object()) {
throw std::runtime_error( throw std::runtime_error(
@@ -252,7 +264,7 @@ const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() {
} }
const boost::json::value root = const boost::json::value root =
ParseJsonFile(filepaths_.surnames_path, "surnames-by-country"); ParseJsonFile(filepaths_.surnames_path);
if (!root.is_object()) { if (!root.is_object()) {
throw std::runtime_error( throw std::runtime_error(

View File

@@ -1,44 +0,0 @@
/**
* @file services/postal_code/xeger_postal_code_service.cc
* @brief XegerPostalCodeService::GeneratePostalCode() implementation.
*/
#include "services/postal_code/xeger_postal_code_service.h"
#include <format>
#include <random>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "services/xeger/xeger.h"
XegerPostalCodeService::XegerPostalCodeService()
: rng_(std::random_device{}()) {}
std::string XegerPostalCodeService::GeneratePostalCode(const City& city) {
const PostalCodeSpec& spec = city.postal_code;
std::vector<std::string_view> city_patterns;
for (const std::string& pattern : spec.city_regexes) {
if (!pattern.empty()) {
city_patterns.push_back(pattern);
}
}
std::string_view pattern;
if (!city_patterns.empty()) {
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
pattern = city_patterns[dist(rng_)];
} else if (!spec.country_format_regex.empty()) {
pattern = spec.country_format_regex;
} else {
throw std::runtime_error(std::format(
"XegerPostalCodeService: no postal-code pattern available for "
"city '{}'",
city.city));
}
return GenerateStringFromRegex(pattern, rng_);
}

View File

@@ -170,13 +170,6 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex, sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
.value = city_id, .value = city_id,
.action = "Failed to bind SQLite brewery address city id"}); .action = "Failed to bind SQLite brewery address city id"});
sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{
.index = sqlite_export_service_internal::
kBreweryAddressPostalCodeBindIndex,
.value = brewery.address.postal_code,
.action = "Failed to bind SQLite brewery address postal code"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_brewery_address_stmt_, insert_brewery_address_stmt_,
sqlite_export_service_internal::BoundParam<double>{ sqlite_export_service_internal::BoundParam<double>{

View File

@@ -87,11 +87,18 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
.action = "Failed to bind SQLite user address city id"}); .action = "Failed to bind SQLite user address city id"});
sqlite_export_service_internal::Bind( sqlite_export_service_internal::Bind(
insert_user_address_stmt_, insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<std::string_view>{ sqlite_export_service_internal::BoundParam<double>{
.index = .index =
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex, sqlite_export_service_internal::kUserAddressLongitudeBindIndex,
.value = user.address.postal_code, .value = user.address.longitude,
.action = "Failed to bind SQLite user address postal code"}); .action = "Failed to bind SQLite user address longitude"});
sqlite_export_service_internal::Bind(
insert_user_address_stmt_,
sqlite_export_service_internal::BoundParam<double>{
.index =
sqlite_export_service_internal::kUserAddressLatitudeBindIndex,
.value = user.address.latitude,
.action = "Failed to bind SQLite user address latitude"});
sqlite_export_service_internal::StepStatement( sqlite_export_service_internal::StepStatement(
db_handle_, insert_user_address_stmt_, db_handle_, insert_user_address_stmt_,

View File

@@ -8,20 +8,20 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="idunno.Password.Generator" Version="2.0.0" /> <PackageReference Include="idunno.Password.Generator" Version="2.0.0"/>
<PackageReference Include="Microsoft.Data.SqlClient" Version="7.0.2" /> <PackageReference Include="Microsoft.Data.SqlClient" Version="7.0.2"/>
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.11" /> <PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.11"/>
<PackageReference Include="Spectre.Console" Version="0.57.2" /> <PackageReference Include="Spectre.Console" Version="0.57.2"/>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\Domain\Domain.Entities\Domain.Entities.csproj" /> <ProjectReference Include="..\..\Domain\Domain.Entities\Domain.Entities.csproj"/>
<ProjectReference Include="..\..\Infrastructure\Infrastructure.Sql\Infrastructure.Sql.csproj" /> <ProjectReference Include="..\..\Infrastructure\Infrastructure.Sql\Infrastructure.Sql.csproj"/>
<ProjectReference Include="..\..\Features\Features.Breweries\Features.Breweries.csproj" /> <ProjectReference Include="..\..\Features\Features.Breweries\Features.Breweries.csproj"/>
<ProjectReference Include="..\..\Features\Features.Auth\Features.Auth.csproj" /> <ProjectReference Include="..\..\Features\Features.Auth\Features.Auth.csproj"/>
<ProjectReference Include="..\..\Features\Features.Locations\Features.Locations.csproj" /> <ProjectReference Include="..\..\Features\Features.Locations\Features.Locations.csproj"/>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None CopyToOutputDirectory="PreserveNewest" Include="SeedData\open-ai-test2.sqlite" /> <None CopyToOutputDirectory="PreserveNewest" Include="SeedData\biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite"/>
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@@ -12,6 +12,5 @@ public class ConnectionStrings
public static string GetSqlServerConnectionString(IConfiguration configuration) => public static string GetSqlServerConnectionString(IConfiguration configuration) =>
SqlConnectionStringHelper.BuildConnectionString(configuration); SqlConnectionStringHelper.BuildConnectionString(configuration);
public static string SqliteConnectionString => "Data Source=SeedData/open-ai-test2.sqlite"; public static string SqliteConnectionString => "Data Source=SeedData/biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite";
// "Data Source=SeedData/biergarten_seed_2026-07-20T05-46-02.993511Z.sqlite";
} }

View File

@@ -0,0 +1,124 @@
using System.Globalization;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
namespace Database.Seed.Geocoding;
public sealed record ReverseGeocodeResult(string AddressLine1, string PostalCode);
internal sealed record NominatimAddress
{
[JsonPropertyName("house_number")]
public string? HouseNumber { get; init; }
[JsonPropertyName("road")]
public string? Road { get; init; }
[JsonPropertyName("postcode")]
public string? Postcode { get; init; }
}
internal sealed record NominatimReverseGeocodeResponse
{
[JsonPropertyName("address")]
public NominatimAddress? Address { get; init; }
[JsonPropertyName("error")]
public string? Error { get; init; }
}
/// <summary>
/// Reverse geocodes coordinates into street addresses via the public Nominatim API
/// (https://nominatim.openstreetmap.org), following its usage policy of a single
/// identifying User-Agent and at most one request per second
/// (https://operations.osmfoundation.org/policies/nominatim/).
/// </summary>
public sealed class NominatimReverseGeocoder : IDisposable
{
private const string ReverseEndpoint = "https://nominatim.openstreetmap.org/reverse";
private static readonly TimeSpan MinRequestInterval = TimeSpan.FromSeconds(1);
private readonly HttpClient _httpClient;
private readonly SemaphoreSlim _rateLimitLock = new(1, 1);
private DateTime _lastRequestUtc = DateTime.MinValue;
public NominatimReverseGeocoder(string userAgent)
{
_httpClient = new HttpClient();
_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
}
/// <summary>
/// Reverse geocodes a coordinate pair, returning <c>null</c> if Nominatim has no address
/// for the point or the request fails so callers can fall back to a placeholder address.
/// </summary>
public async Task<ReverseGeocodeResult?> ReverseGeocodeAsync(
double longitude,
double latitude,
CancellationToken cancellationToken = default
)
{
await WaitForRateLimitAsync(cancellationToken);
string url =
$"{ReverseEndpoint}?format=jsonv2"
+ $"&lat={latitude.ToString(CultureInfo.InvariantCulture)}"
+ $"&lon={longitude.ToString(CultureInfo.InvariantCulture)}"
+ "&zoom=18&addressdetails=1";
try
{
NominatimReverseGeocodeResponse? response =
await _httpClient.GetFromJsonAsync<NominatimReverseGeocodeResponse>(
url,
cancellationToken
);
if (response?.Address is null || response.Error is not null)
return null;
string addressLine1 = string.Join(
' ',
new[] { response.Address.HouseNumber, response.Address.Road }.Where(
part => !string.IsNullOrWhiteSpace(part)
)
);
return string.IsNullOrWhiteSpace(addressLine1)
? null
: new ReverseGeocodeResult(addressLine1, response.Address.Postcode ?? string.Empty);
}
catch (HttpRequestException)
{
return null;
}
catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested)
{
// HttpClient's own timeout elapsed here.
return null;
}
}
private async Task WaitForRateLimitAsync(CancellationToken cancellationToken)
{
await _rateLimitLock.WaitAsync(cancellationToken);
try
{
TimeSpan elapsed = DateTime.UtcNow - _lastRequestUtc;
if (elapsed < MinRequestInterval)
await Task.Delay(MinRequestInterval - elapsed, cancellationToken);
_lastRequestUtc = DateTime.UtcNow;
}
finally
{
_rateLimitLock.Release();
}
}
public void Dispose()
{
_httpClient.Dispose();
_rateLimitLock.Dispose();
}
}

View File

@@ -4,8 +4,6 @@ public sealed record BreweryAddress
{ {
public required City City { get; init; } public required City City { get; init; }
public required string PostalCode { get; init; }
public required double Longitude { get; init; } public required double Longitude { get; init; }
public required double Latitude { get; init; } public required double Latitude { get; init; }

View File

@@ -4,5 +4,7 @@ public sealed record UserAddress
{ {
public required City City { get; init; } public required City City { get; init; }
public required string PostalCode { get; init; } public required double Longitude { get; init; }
public required double Latitude { get; init; }
} }

View File

@@ -1,4 +1,5 @@
using Database.Seed.DatabaseHelpers; using Database.Seed.DatabaseHelpers;
using Database.Seed.Geocoding;
using Database.Seed.PipelineData; using Database.Seed.PipelineData;
using Database.Seed.Sqlite; using Database.Seed.Sqlite;
using Domain.Entities; using Domain.Entities;
@@ -16,6 +17,9 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Spectre.Console; using Spectre.Console;
const string NominatimUserAgent =
"TheBiergartenApp-Seeder/1.0 (+https://github.com/aaronpo97/the-biergarten-app)";
return await RunAsync(); return await RunAsync();
static async Task<int> RunAsync() static async Task<int> RunAsync()
@@ -100,23 +104,34 @@ static async Task<int> RunAsync()
} }
); );
await AnsiConsole List<string> geocodingWarnings = [];
.Status()
.Spinner(Spinner.Known.Dots) using (NominatimReverseGeocoder geocoder = new(NominatimUserAgent))
.SpinnerStyle(Style.Parse("green")) {
.StartAsync( await AnsiConsole
"Loading brewery data into target database...", .Status()
async ctx => .Spinner(Spinner.Known.Dots)
{ .SpinnerStyle(Style.Parse("green"))
await LoadBreweriesIntoDatabaseAsync( .StartAsync(
breweryRepository, "Loading brewery data into target database...",
locationRepository, async ctx =>
seedData.Breweries, {
postedByIds await LoadBreweriesIntoDatabaseAsync(
); breweryRepository,
ctx.Status("Brewery data loaded into target database."); locationRepository,
} geocoder,
); seedData.Breweries,
postedByIds,
geocodingWarnings,
ctx
);
ctx.Status("Brewery data loaded into target database.");
}
);
}
foreach (string warning in geocodingWarnings)
AnsiConsole.MarkupLine($"[yellow]![/] {warning.EscapeMarkup()}");
return 0; return 0;
} }
@@ -165,20 +180,18 @@ static async Task<IReadOnlyList<Guid>> LoadUsersIntoDatabaseAsync(
return userAccountIds; return userAccountIds;
} }
/// <summary>
/// Persists each <paramref name="breweries" /> record, resolving (and creating, if needed) its
/// Country/StateProvince/City chain via <paramref name="locationRepository" />. The pipeline
/// data has no street-level address or a specific user tied to each brewery, so a placeholder
/// address is used and <c>PostedById</c> is assigned round-robin from <paramref name="posterUserIds" />
/// to keep seeding deterministic across runs.
/// </summary>
static async Task LoadBreweriesIntoDatabaseAsync( static async Task LoadBreweriesIntoDatabaseAsync(
IBreweryRepository breweryRepository, IBreweryRepository breweryRepository,
ILocationRepository locationRepository, ILocationRepository locationRepository,
NominatimReverseGeocoder geocoder,
IReadOnlyList<BreweryRecord> breweries, IReadOnlyList<BreweryRecord> breweries,
IReadOnlyList<Guid> posterUserIds IReadOnlyList<Guid> posterUserIds,
List<string> geocodingWarnings,
StatusContext ctx
) )
{ {
const string placeholderAddressLine1 = "Address unavailable";
if (posterUserIds.Count == 0) if (posterUserIds.Count == 0)
throw new InvalidOperationException( throw new InvalidOperationException(
"Cannot load breweries without any registered users."); "Cannot load breweries without any registered users.");
@@ -187,6 +200,10 @@ static async Task LoadBreweriesIntoDatabaseAsync(
{ {
BreweryRecord breweryRecord = breweries[i]; BreweryRecord breweryRecord = breweries[i];
ctx.Status(
$"Geocoding and loading brewery {i + 1}/{breweries.Count} into target database..."
);
Guid cityId = await locationRepository.GetOrCreateCityIdAsync( Guid cityId = await locationRepository.GetOrCreateCityIdAsync(
new CityLocation( new CityLocation(
breweryRecord.Address.City.CityName, breweryRecord.Address.City.CityName,
@@ -197,6 +214,18 @@ static async Task LoadBreweriesIntoDatabaseAsync(
) )
); );
ReverseGeocodeResult? geocoded = await geocoder.ReverseGeocodeAsync(
breweryRecord.Address.Longitude,
breweryRecord.Address.Latitude
);
if (geocoded is null)
geocodingWarnings.Add(
$"Could not reverse geocode '{breweryRecord.Brewery.NameEn}' "
+ $"({breweryRecord.Address.Latitude}, {breweryRecord.Address.Longitude}); "
+ "using a placeholder address."
);
await breweryRepository.CreateAsync( await breweryRepository.CreateAsync(
new BreweryPost new BreweryPost
{ {
@@ -204,12 +233,13 @@ static async Task LoadBreweriesIntoDatabaseAsync(
BreweryName = breweryRecord.Brewery.NameEn, BreweryName = breweryRecord.Brewery.NameEn,
Description = breweryRecord.Brewery.DescriptionEn, Description = breweryRecord.Brewery.DescriptionEn,
PostedById = posterUserIds[i % posterUserIds.Count], PostedById = posterUserIds[i % posterUserIds.Count],
// TODO: Populate Coordinates from breweryRecord.Address.Latitude/Longitude once it goes through NetTopologySuite (NTS).
Location = new BreweryPostLocation Location = new BreweryPostLocation
{ {
BreweryPostLocationId = Guid.NewGuid(), BreweryPostLocationId = Guid.NewGuid(),
CityId = cityId, CityId = cityId,
AddressLine1 = "Address unavailable", AddressLine1 = geocoded?.AddressLine1 ?? placeholderAddressLine1,
PostalCode = breweryRecord.Address.PostalCode, PostalCode = geocoded?.PostalCode ?? string.Empty,
}, },
} }
); );
@@ -224,7 +254,8 @@ static Table BuildBreweryTable(IReadOnlyList<BreweryRecord> breweries)
.AddColumn("City") .AddColumn("City")
.AddColumn("State/Province") .AddColumn("State/Province")
.AddColumn("Country") .AddColumn("Country")
.AddColumn("Postal Code"); .AddColumn("Longitude")
.AddColumn("Latitude");
foreach (BreweryRecord brewery in breweries) foreach (BreweryRecord brewery in breweries)
{ {
@@ -234,7 +265,8 @@ static Table BuildBreweryTable(IReadOnlyList<BreweryRecord> breweries)
brewery.Address.City.CityName, brewery.Address.City.CityName,
brewery.Address.City.StateProvince, brewery.Address.City.StateProvince,
brewery.Address.City.Country, brewery.Address.City.Country,
brewery.Address.PostalCode brewery.Address.Longitude.ToString("F6"),
brewery.Address.Latitude.ToString("F6")
); );
} }

View File

@@ -54,7 +54,7 @@ public sealed class PipelineSeedDataReader
{ {
const string sql = """ const string sql = """
SELECT b.name_en, b.description_en, b.name_local, b.description_local, SELECT b.name_en, b.description_en, b.name_local, b.description_local,
ba.city_id, ba.postal_code, ba.longitude, ba.latitude ba.city_id, ba.longitude, ba.latitude
FROM breweries b FROM breweries b
JOIN brewery_addresses ba ON ba.brewery_id = b.id; JOIN brewery_addresses ba ON ba.brewery_id = b.id;
"""; """;
@@ -67,9 +67,8 @@ public sealed class PipelineSeedDataReader
const int nameLocalIndex = 2; const int nameLocalIndex = 2;
const int descriptionLocalIndex = 3; const int descriptionLocalIndex = 3;
const int cityIdIndex = 4; const int cityIdIndex = 4;
const int postalCodeIndex = 5; const int longitudeIndex = 5;
const int longitudeIndex = 6; const int latitudeIndex = 6;
const int latitudeIndex = 7;
List<BreweryRecord> records = []; List<BreweryRecord> records = [];
while (await reader.ReadAsync(cancellationToken)) while (await reader.ReadAsync(cancellationToken))
@@ -86,7 +85,6 @@ public sealed class PipelineSeedDataReader
Address = new BreweryAddress Address = new BreweryAddress
{ {
City = cities[reader.GetInt64(cityIdIndex)], City = cities[reader.GetInt64(cityIdIndex)],
PostalCode = reader.GetString(postalCodeIndex),
Longitude = reader.GetDouble(longitudeIndex), Longitude = reader.GetDouble(longitudeIndex),
Latitude = reader.GetDouble(latitudeIndex), Latitude = reader.GetDouble(latitudeIndex),
}, },
@@ -105,7 +103,7 @@ public sealed class PipelineSeedDataReader
const string sql = """ const string sql = """
SELECT u.first_name, u.last_name, u.gender, u.username, u.bio, SELECT u.first_name, u.last_name, u.gender, u.username, u.bio,
u.activity_weight, u.email, u.date_of_birth, u.activity_weight, u.email, u.date_of_birth,
ua.city_id, ua.postal_code ua.city_id, ua.longitude, ua.latitude
FROM users u FROM users u
JOIN user_addresses ua ON ua.user_id = u.id; JOIN user_addresses ua ON ua.user_id = u.id;
"""; """;
@@ -122,7 +120,8 @@ public sealed class PipelineSeedDataReader
const int emailIndex = 6; const int emailIndex = 6;
const int dateOfBirthIndex = 7; const int dateOfBirthIndex = 7;
const int cityIdIndex = 8; const int cityIdIndex = 8;
const int postalCodeIndex = 9; const int longitudeIndex = 9;
const int latitudeIndex = 10;
List<UserRecord> records = []; List<UserRecord> records = [];
while (await reader.ReadAsync(cancellationToken)) while (await reader.ReadAsync(cancellationToken))
@@ -143,7 +142,8 @@ public sealed class PipelineSeedDataReader
Address = new UserAddress Address = new UserAddress
{ {
City = cities[reader.GetInt64(cityIdIndex)], City = cities[reader.GetInt64(cityIdIndex)],
PostalCode = reader.GetString(postalCodeIndex), Longitude = reader.GetDouble(longitudeIndex),
Latitude = reader.GetDouble(latitudeIndex),
}, },
} }
); );

View File

@@ -283,7 +283,7 @@ public class BreweryRepository(ISqlConnectionFactory connectionFactory)
} }
catch catch
{ {
await transaction.RollbackAsync(); await RollbackQuietlyAsync(transaction);
throw; throw;
} }
@@ -418,11 +418,28 @@ public class BreweryRepository(ISqlConnectionFactory connectionFactory)
} }
catch catch
{ {
await transaction.RollbackAsync(); await RollbackQuietlyAsync(transaction);
throw; throw;
} }
} }
/// <summary>
/// Rolls back <paramref name="transaction" />, swallowing any exception the rollback itself
/// raises (for example when the provider has already completed the transaction after a
/// connection failure) so the exception that triggered the rollback is what propagates.
/// </summary>
private static async Task RollbackQuietlyAsync(DbTransaction transaction)
{
try
{
await transaction.RollbackAsync();
}
catch
{
// Ignore: the original exception (rethrown by the caller) is what matters here.
}
}
/// <summary> /// <summary>
/// Composes a joined row's <see cref="BreweryPost" />, <see cref="BreweryPostLocation" />, /// Composes a joined row's <see cref="BreweryPost" />, <see cref="BreweryPostLocation" />,
/// <see cref="City" />, <see cref="StateProvince" /> and <see cref="Country" /> fragments into a /// <see cref="City" />, <see cref="StateProvince" /> and <see cref="Country" /> fragments into a