mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-14 17:54:42 +00:00
Add reverse geocoding for brewery street addresses (#245)
* Add geocoding to seed application, remove postal codes from generator Update comments
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,5 +1,5 @@
|
|||||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||||
|
scratchpad.txt
|
||||||
# dependencies
|
# dependencies
|
||||||
/node_modules
|
/node_modules
|
||||||
/.pnp
|
/.pnp
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ low-resource language failures.
|
|||||||
- [Known issues](#known-issues)
|
- [Known issues](#known-issues)
|
||||||
- [Hallucinated brewing techniques](#hallucinated-brewing-techniques)
|
- [Hallucinated brewing techniques](#hallucinated-brewing-techniques)
|
||||||
- [Low-resource language hallucination](#low-resource-language-hallucination)
|
- [Low-resource language hallucination](#low-resource-language-hallucination)
|
||||||
- [Synthetic postal codes](#synthetic-postal-codes)
|
- [Synthetic coordinates](#synthetic-coordinates)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -284,20 +284,17 @@ Output sample: [./french-cities.example](french-cities.example)
|
|||||||
SQLite schema so downstream applications can filter or flag potentially
|
SQLite schema so downstream applications can filter or flag potentially
|
||||||
hallucinated text by language tier.
|
hallucinated text by language tier.
|
||||||
|
|
||||||
### Synthetic postal codes
|
### Synthetic coordinates
|
||||||
|
|
||||||
When the pipeline is run without `--mocked`, `XegerPostalCodeService` generates
|
Each brewery and user address carries a longitude/latitude pair, sampled
|
||||||
a fresh postal code per brewery/user address by reverse-matching one of the
|
uniformly at random within a 5 km disc centred on the city's curated
|
||||||
city's curated postal-code regexes from `locations.json` (falling back to the
|
`latitude`/`longitude` from `locations.json`.
|
||||||
country format regex), rather than returning one of `MockPostalCodeService`'s
|
|
||||||
curated real-world examples.
|
|
||||||
|
|
||||||
The generated code is **format-conformant by construction** — it always
|
The resulting point is **plausible by construction** — it always falls within
|
||||||
satisfies the regex it was generated from — but it is not looked up from any
|
the city's vicinity — but it is not looked up from any real address registry.
|
||||||
real postal registry. It does not correspond to a real address, business, or
|
It does not correspond to a real address, business, or resident, and should
|
||||||
resident, and should not be treated as accurate geocoding or delivery data. This
|
not be treated as accurate geocoding data. This is consistent with the rest of
|
||||||
is consistent with the rest of the dataset: fixture data for a proof-of-concept,
|
the dataset: fixture data for a proof-of-concept, not a source of truth.
|
||||||
not a source of truth.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,9 @@ else()
|
|||||||
if(CUDAToolkit_FOUND)
|
if(CUDAToolkit_FOUND)
|
||||||
message(STATUS "[biergarten] NVIDIA GPU detected — enabling CUDA acceleration.")
|
message(STATUS "[biergarten] NVIDIA GPU detected — enabling CUDA acceleration.")
|
||||||
set(GGML_CUDA ON CACHE BOOL "Enable CUDA for NVIDIA GPUs" FORCE)
|
set(GGML_CUDA ON CACHE BOOL "Enable CUDA for NVIDIA GPUs" FORCE)
|
||||||
set(CMAKE_CUDA_ARCHITECTURES native)
|
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
|
||||||
|
set(CMAKE_CUDA_ARCHITECTURES native)
|
||||||
|
endif()
|
||||||
elseif(hip_FOUND OR DEFINED ENV{ROCM_PATH} OR EXISTS "/opt/rocm")
|
elseif(hip_FOUND OR DEFINED ENV{ROCM_PATH} OR EXISTS "/opt/rocm")
|
||||||
message(STATUS "[biergarten] AMD GPU detected — enabling HIP/ROCm acceleration.")
|
message(STATUS "[biergarten] AMD GPU detected — enabling HIP/ROCm acceleration.")
|
||||||
set(GGML_HIPBLAS ON CACHE BOOL "Enable HIP for AMD GPUs" FORCE)
|
set(GGML_HIPBLAS ON CACHE BOOL "Enable HIP for AMD GPUs" FORCE)
|
||||||
@@ -149,7 +151,6 @@ add_executable(${PROJECT_NAME}
|
|||||||
target_sources(${PROJECT_NAME} PRIVATE
|
target_sources(${PROJECT_NAME} PRIVATE
|
||||||
src/main.cc
|
src/main.cc
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- json_handling ---
|
# --- json_handling ---
|
||||||
target_sources(${PROJECT_NAME} PRIVATE
|
target_sources(${PROJECT_NAME} PRIVATE
|
||||||
src/services/curated_data/curated_json_data_service.cc
|
src/services/curated_data/curated_json_data_service.cc
|
||||||
@@ -224,11 +225,6 @@ target_sources(${PROJECT_NAME} PRIVATE
|
|||||||
src/services/enrichment/wikipedia/get_summary.cc
|
src/services/enrichment/wikipedia/get_summary.cc
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- services: postal_code ---
|
|
||||||
target_sources(${PROJECT_NAME} PRIVATE
|
|
||||||
src/services/postal_code/xeger_postal_code_service.cc
|
|
||||||
)
|
|
||||||
|
|
||||||
# --- services: xeger ---
|
# --- services: xeger ---
|
||||||
target_sources(${PROJECT_NAME} PRIVATE
|
target_sources(${PROJECT_NAME} PRIVATE
|
||||||
src/services/xeger/xeger.cc
|
src/services/xeger/xeger.cc
|
||||||
|
|||||||
@@ -61,10 +61,7 @@
|
|||||||
#include "services/enrichment/mock_enrichment.h"
|
#include "services/enrichment/mock_enrichment.h"
|
||||||
#include "services/enrichment/wikipedia_service.h"
|
#include "services/enrichment/wikipedia_service.h"
|
||||||
|
|
||||||
// --- services: postal_code ---
|
// --- services: xeger ---
|
||||||
#include "services/postal_code/mock_postal_code_service.h"
|
|
||||||
#include "services/postal_code/postal_code_service.h"
|
|
||||||
#include "services/postal_code/xeger_postal_code_service.h"
|
|
||||||
#include "services/xeger/xeger.h"
|
#include "services/xeger/xeger.h"
|
||||||
|
|
||||||
// --- services: logging ---
|
// --- services: logging ---
|
||||||
|
|||||||
@@ -16,7 +16,6 @@
|
|||||||
#include "services/database/export_service.h"
|
#include "services/database/export_service.h"
|
||||||
#include "services/enrichment/enrichment_service.h"
|
#include "services/enrichment/enrichment_service.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
#include "services/postal_code/postal_code_service.h"
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Main orchestrator for the Biergarten data generation pipeline.
|
* @brief Main orchestrator for the Biergarten data generation pipeline.
|
||||||
@@ -35,8 +34,6 @@ class BiergartenPipelineOrchestrator {
|
|||||||
* @param exporter Database backend for persisting generated records.
|
* @param exporter Database backend for persisting generated records.
|
||||||
* @param curated_data_service Loads curated location, persona, and name
|
* @param curated_data_service Loads curated location, persona, and name
|
||||||
* data used to seed generation.
|
* data used to seed generation.
|
||||||
* @param postal_code_service Resolves a postal code for a city from its
|
|
||||||
* curated postal-code data.
|
|
||||||
* @param application_options CLI configuration and paths.
|
* @param application_options CLI configuration and paths.
|
||||||
*/
|
*/
|
||||||
BiergartenPipelineOrchestrator(
|
BiergartenPipelineOrchestrator(
|
||||||
@@ -45,7 +42,6 @@ class BiergartenPipelineOrchestrator {
|
|||||||
std::unique_ptr<DataGenerator> generator,
|
std::unique_ptr<DataGenerator> generator,
|
||||||
std::unique_ptr<IExportService> exporter,
|
std::unique_ptr<IExportService> exporter,
|
||||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||||
std::unique_ptr<IPostalCodeService> postal_code_service,
|
|
||||||
const ApplicationOptions& application_options);
|
const ApplicationOptions& application_options);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -82,7 +78,6 @@ class BiergartenPipelineOrchestrator {
|
|||||||
*/
|
*/
|
||||||
std::unique_ptr<IExportService> exporter_;
|
std::unique_ptr<IExportService> exporter_;
|
||||||
std::unique_ptr<ICuratedDataService> curated_data_service_;
|
std::unique_ptr<ICuratedDataService> curated_data_service_;
|
||||||
std::unique_ptr<IPostalCodeService> postal_code_service_;
|
|
||||||
|
|
||||||
ApplicationOptions application_options_;
|
ApplicationOptions application_options_;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
#ifndef BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_
|
||||||
|
#define BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @file biergarten_pipeline_orchestrator/coords.h
|
||||||
|
* @brief Shared coordinate jitter helper used to place generated
|
||||||
|
* brewery/user addresses near a city's centre.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <cmath>
|
||||||
|
#include <numbers>
|
||||||
|
#include <random>
|
||||||
|
|
||||||
|
namespace biergarten_pipeline_orchestrator_internal {
|
||||||
|
|
||||||
|
struct Coords {
|
||||||
|
double longitude;
|
||||||
|
double latitude;
|
||||||
|
};
|
||||||
|
|
||||||
|
inline constexpr double kEarthRadiusKm = 6371.0;
|
||||||
|
|
||||||
|
inline double DegreesToRadians(double degrees) {
|
||||||
|
return degrees * std::numbers::pi / 180.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline double RadiansToDegrees(double radians) {
|
||||||
|
return radians * 180.0 / std::numbers::pi;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @brief Samples a point uniformly at random within @p distance_km of
|
||||||
|
* @p centre.
|
||||||
|
*/
|
||||||
|
inline Coords GetRandomCoordsWithinRange(const Coords& centre,
|
||||||
|
const int distance_km,
|
||||||
|
std::mt19937& rng) {
|
||||||
|
std::uniform_real_distribution<double> angle_dist(0.0,
|
||||||
|
2.0 * std::numbers::pi);
|
||||||
|
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
|
||||||
|
|
||||||
|
const double angle_radians = angle_dist(rng);
|
||||||
|
// sqrt() keeps points uniformly distributed over the disc's area.
|
||||||
|
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
|
||||||
|
|
||||||
|
const double delta_latitude_degrees =
|
||||||
|
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
|
||||||
|
const double delta_longitude_degrees = RadiansToDegrees(
|
||||||
|
radius_km * std::sin(angle_radians) /
|
||||||
|
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
|
||||||
|
|
||||||
|
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
|
||||||
|
.latitude = centre.latitude + delta_latitude_degrees};
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace biergarten_pipeline_orchestrator_internal
|
||||||
|
|
||||||
|
#endif // BIERGARTEN_PIPELINE_INCLUDES_BIERGARTEN_PIPELINE_ORCHESTRATOR_COORDS_H_
|
||||||
@@ -3,9 +3,9 @@
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @file data_generation/generation_json_validation.h
|
* @file data_generation/generation_json_validation.h
|
||||||
* @brief Generator-agnostic helpers for preparing prompt context and
|
* @brief Generator-agnostic helpers, shared across all DataGenerator
|
||||||
* validating structured JSON output. Shared across all DataGenerator
|
* implementations (Llama, OpenAI, ...), for preparing prompt context and
|
||||||
* implementations (Llama, OpenAI, ...).
|
* validating structured JSON output.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
@@ -36,11 +36,9 @@ std::optional<std::string> ValidateBreweryJson(const std::string& raw,
|
|||||||
BreweryResult& brewery_out);
|
BreweryResult& brewery_out);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Validates and parses user JSON output.
|
* @brief Validates and parses user JSON output, populating only
|
||||||
*
|
* `username`, `bio`, and `activity_weight` (`first_name` and `last_name`
|
||||||
* Only populates `username`, `bio`, and `activity_weight` -- `first_name`
|
* come from the sampled Name).
|
||||||
* and `last_name` are not LLM-authored and are set separately from the
|
|
||||||
* sampled Name.
|
|
||||||
*
|
*
|
||||||
* @param raw Raw model output.
|
* @param raw Raw model output.
|
||||||
* @param user_out Parsed user payload.
|
* @param user_out Parsed user payload.
|
||||||
|
|||||||
@@ -18,13 +18,9 @@
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Data generator implementation backed by the OpenAI Chat
|
* @brief Data generator implementation backed by the OpenAI Chat
|
||||||
* Completions API.
|
* Completions API, sending system/user messages directly and using
|
||||||
*
|
* Structured Outputs (response_format: json_schema, strict: true) to
|
||||||
* Unlike LlamaGenerator, this generator has no local model state and no
|
* guarantee schema-valid JSON.
|
||||||
* chat-template formatting step -- Chat Completions takes system/user
|
|
||||||
* messages directly, and Structured Outputs (response_format: json_schema,
|
|
||||||
* strict: true) guarantee schema-valid JSON, replacing llama.cpp's GBNF
|
|
||||||
* grammar constraint.
|
|
||||||
*/
|
*/
|
||||||
class OpenAIGenerator final : public DataGenerator {
|
class OpenAIGenerator final : public DataGenerator {
|
||||||
public:
|
public:
|
||||||
|
|||||||
@@ -4,14 +4,9 @@
|
|||||||
/**
|
/**
|
||||||
* @file data_generation/openai_json_schemas.h
|
* @file data_generation/openai_json_schemas.h
|
||||||
* @brief JSON Schemas constraining structured output from OpenAIGenerator
|
* @brief JSON Schemas constraining structured output from OpenAIGenerator
|
||||||
* Chat Completions requests, via response_format.json_schema (Structured
|
* Chat Completions requests via response_format.json_schema (Structured
|
||||||
* Outputs, strict mode). This is OpenAI's equivalent of the GBNF grammars
|
* Outputs, strict mode), each setting "additionalProperties": false as
|
||||||
* in json_grammars.h -- the API guarantees the response message content is
|
* strict mode requires.
|
||||||
* a single JSON string that validates against the schema, so there's no
|
|
||||||
* grammar-level "thought-block" prefix to account for.
|
|
||||||
*
|
|
||||||
* OpenAI's strict mode requires "additionalProperties": false on every
|
|
||||||
* object in the schema, which these already have.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
|
|||||||
@@ -88,23 +88,23 @@ struct EnrichedCity {
|
|||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief A physical address: the city plus a concrete postal code selected
|
* @brief A physical address: the city plus a coordinate pair sampled near
|
||||||
* from that city's curated postal-code examples.
|
* the city's centre.
|
||||||
*/
|
*/
|
||||||
struct BreweryAddress {
|
struct BreweryAddress {
|
||||||
City city;
|
City city;
|
||||||
std::string postal_code{};
|
|
||||||
double longitude{};
|
double longitude{};
|
||||||
double latitude{};
|
double latitude{};
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief A physical address for a generated user: the city plus a concrete
|
* @brief A physical address for a generated user: the city plus a
|
||||||
* postal code selected from that city's curated postal-code examples.
|
* coordinate pair sampled near the city's centre.
|
||||||
*/
|
*/
|
||||||
struct UserAddress {
|
struct UserAddress {
|
||||||
City city;
|
City city;
|
||||||
std::string postal_code{};
|
double longitude{};
|
||||||
|
double latitude{};
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -54,7 +54,6 @@ CREATE TABLE IF NOT EXISTS brewery_addresses (
|
|||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
brewery_id INTEGER NOT NULL,
|
brewery_id INTEGER NOT NULL,
|
||||||
city_id INTEGER NOT NULL,
|
city_id INTEGER NOT NULL,
|
||||||
postal_code TEXT NOT NULL,
|
|
||||||
longitude REAL NOT NULL,
|
longitude REAL NOT NULL,
|
||||||
latitude REAL NOT NULL,
|
latitude REAL NOT NULL,
|
||||||
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
|
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
|
||||||
@@ -90,7 +89,8 @@ CREATE TABLE IF NOT EXISTS user_addresses (
|
|||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
user_id INTEGER NOT NULL,
|
user_id INTEGER NOT NULL,
|
||||||
city_id INTEGER NOT NULL,
|
city_id INTEGER NOT NULL,
|
||||||
postal_code TEXT NOT NULL,
|
longitude REAL NOT NULL,
|
||||||
|
latitude REAL NOT NULL,
|
||||||
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
|
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
|
||||||
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
|
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
@@ -130,10 +130,9 @@ inline constexpr std::string_view kInsertBreweryAddressSql = R"sql(
|
|||||||
INSERT INTO brewery_addresses (
|
INSERT INTO brewery_addresses (
|
||||||
brewery_id,
|
brewery_id,
|
||||||
city_id,
|
city_id,
|
||||||
postal_code,
|
|
||||||
longitude,
|
longitude,
|
||||||
latitude
|
latitude
|
||||||
) VALUES (?, ?, ?, ?, ?);
|
) VALUES (?, ?, ?, ?);
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
inline constexpr std::string_view kInsertUserSql = R"sql(
|
inline constexpr std::string_view kInsertUserSql = R"sql(
|
||||||
@@ -153,8 +152,9 @@ inline constexpr std::string_view kInsertUserAddressSql = R"sql(
|
|||||||
INSERT INTO user_addresses (
|
INSERT INTO user_addresses (
|
||||||
user_id,
|
user_id,
|
||||||
city_id,
|
city_id,
|
||||||
postal_code
|
longitude,
|
||||||
) VALUES (?, ?, ?);
|
latitude
|
||||||
|
) VALUES (?, ?, ?, ?);
|
||||||
)sql";
|
)sql";
|
||||||
|
|
||||||
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
// sqlite3_bind_*() parameter indices are 1-based, matching the "?"
|
||||||
@@ -182,7 +182,6 @@ enum BreweryBindIndex {
|
|||||||
enum BreweryAddressBindIndex {
|
enum BreweryAddressBindIndex {
|
||||||
kBreweryAddressBreweryIdBindIndex = 1,
|
kBreweryAddressBreweryIdBindIndex = 1,
|
||||||
kBreweryAddressCityIdBindIndex,
|
kBreweryAddressCityIdBindIndex,
|
||||||
kBreweryAddressPostalCodeBindIndex,
|
|
||||||
kBreweryAddressLongitudeBindIndex,
|
kBreweryAddressLongitudeBindIndex,
|
||||||
kBreweryAddressLatitudeBindIndex,
|
kBreweryAddressLatitudeBindIndex,
|
||||||
};
|
};
|
||||||
@@ -201,7 +200,8 @@ enum UserBindIndex {
|
|||||||
enum UserAddressBindIndex {
|
enum UserAddressBindIndex {
|
||||||
kUserAddressUserIdBindIndex = 1,
|
kUserAddressUserIdBindIndex = 1,
|
||||||
kUserAddressCityIdBindIndex,
|
kUserAddressCityIdBindIndex,
|
||||||
kUserAddressPostalCodeBindIndex,
|
kUserAddressLongitudeBindIndex,
|
||||||
|
kUserAddressLatitudeBindIndex,
|
||||||
};
|
};
|
||||||
|
|
||||||
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
SqliteStatementHandle PrepareStatement(const SqliteDatabaseHandle& db_handle,
|
||||||
|
|||||||
@@ -1,80 +0,0 @@
|
|||||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
|
|
||||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @file services/postal_code/mock_postal_code_service.h
|
|
||||||
* @brief Data-driven postal code service that selects the first curated
|
|
||||||
* example for a city and validates it against the city's postal-code regex.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include <format>
|
|
||||||
#include <regex>
|
|
||||||
#include <stdexcept>
|
|
||||||
#include <string>
|
|
||||||
|
|
||||||
#include "services/postal_code/postal_code_service.h"
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Postal code service backed entirely by the curated locations
|
|
||||||
* dataset: it returns the city's first example postal code after checking it
|
|
||||||
* against the curated postal-code regex.
|
|
||||||
*
|
|
||||||
* A precise city regex is preferred when present; otherwise the permissive
|
|
||||||
* country format regex is used as the fallback. A city with no examples, or an
|
|
||||||
* example that fails validation, throws so the caller can skip the city.
|
|
||||||
*/
|
|
||||||
class MockPostalCodeService final : public IPostalCodeService {
|
|
||||||
public:
|
|
||||||
std::string GeneratePostalCode(const City& city) override {
|
|
||||||
const PostalCodeSpec& spec = city.postal_code;
|
|
||||||
|
|
||||||
if (spec.examples.empty()) {
|
|
||||||
throw std::runtime_error(std::format(
|
|
||||||
"No postal-code examples available for city '{}'", city.city));
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string& candidate = spec.examples.front();
|
|
||||||
|
|
||||||
if (!MatchesAnyPattern(candidate, spec)) {
|
|
||||||
throw std::runtime_error(
|
|
||||||
std::format("Postal code '{}' for city '{}' does not match the "
|
|
||||||
"curated postal-code regex",
|
|
||||||
candidate, city.city));
|
|
||||||
}
|
|
||||||
|
|
||||||
return candidate;
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
/**
|
|
||||||
* @brief Returns true if @p candidate matches any city regex, or the
|
|
||||||
* country format regex when no city regex matches. When the spec carries no
|
|
||||||
* usable regex at all, validation is skipped (returns true).
|
|
||||||
*/
|
|
||||||
static bool MatchesAnyPattern(const std::string& candidate,
|
|
||||||
const PostalCodeSpec& spec) {
|
|
||||||
bool has_pattern = false;
|
|
||||||
|
|
||||||
for (const std::string& pattern : spec.city_regexes) {
|
|
||||||
if (pattern.empty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
has_pattern = true;
|
|
||||||
if (std::regex_match(candidate, std::regex(pattern))) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!spec.country_format_regex.empty()) {
|
|
||||||
has_pattern = true;
|
|
||||||
if (std::regex_match(candidate,
|
|
||||||
std::regex(spec.country_format_regex))) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return !has_pattern;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_MOCK_POSTAL_CODE_SERVICE_H_
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
|
|
||||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @file services/postal_code/postal_code_service.h
|
|
||||||
* @brief Abstraction for resolving a postal code for a city.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include <string>
|
|
||||||
|
|
||||||
#include "data_model/models.h"
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Interface for services that resolve a postal code for a city.
|
|
||||||
*/
|
|
||||||
class IPostalCodeService {
|
|
||||||
public:
|
|
||||||
IPostalCodeService() = default;
|
|
||||||
virtual ~IPostalCodeService() = default;
|
|
||||||
|
|
||||||
IPostalCodeService(const IPostalCodeService&) = delete;
|
|
||||||
IPostalCodeService& operator=(const IPostalCodeService&) = delete;
|
|
||||||
IPostalCodeService(IPostalCodeService&&) = delete;
|
|
||||||
IPostalCodeService& operator=(IPostalCodeService&&) = delete;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Resolves a postal code for @p city.
|
|
||||||
*
|
|
||||||
* @param city City whose curated postal-code examples supply the result.
|
|
||||||
* @return A postal code string, or an empty string if none is available.
|
|
||||||
*/
|
|
||||||
virtual std::string GeneratePostalCode(const City& city) = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_POSTAL_CODE_SERVICE_H_
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
|
|
||||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @file services/postal_code/xeger_postal_code_service.h
|
|
||||||
* @brief Postal code service that generates a fresh, format-conformant code
|
|
||||||
* per call from the curated regex patterns, instead of returning a fixed
|
|
||||||
* example.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include <random>
|
|
||||||
|
|
||||||
#include "services/postal_code/postal_code_service.h"
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @brief Postal code service using regex-driven generation (xeger).
|
|
||||||
*
|
|
||||||
* Picks among the city's curated `city_regexes` (falls back to
|
|
||||||
* `country_format_regex`), then generates a random string matching that pattern
|
|
||||||
* via GenerateStringFromRegex().
|
|
||||||
*/
|
|
||||||
class XegerPostalCodeService final : public IPostalCodeService {
|
|
||||||
public:
|
|
||||||
XegerPostalCodeService();
|
|
||||||
|
|
||||||
std::string GeneratePostalCode(const City& city) override;
|
|
||||||
|
|
||||||
private:
|
|
||||||
std::mt19937 rng_;
|
|
||||||
};
|
|
||||||
|
|
||||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_POSTAL_CODE_XEGER_POSTAL_CODE_SERVICE_H_
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -38,9 +38,8 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
"Number of layers to offload to GPU");
|
"Number of layers to offload to GPU");
|
||||||
};
|
};
|
||||||
|
|
||||||
// --mocked, --model, and --openai are mutually exclusive; validation is
|
// --mocked, --model, and --openai are mutually exclusive, validated below
|
||||||
// enforced below rather than at registration to produce a clear
|
// to produce a clear diagnostic message.
|
||||||
// diagnostic message.
|
|
||||||
auto add_generator_options = [&]() -> void {
|
auto add_generator_options = [&]() -> void {
|
||||||
opt("mocked", prog_opts::bool_switch(),
|
opt("mocked", prog_opts::bool_switch(),
|
||||||
"Use mocked generator for brewery/user data");
|
"Use mocked generator for brewery/user data");
|
||||||
@@ -154,9 +153,7 @@ std::optional<ApplicationOptions> ParseArguments(
|
|||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prompt directory is only meaningful for live inference (Llama or
|
// Prompt directory is required only for live inference (Llama or OpenAI).
|
||||||
// OpenAI) — the mock generator has no use for it and should not
|
|
||||||
// require it to be present.
|
|
||||||
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
|
if (!use_mocked && options.pipeline.prompt_dir.empty()) {
|
||||||
const std::string msg =
|
const std::string msg =
|
||||||
"Invalid arguments: --prompt-dir is required when not using "
|
"Invalid arguments: --prompt-dir is required when not using "
|
||||||
|
|||||||
@@ -13,12 +13,10 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
|
|||||||
std::unique_ptr<DataGenerator> generator,
|
std::unique_ptr<DataGenerator> generator,
|
||||||
std::unique_ptr<IExportService> exporter,
|
std::unique_ptr<IExportService> exporter,
|
||||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||||
std::unique_ptr<IPostalCodeService> postal_code_service,
|
|
||||||
const ApplicationOptions& application_options)
|
const ApplicationOptions& application_options)
|
||||||
: logger_(std::move(logger)),
|
: logger_(std::move(logger)),
|
||||||
context_service_(std::move(context_service)),
|
context_service_(std::move(context_service)),
|
||||||
generator_(std::move(generator)),
|
generator_(std::move(generator)),
|
||||||
exporter_(std::move(exporter)),
|
exporter_(std::move(exporter)),
|
||||||
curated_data_service_(std::move(curated_data_service)),
|
curated_data_service_(std::move(curated_data_service)),
|
||||||
postal_code_service_(std::move(postal_code_service)),
|
|
||||||
application_options_(application_options) {}
|
application_options_(application_options) {}
|
||||||
@@ -4,54 +4,16 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <cmath>
|
|
||||||
#include <format>
|
#include <format>
|
||||||
#include <numbers>
|
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#include <random>
|
#include <random>
|
||||||
|
|
||||||
#include "biergarten_pipeline_orchestrator.h"
|
#include "biergarten_pipeline_orchestrator.h"
|
||||||
|
#include "biergarten_pipeline_orchestrator/coords.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
struct Coords {
|
using biergarten_pipeline_orchestrator_internal::Coords;
|
||||||
double longitude;
|
using biergarten_pipeline_orchestrator_internal::GetRandomCoordsWithinRange;
|
||||||
double latitude;
|
|
||||||
};
|
|
||||||
|
|
||||||
// @todo find a better home for this
|
|
||||||
namespace {
|
|
||||||
constexpr double kEarthRadiusKm = 6371.0;
|
|
||||||
|
|
||||||
double DegreesToRadians(double degrees) {
|
|
||||||
return degrees * std::numbers::pi / 180.0;
|
|
||||||
}
|
|
||||||
|
|
||||||
double RadiansToDegrees(double radians) {
|
|
||||||
return radians * 180.0 / std::numbers::pi;
|
|
||||||
}
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
static Coords GetRandomCoordsWithinRange(const Coords& centre,
|
|
||||||
const int distance_km,
|
|
||||||
std::mt19937& rng) {
|
|
||||||
std::uniform_real_distribution<double> angle_dist(0.0,
|
|
||||||
2.0 * std::numbers::pi);
|
|
||||||
std::uniform_real_distribution<double> unit_dist(0.0, 1.0);
|
|
||||||
|
|
||||||
const double angle_radians = angle_dist(rng);
|
|
||||||
// sqrt() keeps points uniformly distributed over the disc's area rather
|
|
||||||
// than clustering near the centre.
|
|
||||||
const double radius_km = distance_km * std::sqrt(unit_dist(rng));
|
|
||||||
|
|
||||||
const double delta_latitude_degrees =
|
|
||||||
RadiansToDegrees(radius_km * std::cos(angle_radians) / kEarthRadiusKm);
|
|
||||||
const double delta_longitude_degrees = RadiansToDegrees(
|
|
||||||
radius_km * std::sin(angle_radians) /
|
|
||||||
(kEarthRadiusKm * std::cos(DegreesToRadians(centre.latitude))));
|
|
||||||
|
|
||||||
return Coords{.longitude = centre.longitude + delta_longitude_degrees,
|
|
||||||
.latitude = centre.latitude + delta_latitude_degrees};
|
|
||||||
}
|
|
||||||
|
|
||||||
void BiergartenPipelineOrchestrator::GenerateBreweries(
|
void BiergartenPipelineOrchestrator::GenerateBreweries(
|
||||||
std::span<const EnrichedCity> cities) {
|
std::span<const EnrichedCity> cities) {
|
||||||
@@ -70,8 +32,6 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
|||||||
try {
|
try {
|
||||||
const BreweryResult brewery =
|
const BreweryResult brewery =
|
||||||
generator_->GenerateBrewery(enriched_city);
|
generator_->GenerateBrewery(enriched_city);
|
||||||
const std::string postal_code =
|
|
||||||
postal_code_service_->GeneratePostalCode(enriched_city.location);
|
|
||||||
|
|
||||||
constexpr int kMaxDistanceFromCentreKm = 5;
|
constexpr int kMaxDistanceFromCentreKm = 5;
|
||||||
const Coords city_centre{.longitude = enriched_city.location.longitude,
|
const Coords city_centre{.longitude = enriched_city.location.longitude,
|
||||||
@@ -81,7 +41,6 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
|||||||
|
|
||||||
return BreweryRecord{
|
return BreweryRecord{
|
||||||
.address = BreweryAddress{.city = enriched_city.location,
|
.address = BreweryAddress{.city = enriched_city.location,
|
||||||
.postal_code = postal_code,
|
|
||||||
.longitude = brewery_coords.longitude,
|
.longitude = brewery_coords.longitude,
|
||||||
.latitude = brewery_coords.latitude},
|
.latitude = brewery_coords.latitude},
|
||||||
.brewery = brewery};
|
.brewery = brewery};
|
||||||
|
|||||||
@@ -15,9 +15,13 @@
|
|||||||
#include <unordered_set>
|
#include <unordered_set>
|
||||||
|
|
||||||
#include "biergarten_pipeline_orchestrator.h"
|
#include "biergarten_pipeline_orchestrator.h"
|
||||||
|
#include "biergarten_pipeline_orchestrator/coords.h"
|
||||||
#include "services/curated_data/curated_json_data_service.h"
|
#include "services/curated_data/curated_json_data_service.h"
|
||||||
#include "services/logging/logger.h"
|
#include "services/logging/logger.h"
|
||||||
|
|
||||||
|
using biergarten_pipeline_orchestrator_internal::Coords;
|
||||||
|
using biergarten_pipeline_orchestrator_internal::GetRandomCoordsWithinRange;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
std::string Sanitize(std::string_view value) {
|
std::string Sanitize(std::string_view value) {
|
||||||
std::string out;
|
std::string out;
|
||||||
@@ -135,12 +139,17 @@ void BiergartenPipelineOrchestrator::GenerateUsers(
|
|||||||
try {
|
try {
|
||||||
const UserResult user =
|
const UserResult user =
|
||||||
generator_->GenerateUser(city, persona, sampled_name);
|
generator_->GenerateUser(city, persona, sampled_name);
|
||||||
const std::string postal_code =
|
|
||||||
postal_code_service_->GeneratePostalCode(city.location);
|
constexpr int kMaxDistanceFromCentreKm = 5;
|
||||||
|
const Coords city_centre{.longitude = city.location.longitude,
|
||||||
|
.latitude = city.location.latitude};
|
||||||
|
const Coords user_coords = GetRandomCoordsWithinRange(
|
||||||
|
city_centre, kMaxDistanceFromCentreKm, rng);
|
||||||
|
|
||||||
return UserRecord{
|
return UserRecord{
|
||||||
.address =
|
.address = UserAddress{.city = city.location,
|
||||||
UserAddress{.city = city.location, .postal_code = postal_code},
|
.longitude = user_coords.longitude,
|
||||||
|
.latitude = user_coords.latitude},
|
||||||
.user = user,
|
.user = user,
|
||||||
.email = BuildEmail(sampled_name, used_email_local_parts),
|
.email = BuildEmail(sampled_name, used_email_local_parts),
|
||||||
.date_of_birth = GenerateDateOfBirth(rng),
|
.date_of_birth = GenerateDateOfBirth(rng),
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ void BiergartenPipelineOrchestrator::LogResults() const {
|
|||||||
{"country", city.country},
|
{"country", city.country},
|
||||||
{"state_province", city.state_province},
|
{"state_province", city.state_province},
|
||||||
{"iso3166_2", city.iso3166_2},
|
{"iso3166_2", city.iso3166_2},
|
||||||
{"postal_code", address.postal_code},
|
|
||||||
{"longitude", address.longitude},
|
{"longitude", address.longitude},
|
||||||
{"latitude", address.latitude},
|
{"latitude", address.latitude},
|
||||||
}}});
|
}}});
|
||||||
|
|||||||
@@ -58,9 +58,7 @@ std::string CondenseWhitespace(std::string_view text) {
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Guard against truncating in the first half of the string.
|
// Keeps truncation past the string's midpoint, preserving the critical opening content.
|
||||||
// This preserves the critical opening content and avoids cutting critical
|
|
||||||
// context words early in the region description.
|
|
||||||
constexpr size_t kTruncationGuardDivisor = 2;
|
constexpr size_t kTruncationGuardDivisor = 2;
|
||||||
|
|
||||||
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
|
bool ReadRequiredTrimmedStringField(const boost::json::object& obj,
|
||||||
|
|||||||
@@ -36,10 +36,7 @@ std::string FormatLocalLanguageCodes(
|
|||||||
|
|
||||||
constexpr int kBreweryMaxTokens = 2800;
|
constexpr int kBreweryMaxTokens = 2800;
|
||||||
|
|
||||||
// Structured outputs already guarantee schema-valid JSON, so retries here
|
// Structured outputs already guarantee schema-valid JSON, so retries here only cover transient failures or the model returning placeholder text.
|
||||||
// are only for transient failures or the model returning placeholder text
|
|
||||||
// -- not for malformed JSON shape, unlike LlamaGenerator's grammar-assisted
|
|
||||||
// retry loop.
|
|
||||||
constexpr int kMaxAttempts = 2;
|
constexpr int kMaxAttempts = 2;
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
@@ -54,9 +51,7 @@ BreweryResult OpenAIGenerator::GenerateBrewery(
|
|||||||
FormatLocalLanguageCodes(location.local_languages);
|
FormatLocalLanguageCodes(location.local_languages);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load brewery system prompt via the injected prompt directory. The
|
* Loads the backend-agnostic brewery system prompt via the injected prompt directory.
|
||||||
* same file LlamaGenerator uses -- the prompt content is backend
|
|
||||||
* agnostic.
|
|
||||||
*/
|
*/
|
||||||
const std::string system_prompt =
|
const std::string system_prompt =
|
||||||
prompt_directory_->Load("BREWERY_GENERATION");
|
prompt_directory_->Load("BREWERY_GENERATION");
|
||||||
|
|||||||
@@ -16,8 +16,7 @@
|
|||||||
namespace {
|
namespace {
|
||||||
constexpr int kUserMaxTokens = 1200;
|
constexpr int kUserMaxTokens = 1200;
|
||||||
|
|
||||||
// Structured outputs already guarantee schema-valid JSON, so retries here
|
// Structured outputs already guarantee schema-valid JSON, so retries here only cover transient failures or placeholder text.
|
||||||
// are only for transient failures or placeholder text -- not JSON shape.
|
|
||||||
constexpr int kMaxAttempts = 2;
|
constexpr int kMaxAttempts = 2;
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
|
|||||||
@@ -87,36 +87,16 @@ int main(const int argc, char** argv) {
|
|||||||
di::bind<ApplicationOptions>().to(options),
|
di::bind<ApplicationOptions>().to(options),
|
||||||
di::bind<std::string>().to(model_path),
|
di::bind<std::string>().to(model_path),
|
||||||
di::bind<IExportService>().to<SqliteExportService>(),
|
di::bind<IExportService>().to<SqliteExportService>(),
|
||||||
di::bind<IPostalCodeService>().to(
|
|
||||||
[options,
|
|
||||||
&log_producer]() -> std::unique_ptr<IPostalCodeService> {
|
|
||||||
if (options.generator.mode == GeneratorMode::kMock) {
|
|
||||||
log_producer->Log(
|
|
||||||
{.level = LogLevel::Info,
|
|
||||||
.phase = PipelinePhase::Startup,
|
|
||||||
.message = "Postal code: mock (curated examples)"});
|
|
||||||
|
|
||||||
return std::make_unique<MockPostalCodeService>();
|
|
||||||
}
|
|
||||||
|
|
||||||
log_producer->Log(
|
|
||||||
{.level = LogLevel::Info,
|
|
||||||
.phase = PipelinePhase::Startup,
|
|
||||||
.message =
|
|
||||||
"Postal code: Xeger (regex-based generation)"});
|
|
||||||
|
|
||||||
return std::make_unique<XegerPostalCodeService>();
|
|
||||||
}),
|
|
||||||
di::bind<ICuratedDataService>().to(
|
di::bind<ICuratedDataService>().to(
|
||||||
[options,
|
[options,
|
||||||
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
|
&log_producer]() -> std::unique_ptr<ICuratedDataService> {
|
||||||
// if (options.generator.mode == GeneratorMode::kMock) {
|
if (options.generator.mode == GeneratorMode::kMock) {
|
||||||
// log_producer->Log({.level = LogLevel::Info,
|
log_producer->Log({.level = LogLevel::Info,
|
||||||
// .phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
// .message = "Curated data: mock"});
|
.message = "Curated data: mock"});
|
||||||
//
|
|
||||||
// return std::make_unique<MockCuratedDataService>();
|
return std::make_unique<MockCuratedDataService>();
|
||||||
// }
|
}
|
||||||
|
|
||||||
log_producer->Log({.level = LogLevel::Info,
|
log_producer->Log({.level = LogLevel::Info,
|
||||||
.phase = PipelinePhase::Startup,
|
.phase = PipelinePhase::Startup,
|
||||||
@@ -214,9 +194,7 @@ int main(const int argc, char** argv) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#ifdef BIERGARTEN_MOCK_ONLY
|
#ifdef BIERGARTEN_MOCK_ONLY
|
||||||
// Only reachable when mode == Llama: Mock and OpenAI are
|
// Only reachable when mode == Llama, which requires llama.cpp and is excluded from a BIERGARTEN_MOCK_ONLY build.
|
||||||
// handled above and don't need llama.cpp, but LlamaGenerator
|
|
||||||
// itself is not linked into a BIERGARTEN_MOCK_ONLY build.
|
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
|
"LlamaGenerator is unavailable in a BIERGARTEN_MOCK_ONLY "
|
||||||
"build");
|
"build");
|
||||||
|
|||||||
@@ -77,12 +77,11 @@ static PostalCodeSpec ReadPostalCodeSpec(const boost::json::object& object) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
|
boost::json::value ParseJsonFile(const std::filesystem::path& filepath) {
|
||||||
const char* what) {
|
|
||||||
std::ifstream input(filepath);
|
std::ifstream input(filepath);
|
||||||
if (!input.is_open()) {
|
if (!input.is_open()) {
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
std::format("Failed to open {} file: {}", what, filepath.string()));
|
std::format("Failed to open file: {}", filepath.string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::stringstream buffer;
|
std::stringstream buffer;
|
||||||
@@ -92,7 +91,7 @@ boost::json::value ParseJsonFile(const std::filesystem::path& filepath,
|
|||||||
boost::json::value root = boost::json::parse(buffer.str(), error);
|
boost::json::value root = boost::json::parse(buffer.str(), error);
|
||||||
if (error) {
|
if (error) {
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
std::format("Failed to parse {} JSON: {}", what, error.message()));
|
std::format("Failed to parse JSON: {}", error.message()));
|
||||||
}
|
}
|
||||||
return root;
|
return root;
|
||||||
}
|
}
|
||||||
@@ -126,34 +125,47 @@ const LocationsList& CuratedJsonDataService::LoadLocations() {
|
|||||||
if (!cache_.locations.empty()) {
|
if (!cache_.locations.empty()) {
|
||||||
return cache_.locations;
|
return cache_.locations;
|
||||||
}
|
}
|
||||||
|
boost::json::value root_val =
|
||||||
|
ParseJsonFile(filepaths_.locations_path);
|
||||||
|
|
||||||
const boost::json::value root =
|
if (!root_val.is_object()) {
|
||||||
ParseJsonFile(filepaths_.locations_path, "locations");
|
|
||||||
|
|
||||||
if (!root.is_array()) {
|
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
"Invalid locations JSON: root element must be an array");
|
"Invalid locations JSON: root element must be an object");
|
||||||
}
|
}
|
||||||
|
|
||||||
LocationsList locations;
|
boost::json::object root_obj = root_val.as_object();
|
||||||
const auto& items = root.as_array();
|
|
||||||
locations.reserve(items.size());
|
|
||||||
|
|
||||||
for (const auto& item : items) {
|
if (!root_obj.contains("cities"))
|
||||||
|
{
|
||||||
|
throw std::runtime_error(
|
||||||
|
"Invalid locations JSON: root object must contain a 'cities' key");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!root_obj["cities"].is_array())
|
||||||
|
{
|
||||||
|
throw std::runtime_error(
|
||||||
|
"Invalid locations JSON: 'cities' key must be an array");
|
||||||
|
}
|
||||||
|
|
||||||
|
LocationsList locations;
|
||||||
|
const boost::json::array& cities = root_obj["cities"].as_array();
|
||||||
|
locations.reserve(cities.size());
|
||||||
|
|
||||||
|
for (const boost::json::value& item : cities) {
|
||||||
if (!item.is_object()) {
|
if (!item.is_object()) {
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
"Invalid locations JSON: each entry must be an object");
|
"Invalid locations JSON: each entry must be an object");
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& object = item.as_object();
|
const boost::json::object& object = item.as_object();
|
||||||
locations.push_back(City{
|
locations.push_back(City{
|
||||||
.city = ReadRequiredString(object, "city"),
|
.city = ReadRequiredString(object, "city"),
|
||||||
.state_province = ReadRequiredString(object, "state_province"),
|
.state_province = ReadRequiredString(object, "state_province"),
|
||||||
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
|
.iso3166_2 = ReadRequiredString(object, "iso3166_2"),
|
||||||
.country = ReadRequiredString(object, "country"),
|
.country = ReadRequiredString(object, "country"),
|
||||||
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
|
.iso3166_1 = ReadRequiredString(object, "iso3166_1"),
|
||||||
.latitude = ReadRequiredDouble(object, "latitude"),
|
|
||||||
.longitude = ReadRequiredDouble(object, "longitude"),
|
.longitude = ReadRequiredDouble(object, "longitude"),
|
||||||
|
.latitude = ReadRequiredDouble(object, "latitude"),
|
||||||
.local_languages = ReadRequiredStringArray(object, "local_languages"),
|
.local_languages = ReadRequiredStringArray(object, "local_languages"),
|
||||||
.postal_code = ReadPostalCodeSpec(object),
|
.postal_code = ReadPostalCodeSpec(object),
|
||||||
});
|
});
|
||||||
@@ -168,7 +180,7 @@ const PersonasList& CuratedJsonDataService::LoadPersonas() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const boost::json::value root =
|
const boost::json::value root =
|
||||||
ParseJsonFile(filepaths_.personas_path, "personas");
|
ParseJsonFile(filepaths_.personas_path);
|
||||||
|
|
||||||
if (!root.is_array()) {
|
if (!root.is_array()) {
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
@@ -176,10 +188,10 @@ const PersonasList& CuratedJsonDataService::LoadPersonas() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
PersonasList personas;
|
PersonasList personas;
|
||||||
const auto& items = root.as_array();
|
const boost::json::array& items = root.as_array();
|
||||||
personas.reserve(items.size());
|
personas.reserve(items.size());
|
||||||
|
|
||||||
for (const auto& item : items) {
|
for (const boost::json::value& item : items) {
|
||||||
if (!item.is_object()) {
|
if (!item.is_object()) {
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
"Invalid personas JSON: each entry must be an object");
|
"Invalid personas JSON: each entry must be an object");
|
||||||
@@ -204,7 +216,7 @@ const ForenamesByCountryMap& CuratedJsonDataService::LoadForenamesByCountry() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const boost::json::value root =
|
const boost::json::value root =
|
||||||
ParseJsonFile(filepaths_.forenames_path, "forenames-by-country");
|
ParseJsonFile(filepaths_.forenames_path);
|
||||||
|
|
||||||
if (!root.is_object()) {
|
if (!root.is_object()) {
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
@@ -252,7 +264,7 @@ const SurnamesByCountryMap& CuratedJsonDataService::LoadSurnamesByCountry() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const boost::json::value root =
|
const boost::json::value root =
|
||||||
ParseJsonFile(filepaths_.surnames_path, "surnames-by-country");
|
ParseJsonFile(filepaths_.surnames_path);
|
||||||
|
|
||||||
if (!root.is_object()) {
|
if (!root.is_object()) {
|
||||||
throw std::runtime_error(
|
throw std::runtime_error(
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
/**
|
|
||||||
* @file services/postal_code/xeger_postal_code_service.cc
|
|
||||||
* @brief XegerPostalCodeService::GeneratePostalCode() implementation.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include "services/postal_code/xeger_postal_code_service.h"
|
|
||||||
|
|
||||||
#include <format>
|
|
||||||
#include <random>
|
|
||||||
#include <stdexcept>
|
|
||||||
#include <string>
|
|
||||||
#include <string_view>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "services/xeger/xeger.h"
|
|
||||||
|
|
||||||
XegerPostalCodeService::XegerPostalCodeService()
|
|
||||||
: rng_(std::random_device{}()) {}
|
|
||||||
|
|
||||||
std::string XegerPostalCodeService::GeneratePostalCode(const City& city) {
|
|
||||||
const PostalCodeSpec& spec = city.postal_code;
|
|
||||||
|
|
||||||
std::vector<std::string_view> city_patterns;
|
|
||||||
for (const std::string& pattern : spec.city_regexes) {
|
|
||||||
if (!pattern.empty()) {
|
|
||||||
city_patterns.push_back(pattern);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string_view pattern;
|
|
||||||
if (!city_patterns.empty()) {
|
|
||||||
std::uniform_int_distribution<size_t> dist(0, city_patterns.size() - 1);
|
|
||||||
pattern = city_patterns[dist(rng_)];
|
|
||||||
} else if (!spec.country_format_regex.empty()) {
|
|
||||||
pattern = spec.country_format_regex;
|
|
||||||
} else {
|
|
||||||
throw std::runtime_error(std::format(
|
|
||||||
"XegerPostalCodeService: no postal-code pattern available for "
|
|
||||||
"city '{}'",
|
|
||||||
city.city));
|
|
||||||
}
|
|
||||||
|
|
||||||
return GenerateStringFromRegex(pattern, rng_);
|
|
||||||
}
|
|
||||||
@@ -170,13 +170,6 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
|
|||||||
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
|
sqlite_export_service_internal::kBreweryAddressCityIdBindIndex,
|
||||||
.value = city_id,
|
.value = city_id,
|
||||||
.action = "Failed to bind SQLite brewery address city id"});
|
.action = "Failed to bind SQLite brewery address city id"});
|
||||||
sqlite_export_service_internal::Bind(
|
|
||||||
insert_brewery_address_stmt_,
|
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
|
||||||
.index = sqlite_export_service_internal::
|
|
||||||
kBreweryAddressPostalCodeBindIndex,
|
|
||||||
.value = brewery.address.postal_code,
|
|
||||||
.action = "Failed to bind SQLite brewery address postal code"});
|
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_brewery_address_stmt_,
|
insert_brewery_address_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<double>{
|
sqlite_export_service_internal::BoundParam<double>{
|
||||||
|
|||||||
@@ -87,11 +87,18 @@ uint64_t SqliteExportService::ProcessRecord(const UserRecord& user) {
|
|||||||
.action = "Failed to bind SQLite user address city id"});
|
.action = "Failed to bind SQLite user address city id"});
|
||||||
sqlite_export_service_internal::Bind(
|
sqlite_export_service_internal::Bind(
|
||||||
insert_user_address_stmt_,
|
insert_user_address_stmt_,
|
||||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
sqlite_export_service_internal::BoundParam<double>{
|
||||||
.index =
|
.index =
|
||||||
sqlite_export_service_internal::kUserAddressPostalCodeBindIndex,
|
sqlite_export_service_internal::kUserAddressLongitudeBindIndex,
|
||||||
.value = user.address.postal_code,
|
.value = user.address.longitude,
|
||||||
.action = "Failed to bind SQLite user address postal code"});
|
.action = "Failed to bind SQLite user address longitude"});
|
||||||
|
sqlite_export_service_internal::Bind(
|
||||||
|
insert_user_address_stmt_,
|
||||||
|
sqlite_export_service_internal::BoundParam<double>{
|
||||||
|
.index =
|
||||||
|
sqlite_export_service_internal::kUserAddressLatitudeBindIndex,
|
||||||
|
.value = user.address.latitude,
|
||||||
|
.action = "Failed to bind SQLite user address latitude"});
|
||||||
|
|
||||||
sqlite_export_service_internal::StepStatement(
|
sqlite_export_service_internal::StepStatement(
|
||||||
db_handle_, insert_user_address_stmt_,
|
db_handle_, insert_user_address_stmt_,
|
||||||
|
|||||||
@@ -8,20 +8,20 @@
|
|||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="idunno.Password.Generator" Version="2.0.0" />
|
<PackageReference Include="idunno.Password.Generator" Version="2.0.0"/>
|
||||||
<PackageReference Include="Microsoft.Data.SqlClient" Version="7.0.2" />
|
<PackageReference Include="Microsoft.Data.SqlClient" Version="7.0.2"/>
|
||||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.11" />
|
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.11"/>
|
||||||
<PackageReference Include="Spectre.Console" Version="0.57.2" />
|
<PackageReference Include="Spectre.Console" Version="0.57.2"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\..\Domain\Domain.Entities\Domain.Entities.csproj" />
|
<ProjectReference Include="..\..\Domain\Domain.Entities\Domain.Entities.csproj"/>
|
||||||
<ProjectReference Include="..\..\Infrastructure\Infrastructure.Sql\Infrastructure.Sql.csproj" />
|
<ProjectReference Include="..\..\Infrastructure\Infrastructure.Sql\Infrastructure.Sql.csproj"/>
|
||||||
<ProjectReference Include="..\..\Features\Features.Breweries\Features.Breweries.csproj" />
|
<ProjectReference Include="..\..\Features\Features.Breweries\Features.Breweries.csproj"/>
|
||||||
<ProjectReference Include="..\..\Features\Features.Auth\Features.Auth.csproj" />
|
<ProjectReference Include="..\..\Features\Features.Auth\Features.Auth.csproj"/>
|
||||||
<ProjectReference Include="..\..\Features\Features.Locations\Features.Locations.csproj" />
|
<ProjectReference Include="..\..\Features\Features.Locations\Features.Locations.csproj"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None CopyToOutputDirectory="PreserveNewest" Include="SeedData\open-ai-test2.sqlite" />
|
<None CopyToOutputDirectory="PreserveNewest" Include="SeedData\biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -12,6 +12,5 @@ public class ConnectionStrings
|
|||||||
public static string GetSqlServerConnectionString(IConfiguration configuration) =>
|
public static string GetSqlServerConnectionString(IConfiguration configuration) =>
|
||||||
SqlConnectionStringHelper.BuildConnectionString(configuration);
|
SqlConnectionStringHelper.BuildConnectionString(configuration);
|
||||||
|
|
||||||
public static string SqliteConnectionString => "Data Source=SeedData/open-ai-test2.sqlite";
|
public static string SqliteConnectionString => "Data Source=SeedData/biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite";
|
||||||
// "Data Source=SeedData/biergarten_seed_2026-07-20T05-46-02.993511Z.sqlite";
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using System.Net.Http.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace Database.Seed.Geocoding;
|
||||||
|
|
||||||
|
public sealed record ReverseGeocodeResult(string AddressLine1, string PostalCode);
|
||||||
|
|
||||||
|
internal sealed record NominatimAddress
|
||||||
|
{
|
||||||
|
[JsonPropertyName("house_number")]
|
||||||
|
public string? HouseNumber { get; init; }
|
||||||
|
|
||||||
|
[JsonPropertyName("road")]
|
||||||
|
public string? Road { get; init; }
|
||||||
|
|
||||||
|
[JsonPropertyName("postcode")]
|
||||||
|
public string? Postcode { get; init; }
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed record NominatimReverseGeocodeResponse
|
||||||
|
{
|
||||||
|
[JsonPropertyName("address")]
|
||||||
|
public NominatimAddress? Address { get; init; }
|
||||||
|
|
||||||
|
[JsonPropertyName("error")]
|
||||||
|
public string? Error { get; init; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reverse geocodes coordinates into street addresses via the public Nominatim API
|
||||||
|
/// (https://nominatim.openstreetmap.org), following its usage policy of a single
|
||||||
|
/// identifying User-Agent and at most one request per second
|
||||||
|
/// (https://operations.osmfoundation.org/policies/nominatim/).
|
||||||
|
/// </summary>
|
||||||
|
public sealed class NominatimReverseGeocoder : IDisposable
|
||||||
|
{
|
||||||
|
private const string ReverseEndpoint = "https://nominatim.openstreetmap.org/reverse";
|
||||||
|
private static readonly TimeSpan MinRequestInterval = TimeSpan.FromSeconds(1);
|
||||||
|
|
||||||
|
private readonly HttpClient _httpClient;
|
||||||
|
private readonly SemaphoreSlim _rateLimitLock = new(1, 1);
|
||||||
|
private DateTime _lastRequestUtc = DateTime.MinValue;
|
||||||
|
|
||||||
|
public NominatimReverseGeocoder(string userAgent)
|
||||||
|
{
|
||||||
|
_httpClient = new HttpClient();
|
||||||
|
_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reverse geocodes a coordinate pair, returning <c>null</c> if Nominatim has no address
|
||||||
|
/// for the point or the request fails so callers can fall back to a placeholder address.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<ReverseGeocodeResult?> ReverseGeocodeAsync(
|
||||||
|
double longitude,
|
||||||
|
double latitude,
|
||||||
|
CancellationToken cancellationToken = default
|
||||||
|
)
|
||||||
|
{
|
||||||
|
await WaitForRateLimitAsync(cancellationToken);
|
||||||
|
|
||||||
|
string url =
|
||||||
|
$"{ReverseEndpoint}?format=jsonv2"
|
||||||
|
+ $"&lat={latitude.ToString(CultureInfo.InvariantCulture)}"
|
||||||
|
+ $"&lon={longitude.ToString(CultureInfo.InvariantCulture)}"
|
||||||
|
+ "&zoom=18&addressdetails=1";
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
NominatimReverseGeocodeResponse? response =
|
||||||
|
await _httpClient.GetFromJsonAsync<NominatimReverseGeocodeResponse>(
|
||||||
|
url,
|
||||||
|
cancellationToken
|
||||||
|
);
|
||||||
|
|
||||||
|
if (response?.Address is null || response.Error is not null)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
string addressLine1 = string.Join(
|
||||||
|
' ',
|
||||||
|
new[] { response.Address.HouseNumber, response.Address.Road }.Where(
|
||||||
|
part => !string.IsNullOrWhiteSpace(part)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
return string.IsNullOrWhiteSpace(addressLine1)
|
||||||
|
? null
|
||||||
|
: new ReverseGeocodeResult(addressLine1, response.Address.Postcode ?? string.Empty);
|
||||||
|
}
|
||||||
|
catch (HttpRequestException)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
// HttpClient's own timeout elapsed here.
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task WaitForRateLimitAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await _rateLimitLock.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
TimeSpan elapsed = DateTime.UtcNow - _lastRequestUtc;
|
||||||
|
if (elapsed < MinRequestInterval)
|
||||||
|
await Task.Delay(MinRequestInterval - elapsed, cancellationToken);
|
||||||
|
|
||||||
|
_lastRequestUtc = DateTime.UtcNow;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_rateLimitLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_httpClient.Dispose();
|
||||||
|
_rateLimitLock.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,8 +4,6 @@ public sealed record BreweryAddress
|
|||||||
{
|
{
|
||||||
public required City City { get; init; }
|
public required City City { get; init; }
|
||||||
|
|
||||||
public required string PostalCode { get; init; }
|
|
||||||
|
|
||||||
public required double Longitude { get; init; }
|
public required double Longitude { get; init; }
|
||||||
|
|
||||||
public required double Latitude { get; init; }
|
public required double Latitude { get; init; }
|
||||||
|
|||||||
@@ -4,5 +4,7 @@ public sealed record UserAddress
|
|||||||
{
|
{
|
||||||
public required City City { get; init; }
|
public required City City { get; init; }
|
||||||
|
|
||||||
public required string PostalCode { get; init; }
|
public required double Longitude { get; init; }
|
||||||
|
|
||||||
|
public required double Latitude { get; init; }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
using Database.Seed.DatabaseHelpers;
|
using Database.Seed.DatabaseHelpers;
|
||||||
|
using Database.Seed.Geocoding;
|
||||||
using Database.Seed.PipelineData;
|
using Database.Seed.PipelineData;
|
||||||
using Database.Seed.Sqlite;
|
using Database.Seed.Sqlite;
|
||||||
using Domain.Entities;
|
using Domain.Entities;
|
||||||
@@ -16,6 +17,9 @@ using Microsoft.Extensions.Configuration;
|
|||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Spectre.Console;
|
using Spectre.Console;
|
||||||
|
|
||||||
|
const string NominatimUserAgent =
|
||||||
|
"TheBiergartenApp-Seeder/1.0 (+https://github.com/aaronpo97/the-biergarten-app)";
|
||||||
|
|
||||||
return await RunAsync();
|
return await RunAsync();
|
||||||
|
|
||||||
static async Task<int> RunAsync()
|
static async Task<int> RunAsync()
|
||||||
@@ -100,23 +104,34 @@ static async Task<int> RunAsync()
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
await AnsiConsole
|
List<string> geocodingWarnings = [];
|
||||||
.Status()
|
|
||||||
.Spinner(Spinner.Known.Dots)
|
using (NominatimReverseGeocoder geocoder = new(NominatimUserAgent))
|
||||||
.SpinnerStyle(Style.Parse("green"))
|
{
|
||||||
.StartAsync(
|
await AnsiConsole
|
||||||
"Loading brewery data into target database...",
|
.Status()
|
||||||
async ctx =>
|
.Spinner(Spinner.Known.Dots)
|
||||||
{
|
.SpinnerStyle(Style.Parse("green"))
|
||||||
await LoadBreweriesIntoDatabaseAsync(
|
.StartAsync(
|
||||||
breweryRepository,
|
"Loading brewery data into target database...",
|
||||||
locationRepository,
|
async ctx =>
|
||||||
seedData.Breweries,
|
{
|
||||||
postedByIds
|
await LoadBreweriesIntoDatabaseAsync(
|
||||||
);
|
breweryRepository,
|
||||||
ctx.Status("Brewery data loaded into target database.");
|
locationRepository,
|
||||||
}
|
geocoder,
|
||||||
);
|
seedData.Breweries,
|
||||||
|
postedByIds,
|
||||||
|
geocodingWarnings,
|
||||||
|
ctx
|
||||||
|
);
|
||||||
|
ctx.Status("Brewery data loaded into target database.");
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (string warning in geocodingWarnings)
|
||||||
|
AnsiConsole.MarkupLine($"[yellow]![/] {warning.EscapeMarkup()}");
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@@ -165,20 +180,18 @@ static async Task<IReadOnlyList<Guid>> LoadUsersIntoDatabaseAsync(
|
|||||||
return userAccountIds;
|
return userAccountIds;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Persists each <paramref name="breweries" /> record, resolving (and creating, if needed) its
|
|
||||||
/// Country/StateProvince/City chain via <paramref name="locationRepository" />. The pipeline
|
|
||||||
/// data has no street-level address or a specific user tied to each brewery, so a placeholder
|
|
||||||
/// address is used and <c>PostedById</c> is assigned round-robin from <paramref name="posterUserIds" />
|
|
||||||
/// to keep seeding deterministic across runs.
|
|
||||||
/// </summary>
|
|
||||||
static async Task LoadBreweriesIntoDatabaseAsync(
|
static async Task LoadBreweriesIntoDatabaseAsync(
|
||||||
IBreweryRepository breweryRepository,
|
IBreweryRepository breweryRepository,
|
||||||
ILocationRepository locationRepository,
|
ILocationRepository locationRepository,
|
||||||
|
NominatimReverseGeocoder geocoder,
|
||||||
IReadOnlyList<BreweryRecord> breweries,
|
IReadOnlyList<BreweryRecord> breweries,
|
||||||
IReadOnlyList<Guid> posterUserIds
|
IReadOnlyList<Guid> posterUserIds,
|
||||||
|
List<string> geocodingWarnings,
|
||||||
|
StatusContext ctx
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
|
const string placeholderAddressLine1 = "Address unavailable";
|
||||||
|
|
||||||
if (posterUserIds.Count == 0)
|
if (posterUserIds.Count == 0)
|
||||||
throw new InvalidOperationException(
|
throw new InvalidOperationException(
|
||||||
"Cannot load breweries without any registered users.");
|
"Cannot load breweries without any registered users.");
|
||||||
@@ -187,6 +200,10 @@ static async Task LoadBreweriesIntoDatabaseAsync(
|
|||||||
{
|
{
|
||||||
BreweryRecord breweryRecord = breweries[i];
|
BreweryRecord breweryRecord = breweries[i];
|
||||||
|
|
||||||
|
ctx.Status(
|
||||||
|
$"Geocoding and loading brewery {i + 1}/{breweries.Count} into target database..."
|
||||||
|
);
|
||||||
|
|
||||||
Guid cityId = await locationRepository.GetOrCreateCityIdAsync(
|
Guid cityId = await locationRepository.GetOrCreateCityIdAsync(
|
||||||
new CityLocation(
|
new CityLocation(
|
||||||
breweryRecord.Address.City.CityName,
|
breweryRecord.Address.City.CityName,
|
||||||
@@ -197,6 +214,18 @@ static async Task LoadBreweriesIntoDatabaseAsync(
|
|||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
ReverseGeocodeResult? geocoded = await geocoder.ReverseGeocodeAsync(
|
||||||
|
breweryRecord.Address.Longitude,
|
||||||
|
breweryRecord.Address.Latitude
|
||||||
|
);
|
||||||
|
|
||||||
|
if (geocoded is null)
|
||||||
|
geocodingWarnings.Add(
|
||||||
|
$"Could not reverse geocode '{breweryRecord.Brewery.NameEn}' "
|
||||||
|
+ $"({breweryRecord.Address.Latitude}, {breweryRecord.Address.Longitude}); "
|
||||||
|
+ "using a placeholder address."
|
||||||
|
);
|
||||||
|
|
||||||
await breweryRepository.CreateAsync(
|
await breweryRepository.CreateAsync(
|
||||||
new BreweryPost
|
new BreweryPost
|
||||||
{
|
{
|
||||||
@@ -204,12 +233,13 @@ static async Task LoadBreweriesIntoDatabaseAsync(
|
|||||||
BreweryName = breweryRecord.Brewery.NameEn,
|
BreweryName = breweryRecord.Brewery.NameEn,
|
||||||
Description = breweryRecord.Brewery.DescriptionEn,
|
Description = breweryRecord.Brewery.DescriptionEn,
|
||||||
PostedById = posterUserIds[i % posterUserIds.Count],
|
PostedById = posterUserIds[i % posterUserIds.Count],
|
||||||
|
// TODO: Populate Coordinates from breweryRecord.Address.Latitude/Longitude once it goes through NetTopologySuite (NTS).
|
||||||
Location = new BreweryPostLocation
|
Location = new BreweryPostLocation
|
||||||
{
|
{
|
||||||
BreweryPostLocationId = Guid.NewGuid(),
|
BreweryPostLocationId = Guid.NewGuid(),
|
||||||
CityId = cityId,
|
CityId = cityId,
|
||||||
AddressLine1 = "Address unavailable",
|
AddressLine1 = geocoded?.AddressLine1 ?? placeholderAddressLine1,
|
||||||
PostalCode = breweryRecord.Address.PostalCode,
|
PostalCode = geocoded?.PostalCode ?? string.Empty,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@@ -224,7 +254,8 @@ static Table BuildBreweryTable(IReadOnlyList<BreweryRecord> breweries)
|
|||||||
.AddColumn("City")
|
.AddColumn("City")
|
||||||
.AddColumn("State/Province")
|
.AddColumn("State/Province")
|
||||||
.AddColumn("Country")
|
.AddColumn("Country")
|
||||||
.AddColumn("Postal Code");
|
.AddColumn("Longitude")
|
||||||
|
.AddColumn("Latitude");
|
||||||
|
|
||||||
foreach (BreweryRecord brewery in breweries)
|
foreach (BreweryRecord brewery in breweries)
|
||||||
{
|
{
|
||||||
@@ -234,7 +265,8 @@ static Table BuildBreweryTable(IReadOnlyList<BreweryRecord> breweries)
|
|||||||
brewery.Address.City.CityName,
|
brewery.Address.City.CityName,
|
||||||
brewery.Address.City.StateProvince,
|
brewery.Address.City.StateProvince,
|
||||||
brewery.Address.City.Country,
|
brewery.Address.City.Country,
|
||||||
brewery.Address.PostalCode
|
brewery.Address.Longitude.ToString("F6"),
|
||||||
|
brewery.Address.Latitude.ToString("F6")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -54,7 +54,7 @@ public sealed class PipelineSeedDataReader
|
|||||||
{
|
{
|
||||||
const string sql = """
|
const string sql = """
|
||||||
SELECT b.name_en, b.description_en, b.name_local, b.description_local,
|
SELECT b.name_en, b.description_en, b.name_local, b.description_local,
|
||||||
ba.city_id, ba.postal_code, ba.longitude, ba.latitude
|
ba.city_id, ba.longitude, ba.latitude
|
||||||
FROM breweries b
|
FROM breweries b
|
||||||
JOIN brewery_addresses ba ON ba.brewery_id = b.id;
|
JOIN brewery_addresses ba ON ba.brewery_id = b.id;
|
||||||
""";
|
""";
|
||||||
@@ -67,9 +67,8 @@ public sealed class PipelineSeedDataReader
|
|||||||
const int nameLocalIndex = 2;
|
const int nameLocalIndex = 2;
|
||||||
const int descriptionLocalIndex = 3;
|
const int descriptionLocalIndex = 3;
|
||||||
const int cityIdIndex = 4;
|
const int cityIdIndex = 4;
|
||||||
const int postalCodeIndex = 5;
|
const int longitudeIndex = 5;
|
||||||
const int longitudeIndex = 6;
|
const int latitudeIndex = 6;
|
||||||
const int latitudeIndex = 7;
|
|
||||||
|
|
||||||
List<BreweryRecord> records = [];
|
List<BreweryRecord> records = [];
|
||||||
while (await reader.ReadAsync(cancellationToken))
|
while (await reader.ReadAsync(cancellationToken))
|
||||||
@@ -86,7 +85,6 @@ public sealed class PipelineSeedDataReader
|
|||||||
Address = new BreweryAddress
|
Address = new BreweryAddress
|
||||||
{
|
{
|
||||||
City = cities[reader.GetInt64(cityIdIndex)],
|
City = cities[reader.GetInt64(cityIdIndex)],
|
||||||
PostalCode = reader.GetString(postalCodeIndex),
|
|
||||||
Longitude = reader.GetDouble(longitudeIndex),
|
Longitude = reader.GetDouble(longitudeIndex),
|
||||||
Latitude = reader.GetDouble(latitudeIndex),
|
Latitude = reader.GetDouble(latitudeIndex),
|
||||||
},
|
},
|
||||||
@@ -105,7 +103,7 @@ public sealed class PipelineSeedDataReader
|
|||||||
const string sql = """
|
const string sql = """
|
||||||
SELECT u.first_name, u.last_name, u.gender, u.username, u.bio,
|
SELECT u.first_name, u.last_name, u.gender, u.username, u.bio,
|
||||||
u.activity_weight, u.email, u.date_of_birth,
|
u.activity_weight, u.email, u.date_of_birth,
|
||||||
ua.city_id, ua.postal_code
|
ua.city_id, ua.longitude, ua.latitude
|
||||||
FROM users u
|
FROM users u
|
||||||
JOIN user_addresses ua ON ua.user_id = u.id;
|
JOIN user_addresses ua ON ua.user_id = u.id;
|
||||||
""";
|
""";
|
||||||
@@ -122,7 +120,8 @@ public sealed class PipelineSeedDataReader
|
|||||||
const int emailIndex = 6;
|
const int emailIndex = 6;
|
||||||
const int dateOfBirthIndex = 7;
|
const int dateOfBirthIndex = 7;
|
||||||
const int cityIdIndex = 8;
|
const int cityIdIndex = 8;
|
||||||
const int postalCodeIndex = 9;
|
const int longitudeIndex = 9;
|
||||||
|
const int latitudeIndex = 10;
|
||||||
|
|
||||||
List<UserRecord> records = [];
|
List<UserRecord> records = [];
|
||||||
while (await reader.ReadAsync(cancellationToken))
|
while (await reader.ReadAsync(cancellationToken))
|
||||||
@@ -143,7 +142,8 @@ public sealed class PipelineSeedDataReader
|
|||||||
Address = new UserAddress
|
Address = new UserAddress
|
||||||
{
|
{
|
||||||
City = cities[reader.GetInt64(cityIdIndex)],
|
City = cities[reader.GetInt64(cityIdIndex)],
|
||||||
PostalCode = reader.GetString(postalCodeIndex),
|
Longitude = reader.GetDouble(longitudeIndex),
|
||||||
|
Latitude = reader.GetDouble(latitudeIndex),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -283,7 +283,7 @@ public class BreweryRepository(ISqlConnectionFactory connectionFactory)
|
|||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
{
|
{
|
||||||
await transaction.RollbackAsync();
|
await RollbackQuietlyAsync(transaction);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -418,11 +418,28 @@ public class BreweryRepository(ISqlConnectionFactory connectionFactory)
|
|||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
{
|
{
|
||||||
await transaction.RollbackAsync();
|
await RollbackQuietlyAsync(transaction);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Rolls back <paramref name="transaction" />, swallowing any exception the rollback itself
|
||||||
|
/// raises (for example when the provider has already completed the transaction after a
|
||||||
|
/// connection failure) so the exception that triggered the rollback is what propagates.
|
||||||
|
/// </summary>
|
||||||
|
private static async Task RollbackQuietlyAsync(DbTransaction transaction)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await transaction.RollbackAsync();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Ignore: the original exception (rethrown by the caller) is what matters here.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Composes a joined row's <see cref="BreweryPost" />, <see cref="BreweryPostLocation" />,
|
/// Composes a joined row's <see cref="BreweryPost" />, <see cref="BreweryPostLocation" />,
|
||||||
/// <see cref="City" />, <see cref="StateProvince" /> and <see cref="Country" /> fragments into a
|
/// <see cref="City" />, <see cref="StateProvince" /> and <see cref="Country" /> fragments into a
|
||||||
|
|||||||
Reference in New Issue
Block a user