mirror of
https://github.com/aaronpo97/the-biergarten-app.git
synced 2026-10-14 17:54:42 +00:00
Move brewery reverse geocoding from C# seeding to C++ pipeline (#249)
This commit is contained in:
@@ -144,6 +144,8 @@ FetchContent_MakeAvailable(cpp-httplib)
|
||||
# 5. Executable & Sources
|
||||
add_executable(${PROJECT_NAME}
|
||||
includes/services/enrichment/mock_enrichment.h
|
||||
includes/services/address/address_service.h
|
||||
includes/services/address/mock_address_service.h
|
||||
includes/services/curated_data/mock_curated_data_service.h
|
||||
includes/json_handling/pretty_print.h)
|
||||
|
||||
@@ -225,6 +227,11 @@ target_sources(${PROJECT_NAME} PRIVATE
|
||||
src/services/enrichment/wikipedia/get_summary.cc
|
||||
)
|
||||
|
||||
# --- services: address ---
|
||||
target_sources(${PROJECT_NAME} PRIVATE
|
||||
src/services/address/nominatim_address_service.cc
|
||||
)
|
||||
|
||||
# --- services: xeger ---
|
||||
target_sources(${PROJECT_NAME} PRIVATE
|
||||
src/services/xeger/xeger.cc
|
||||
|
||||
@@ -13,6 +13,11 @@
|
||||
// --- concurrency ---
|
||||
#include "concurrency/bounded_channel.h"
|
||||
|
||||
// --- services: address ---
|
||||
#include "services/address/address_service.h"
|
||||
#include "services/address/mock_address_service.h"
|
||||
#include "services/address/nominatim_address_service.h"
|
||||
|
||||
// --- data_generation ---
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_generation/generation_json_validation.h"
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
|
||||
#include "data_generation/data_generator.h"
|
||||
#include "data_model/generated_models.h"
|
||||
#include "services/address/address_service.h"
|
||||
#include "services/curated_data/curated_data_service.h"
|
||||
#include "services/database/export_service.h"
|
||||
#include "services/enrichment/enrichment_service.h"
|
||||
@@ -34,6 +35,8 @@ class BiergartenPipelineOrchestrator {
|
||||
* @param exporter Database backend for persisting generated records.
|
||||
* @param curated_data_service Loads curated location, persona, and name
|
||||
* data used to seed generation.
|
||||
* @param address_service Resolves a street address for generated
|
||||
* brewery coordinates.
|
||||
* @param application_options CLI configuration and paths.
|
||||
*/
|
||||
BiergartenPipelineOrchestrator(
|
||||
@@ -42,6 +45,7 @@ class BiergartenPipelineOrchestrator {
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||
std::unique_ptr<IAddressService> address_service,
|
||||
const ApplicationOptions& application_options);
|
||||
|
||||
/**
|
||||
@@ -78,6 +82,7 @@ class BiergartenPipelineOrchestrator {
|
||||
*/
|
||||
std::unique_ptr<IExportService> exporter_;
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service_;
|
||||
std::unique_ptr<IAddressService> address_service_;
|
||||
|
||||
ApplicationOptions application_options_;
|
||||
|
||||
|
||||
@@ -95,6 +95,18 @@ struct BreweryAddress {
|
||||
City city;
|
||||
double longitude{};
|
||||
double latitude{};
|
||||
|
||||
/**
|
||||
* @brief Street address line (e.g. house number + road), resolved once
|
||||
* via reverse geocoding at generation time.
|
||||
*/
|
||||
std::string address_line1{};
|
||||
|
||||
/**
|
||||
* @brief Postal code, resolved once via reverse geocoding at generation
|
||||
* time.
|
||||
*/
|
||||
std::string postal_code{};
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
40
tooling/pipeline/includes/services/address/address_service.h
Normal file
40
tooling/pipeline/includes/services/address/address_service.h
Normal file
@@ -0,0 +1,40 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_ADDRESS_SERVICE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_ADDRESS_SERVICE_H_
|
||||
|
||||
/**
|
||||
* @file services/address/address_service.h
|
||||
* @brief Abstraction for resolving a street address from coordinates.
|
||||
*/
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
/**
|
||||
* @brief Street-level address resolved from a coordinate pair.
|
||||
*/
|
||||
struct AddressLookupResult {
|
||||
std::string address_line1;
|
||||
std::string postal_code;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Interface for services that reverse-geocode coordinates into a
|
||||
* street address.
|
||||
*/
|
||||
class IAddressService {
|
||||
public:
|
||||
virtual ~IAddressService() = default;
|
||||
|
||||
/**
|
||||
* @brief Reverse-geocodes a coordinate pair into a street address.
|
||||
*
|
||||
* @param longitude Longitude of the point to resolve.
|
||||
* @param latitude Latitude of the point to resolve.
|
||||
* @return The resolved address, or std::nullopt if it could not be
|
||||
* resolved.
|
||||
*/
|
||||
virtual std::optional<AddressLookupResult> ReverseGeocode(
|
||||
double longitude, double latitude) = 0;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_ADDRESS_SERVICE_H_
|
||||
@@ -0,0 +1,27 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_MOCK_ADDRESS_SERVICE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_MOCK_ADDRESS_SERVICE_H_
|
||||
|
||||
/**
|
||||
* @file services/address/mock_address_service.h
|
||||
* @brief Deterministic IAddressService used when network geocoding is
|
||||
* disabled.
|
||||
*/
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "services/address/address_service.h"
|
||||
|
||||
/**
|
||||
* @brief Address service that returns a fixed placeholder address for any
|
||||
* coordinate pair, without making network calls.
|
||||
*/
|
||||
class MockAddressService final : public IAddressService {
|
||||
public:
|
||||
std::optional<AddressLookupResult> ReverseGeocode(
|
||||
double /*longitude*/, double /*latitude*/) override {
|
||||
return AddressLookupResult{.address_line1 = "123 Mock Street",
|
||||
.postal_code = "00000"};
|
||||
}
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_MOCK_ADDRESS_SERVICE_H_
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_NOMINATIM_ADDRESS_SERVICE_H_
|
||||
#define BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_NOMINATIM_ADDRESS_SERVICE_H_
|
||||
|
||||
/**
|
||||
* @file services/address/nominatim_address_service.h
|
||||
* @brief Reverse geocoding backed by the public Nominatim API.
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
||||
#include "services/address/address_service.h"
|
||||
#include "services/logging/logger.h"
|
||||
#include "web_client/web_client.h"
|
||||
|
||||
/**
|
||||
* @brief Reverse-geocodes coordinates into street addresses via the public
|
||||
* Nominatim API (https://nominatim.openstreetmap.org).
|
||||
*
|
||||
* Required for API usage:
|
||||
* - single identifying User-Agent and at most one request per
|
||||
* second (https://operations.osmfoundation.org/policies/nominatim/).
|
||||
*/
|
||||
class NominatimAddressService final : public IAddressService {
|
||||
public:
|
||||
explicit NominatimAddressService(std::unique_ptr<WebClient> client,
|
||||
std::shared_ptr<ILogger> logger);
|
||||
|
||||
std::optional<AddressLookupResult> ReverseGeocode(
|
||||
double longitude, double latitude) override;
|
||||
|
||||
private:
|
||||
std::unique_ptr<WebClient> client_;
|
||||
std::shared_ptr<ILogger> logger_;
|
||||
};
|
||||
|
||||
#endif // BIERGARTEN_PIPELINE_INCLUDES_SERVICES_ADDRESS_NOMINATIM_ADDRESS_SERVICE_H_
|
||||
@@ -56,6 +56,8 @@ CREATE TABLE IF NOT EXISTS brewery_addresses (
|
||||
city_id INTEGER NOT NULL,
|
||||
longitude REAL NOT NULL,
|
||||
latitude REAL NOT NULL,
|
||||
address_line1 TEXT NOT NULL,
|
||||
postal_code TEXT NOT NULL,
|
||||
FOREIGN KEY(brewery_id) REFERENCES breweries(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(city_id) REFERENCES cities(id) ON DELETE CASCADE
|
||||
);
|
||||
@@ -131,8 +133,10 @@ INSERT INTO brewery_addresses (
|
||||
brewery_id,
|
||||
city_id,
|
||||
longitude,
|
||||
latitude
|
||||
) VALUES (?, ?, ?, ?);
|
||||
latitude,
|
||||
address_line1,
|
||||
postal_code
|
||||
) VALUES (?, ?, ?, ?, ?, ?);
|
||||
)sql";
|
||||
|
||||
inline constexpr std::string_view kInsertUserSql = R"sql(
|
||||
@@ -184,6 +188,8 @@ enum BreweryAddressBindIndex {
|
||||
kBreweryAddressCityIdBindIndex,
|
||||
kBreweryAddressLongitudeBindIndex,
|
||||
kBreweryAddressLatitudeBindIndex,
|
||||
kBreweryAddressLine1BindIndex,
|
||||
kBreweryAddressPostalCodeBindIndex,
|
||||
};
|
||||
|
||||
enum UserBindIndex {
|
||||
|
||||
@@ -13,10 +13,12 @@ BiergartenPipelineOrchestrator::BiergartenPipelineOrchestrator(
|
||||
std::unique_ptr<DataGenerator> generator,
|
||||
std::unique_ptr<IExportService> exporter,
|
||||
std::unique_ptr<ICuratedDataService> curated_data_service,
|
||||
std::unique_ptr<IAddressService> address_service,
|
||||
const ApplicationOptions& application_options)
|
||||
: logger_(std::move(logger)),
|
||||
context_service_(std::move(context_service)),
|
||||
generator_(std::move(generator)),
|
||||
exporter_(std::move(exporter)),
|
||||
curated_data_service_(std::move(curated_data_service)),
|
||||
address_service_(std::move(address_service)),
|
||||
application_options_(application_options) {}
|
||||
@@ -7,14 +7,21 @@
|
||||
#include <format>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "biergarten_pipeline_orchestrator.h"
|
||||
#include "biergarten_pipeline_orchestrator/coords.h"
|
||||
#include "services/address/address_service.h"
|
||||
#include "services/logging/logger.h"
|
||||
|
||||
using biergarten_pipeline_orchestrator_internal::Coords;
|
||||
using biergarten_pipeline_orchestrator_internal::GetRandomCoordsWithinRange;
|
||||
|
||||
namespace {
|
||||
constexpr std::string_view kPlaceholderAddressLine1 = "Address unavailable";
|
||||
} // namespace
|
||||
|
||||
void BiergartenPipelineOrchestrator::GenerateBreweries(
|
||||
std::span<const EnrichedCity> cities) {
|
||||
logger_->Log({.level = LogLevel::Info,
|
||||
@@ -39,10 +46,32 @@ void BiergartenPipelineOrchestrator::GenerateBreweries(
|
||||
const Coords brewery_coords = GetRandomCoordsWithinRange(
|
||||
city_centre, kMaxDistanceFromCentreKm, rng);
|
||||
|
||||
const std::optional<AddressLookupResult> geocoded =
|
||||
address_service_->ReverseGeocode(brewery_coords.longitude,
|
||||
brewery_coords.latitude);
|
||||
|
||||
if (!geocoded.has_value()) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::BreweryAndBeerGeneration,
|
||||
.message = std::format(
|
||||
"[Pipeline] Could not reverse geocode brewery in '{}' "
|
||||
"({}, {}); using a placeholder address.",
|
||||
enriched_city.location.city, brewery_coords.latitude,
|
||||
brewery_coords.longitude)});
|
||||
}
|
||||
|
||||
return BreweryRecord{
|
||||
.address = BreweryAddress{.city = enriched_city.location,
|
||||
.address =
|
||||
BreweryAddress{
|
||||
.city = enriched_city.location,
|
||||
.longitude = brewery_coords.longitude,
|
||||
.latitude = brewery_coords.latitude},
|
||||
.latitude = brewery_coords.latitude,
|
||||
.address_line1 = geocoded.has_value()
|
||||
? geocoded->address_line1
|
||||
: std::string(kPlaceholderAddressLine1),
|
||||
.postal_code =
|
||||
geocoded.has_value() ? geocoded->postal_code : ""},
|
||||
.brewery = brewery};
|
||||
} catch (const std::exception& e) {
|
||||
++skipped_count;
|
||||
|
||||
@@ -166,6 +166,26 @@ int main(const int argc, char** argv) {
|
||||
inj.template create<std::unique_ptr<WebClient>>(),
|
||||
log_producer);
|
||||
}),
|
||||
di::bind<IAddressService>().to(
|
||||
[options, &log_producer](
|
||||
const auto& inj) -> std::unique_ptr<IAddressService> {
|
||||
if (options.generator.mode == GeneratorMode::kMock) {
|
||||
log_producer->Log({.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "Address service: mock"});
|
||||
|
||||
return std::make_unique<MockAddressService>();
|
||||
}
|
||||
|
||||
log_producer->Log(
|
||||
{.level = LogLevel::Info,
|
||||
.phase = PipelinePhase::Startup,
|
||||
.message = "Address service: Nominatim"});
|
||||
|
||||
return std::make_unique<NominatimAddressService>(
|
||||
inj.template create<std::unique_ptr<WebClient>>(),
|
||||
log_producer);
|
||||
}),
|
||||
di::bind<DataGenerator>().to([&options, &model_path, &sampling,
|
||||
&prompt_directory,
|
||||
&log_producer](const auto& inj)
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* @file services/address/nominatim_address_service.cc
|
||||
* @brief NominatimAddressService implementation.
|
||||
*/
|
||||
|
||||
#include "services/address/nominatim_address_service.h"
|
||||
|
||||
#include <boost/json.hpp>
|
||||
#include <chrono>
|
||||
#include <format>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
using namespace boost;
|
||||
|
||||
NominatimAddressService::NominatimAddressService(
|
||||
std::unique_ptr<WebClient> client, std::shared_ptr<ILogger> logger)
|
||||
: client_(std::move(client)), logger_(std::move(logger)) {}
|
||||
|
||||
std::optional<AddressLookupResult> NominatimAddressService::ReverseGeocode(
|
||||
double longitude, double latitude) {
|
||||
if (!client_) {
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = "Nominatim client is nullptr."});
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string url = std::format(
|
||||
"https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat={}&"
|
||||
"lon={}&zoom=18&addressdetails=1",
|
||||
latitude, longitude);
|
||||
|
||||
std::string body;
|
||||
try {
|
||||
body = client_->Get(url);
|
||||
} catch (const std::runtime_error& e) {
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format(
|
||||
"NominatimAddressService: request failed for "
|
||||
"({}, {}): {}",
|
||||
latitude, longitude, e.what())});
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
{
|
||||
using namespace std::literals::chrono_literals;
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
|
||||
system::error_code ec;
|
||||
const json::value doc = json::parse(body, ec);
|
||||
|
||||
if (ec) {
|
||||
if (logger_) {
|
||||
logger_->Log({.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format(
|
||||
"NominatimAddressService: JSON parse error for "
|
||||
"({}, {}): {}",
|
||||
latitude, longitude, ec.message())});
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const json::object* obj = doc.if_object();
|
||||
if ((obj == nullptr) || obj->contains("error")) {
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format("NominatimAddressService: no address "
|
||||
"found for ({}, {})",
|
||||
latitude, longitude)});
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const json::value* address_ptr = obj->if_contains("address");
|
||||
if ((address_ptr == nullptr) || !address_ptr->is_object()) {
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format("NominatimAddressService: missing "
|
||||
"'address' for ({}, {})",
|
||||
latitude, longitude)});
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const json::object& address = address_ptr->get_object();
|
||||
|
||||
const auto extract_string = [&address](std::string_view key) -> std::string {
|
||||
const json::value* value_ptr = address.if_contains(key);
|
||||
if ((value_ptr == nullptr) || !value_ptr->is_string()) {
|
||||
return {};
|
||||
}
|
||||
return std::string(value_ptr->get_string());
|
||||
};
|
||||
|
||||
const std::string house_number = extract_string("house_number");
|
||||
const std::string road = extract_string("road");
|
||||
const std::string postal_code = extract_string("postcode");
|
||||
|
||||
std::string address_line1 = road;
|
||||
if (!house_number.empty()) {
|
||||
address_line1 = address_line1.empty() ? house_number
|
||||
: house_number + " " + road;
|
||||
}
|
||||
|
||||
if (address_line1.empty()) {
|
||||
if (logger_) {
|
||||
logger_->Log(
|
||||
{.level = LogLevel::Warn,
|
||||
.phase = PipelinePhase::Enrichment,
|
||||
.message = std::format("NominatimAddressService: no usable "
|
||||
"street address for ({}, {})",
|
||||
latitude, longitude)});
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return AddressLookupResult{.address_line1 = address_line1,
|
||||
.postal_code = postal_code};
|
||||
}
|
||||
@@ -184,6 +184,20 @@ uint64_t SqliteExportService::ProcessRecord(const BreweryRecord& brewery) {
|
||||
sqlite_export_service_internal::kBreweryAddressLatitudeBindIndex,
|
||||
.value = brewery.address.latitude,
|
||||
.action = "Failed to bind SQLite brewery address latitude"});
|
||||
sqlite_export_service_internal::Bind(
|
||||
insert_brewery_address_stmt_,
|
||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||
.index =
|
||||
sqlite_export_service_internal::kBreweryAddressLine1BindIndex,
|
||||
.value = brewery.address.address_line1,
|
||||
.action = "Failed to bind SQLite brewery address line 1"});
|
||||
sqlite_export_service_internal::Bind(
|
||||
insert_brewery_address_stmt_,
|
||||
sqlite_export_service_internal::BoundParam<std::string_view>{
|
||||
.index = sqlite_export_service_internal::
|
||||
kBreweryAddressPostalCodeBindIndex,
|
||||
.value = brewery.address.postal_code,
|
||||
.action = "Failed to bind SQLite brewery address postal code"});
|
||||
|
||||
sqlite_export_service_internal::StepStatement(
|
||||
db_handle_, insert_brewery_address_stmt_,
|
||||
|
||||
@@ -32,6 +32,7 @@ COPY ["Features/Features.Locations/Features.Locations.csproj", "Features/Feature
|
||||
COPY ["Features/Features.Locations.Tests/Features.Locations.Tests.csproj", "Features/Features.Locations.Tests/"]
|
||||
COPY ["Features/Features.UserManagement/Features.UserManagement.csproj", "Features/Features.UserManagement/"]
|
||||
COPY ["Features/Features.UserManagement.Tests/Features.UserManagement.Tests.csproj", "Features/Features.UserManagement.Tests/"]
|
||||
COPY ["Infrastructure/Infrastructure.Configuration/Infrastructure.Configuration.csproj", "Infrastructure/Infrastructure.Configuration/"]
|
||||
COPY ["Infrastructure/Infrastructure.Email/Infrastructure.Email.csproj", "Infrastructure/Infrastructure.Email/"]
|
||||
COPY ["Infrastructure/Infrastructure.Email.Templates/Infrastructure.Email.Templates.csproj", "Infrastructure/Infrastructure.Email.Templates/"]
|
||||
COPY ["Infrastructure/Infrastructure.Jwt/Infrastructure.Jwt.csproj", "Infrastructure/Infrastructure.Jwt/"]
|
||||
|
||||
@@ -22,6 +22,7 @@ COPY ["Features/Features.Locations/Features.Locations.csproj", "Features/Feature
|
||||
COPY ["Features/Features.Locations.Tests/Features.Locations.Tests.csproj", "Features/Features.Locations.Tests/"]
|
||||
COPY ["Features/Features.UserManagement/Features.UserManagement.csproj", "Features/Features.UserManagement/"]
|
||||
COPY ["Features/Features.UserManagement.Tests/Features.UserManagement.Tests.csproj", "Features/Features.UserManagement.Tests/"]
|
||||
COPY ["Infrastructure/Infrastructure.Configuration/Infrastructure.Configuration.csproj", "Infrastructure/Infrastructure.Configuration/"]
|
||||
COPY ["Infrastructure/Infrastructure.Email/Infrastructure.Email.csproj", "Infrastructure/Infrastructure.Email/"]
|
||||
COPY ["Infrastructure/Infrastructure.Email.Templates/Infrastructure.Email.Templates.csproj", "Infrastructure/Infrastructure.Email.Templates/"]
|
||||
COPY ["Infrastructure/Infrastructure.Jwt/Infrastructure.Jwt.csproj", "Infrastructure/Infrastructure.Jwt/"]
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
<Project Path="Domain/Domain.Exceptions/Domain.Exceptions.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Infrastructure/">
|
||||
<Project Path="Infrastructure/Infrastructure.Configuration/Infrastructure.Configuration.csproj" />
|
||||
<Project Path="Infrastructure/Infrastructure.Email/Infrastructure.Email.csproj" />
|
||||
<Project Path="Infrastructure/Infrastructure.Email.Templates/Infrastructure.Email.Templates.csproj" />
|
||||
<Project Path="Infrastructure/Infrastructure.Jwt/Infrastructure.Jwt.csproj" />
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
<ItemGroup>
|
||||
<None
|
||||
CopyToOutputDirectory="PreserveNewest"
|
||||
Include="SeedData\biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite"
|
||||
Include="SeedData\biergarten_seed_2026-08-25T20-45-50.697244Z.sqlite"
|
||||
/>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -20,5 +20,5 @@ public static class ConnectionStrings
|
||||
);
|
||||
|
||||
public static string SqliteConnectionString =>
|
||||
"Data Source=SeedData/biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite";
|
||||
"Data Source=SeedData/biergarten_seed_2026-08-25T20-45-50.697244Z.sqlite";
|
||||
}
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
using System.Globalization;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Database.Seed.Geocoding;
|
||||
|
||||
public sealed record ReverseGeocodeResult(string AddressLine1, string PostalCode);
|
||||
|
||||
internal sealed record NominatimAddress
|
||||
{
|
||||
[JsonPropertyName("house_number")]
|
||||
public string? HouseNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("road")]
|
||||
public string? Road { get; init; }
|
||||
|
||||
[JsonPropertyName("postcode")]
|
||||
public string? Postcode { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record NominatimReverseGeocodeResponse
|
||||
{
|
||||
[JsonPropertyName("address")]
|
||||
public NominatimAddress? Address { get; init; }
|
||||
|
||||
[JsonPropertyName("error")]
|
||||
public string? Error { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reverse geocodes coordinates into street addresses via the public Nominatim API
|
||||
/// (https://nominatim.openstreetmap.org), following its usage policy of a single
|
||||
/// identifying User-Agent and at most one request per second
|
||||
/// (https://operations.osmfoundation.org/policies/nominatim/).
|
||||
/// </summary>
|
||||
public sealed class NominatimReverseGeocoder : IDisposable
|
||||
{
|
||||
private const string ReverseEndpoint = "https://nominatim.openstreetmap.org/reverse";
|
||||
private static readonly TimeSpan MinRequestInterval = TimeSpan.FromSeconds(1);
|
||||
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly SemaphoreSlim _rateLimitLock = new(1, 1);
|
||||
private DateTime _lastRequestUtc = DateTime.MinValue;
|
||||
|
||||
public NominatimReverseGeocoder(string userAgent)
|
||||
{
|
||||
_httpClient = new HttpClient();
|
||||
_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reverse geocodes a coordinate pair, returning <c>null</c> if Nominatim has no address
|
||||
/// for the point or the request fails so callers can fall back to a placeholder address.
|
||||
/// </summary>
|
||||
public async Task<ReverseGeocodeResult?> ReverseGeocodeAsync(
|
||||
double longitude,
|
||||
double latitude,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
await WaitForRateLimitAsync(cancellationToken);
|
||||
|
||||
string url =
|
||||
$"{ReverseEndpoint}?format=jsonv2"
|
||||
+ $"&lat={latitude.ToString(CultureInfo.InvariantCulture)}"
|
||||
+ $"&lon={longitude.ToString(CultureInfo.InvariantCulture)}"
|
||||
+ "&zoom=18&addressdetails=1";
|
||||
|
||||
try
|
||||
{
|
||||
NominatimReverseGeocodeResponse? response =
|
||||
await _httpClient.GetFromJsonAsync<NominatimReverseGeocodeResponse>(
|
||||
url,
|
||||
cancellationToken
|
||||
);
|
||||
|
||||
if (response?.Address is null || response.Error is not null)
|
||||
return null;
|
||||
|
||||
string addressLine1 = string.Join(
|
||||
' ',
|
||||
new[] { response.Address.HouseNumber, response.Address.Road }.Where(part =>
|
||||
!string.IsNullOrWhiteSpace(part)
|
||||
)
|
||||
);
|
||||
|
||||
return string.IsNullOrWhiteSpace(addressLine1)
|
||||
? null
|
||||
: new ReverseGeocodeResult(addressLine1, response.Address.Postcode ?? string.Empty);
|
||||
}
|
||||
catch (HttpRequestException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// HttpClient's own timeout elapsed here.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WaitForRateLimitAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _rateLimitLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
TimeSpan elapsed = DateTime.UtcNow - _lastRequestUtc;
|
||||
if (elapsed < MinRequestInterval)
|
||||
await Task.Delay(MinRequestInterval - elapsed, cancellationToken);
|
||||
|
||||
_lastRequestUtc = DateTime.UtcNow;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_rateLimitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_httpClient.Dispose();
|
||||
_rateLimitLock.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -7,4 +7,8 @@ public sealed record BreweryAddress
|
||||
public required double Longitude { get; init; }
|
||||
|
||||
public required double Latitude { get; init; }
|
||||
|
||||
public required string AddressLine1 { get; init; }
|
||||
|
||||
public required string PostalCode { get; init; }
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using Database.Connection.DependencyInjection;
|
||||
using Database.Seed.DatabaseHelpers;
|
||||
using Database.Seed.Geocoding;
|
||||
using Database.Seed.PipelineData;
|
||||
using Database.Seed.Sqlite;
|
||||
using Domain.Entities;
|
||||
@@ -17,9 +16,6 @@ using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Spectre.Console;
|
||||
|
||||
const string NominatimUserAgent =
|
||||
"TheBiergartenApp-Seeder/1.0 (+https://github.com/aaronpo97/the-biergarten-app)";
|
||||
|
||||
return await RunAsync();
|
||||
|
||||
static async Task<int> RunAsync()
|
||||
@@ -96,10 +92,6 @@ static async Task<int> RunAsync()
|
||||
}
|
||||
);
|
||||
|
||||
List<string> geocodingWarnings = [];
|
||||
|
||||
using (NominatimReverseGeocoder geocoder = new(NominatimUserAgent))
|
||||
{
|
||||
await AnsiConsole
|
||||
.Status()
|
||||
.Spinner(Spinner.Known.Dots)
|
||||
@@ -111,19 +103,13 @@ static async Task<int> RunAsync()
|
||||
await LoadBreweriesIntoDatabaseAsync(
|
||||
breweryRepository,
|
||||
locationRepository,
|
||||
geocoder,
|
||||
seedData.Breweries,
|
||||
postedByIds,
|
||||
geocodingWarnings,
|
||||
ctx
|
||||
);
|
||||
ctx.Status("Brewery data loaded into target database.");
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
foreach (string warning in geocodingWarnings)
|
||||
AnsiConsole.MarkupLine($"[yellow]![/] {warning.EscapeMarkup()}");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -175,15 +161,11 @@ static async Task<IReadOnlyList<Guid>> LoadUsersIntoDatabaseAsync(
|
||||
static async Task LoadBreweriesIntoDatabaseAsync(
|
||||
IBreweryRepository breweryRepository,
|
||||
ILocationRepository locationRepository,
|
||||
NominatimReverseGeocoder geocoder,
|
||||
IReadOnlyList<BreweryRecord> breweries,
|
||||
IReadOnlyList<Guid> posterUserIds,
|
||||
List<string> geocodingWarnings,
|
||||
StatusContext ctx
|
||||
)
|
||||
{
|
||||
const string placeholderAddressLine1 = "Address unavailable";
|
||||
|
||||
if (posterUserIds.Count == 0)
|
||||
throw new InvalidOperationException("Cannot load breweries without any registered users.");
|
||||
|
||||
@@ -191,9 +173,7 @@ static async Task LoadBreweriesIntoDatabaseAsync(
|
||||
{
|
||||
BreweryRecord breweryRecord = breweries[i];
|
||||
|
||||
ctx.Status(
|
||||
$"Geocoding and loading brewery {i + 1}/{breweries.Count} into target database..."
|
||||
);
|
||||
ctx.Status($"Loading brewery {i + 1}/{breweries.Count} into target database...");
|
||||
|
||||
Guid cityId = await locationRepository.GetOrCreateCityIdAsync(
|
||||
new CityLocation(
|
||||
@@ -205,18 +185,6 @@ static async Task LoadBreweriesIntoDatabaseAsync(
|
||||
)
|
||||
);
|
||||
|
||||
ReverseGeocodeResult? geocoded = await geocoder.ReverseGeocodeAsync(
|
||||
breweryRecord.Address.Longitude,
|
||||
breweryRecord.Address.Latitude
|
||||
);
|
||||
|
||||
if (geocoded is null)
|
||||
geocodingWarnings.Add(
|
||||
$"Could not reverse geocode '{breweryRecord.Brewery.NameEn}' "
|
||||
+ $"({breweryRecord.Address.Latitude}, {breweryRecord.Address.Longitude}); "
|
||||
+ "using a placeholder address."
|
||||
);
|
||||
|
||||
await breweryRepository.CreateAsync(
|
||||
new BreweryPost
|
||||
{
|
||||
@@ -229,8 +197,8 @@ static async Task LoadBreweriesIntoDatabaseAsync(
|
||||
{
|
||||
BreweryPostLocationId = Guid.NewGuid(),
|
||||
CityId = cityId,
|
||||
AddressLine1 = geocoded?.AddressLine1 ?? placeholderAddressLine1,
|
||||
PostalCode = geocoded?.PostalCode ?? string.Empty,
|
||||
AddressLine1 = breweryRecord.Address.AddressLine1,
|
||||
PostalCode = breweryRecord.Address.PostalCode,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -54,7 +54,7 @@ public sealed class PipelineSeedDataReader
|
||||
{
|
||||
const string sql = """
|
||||
SELECT b.name_en, b.description_en, b.name_local, b.description_local,
|
||||
ba.city_id, ba.longitude, ba.latitude
|
||||
ba.city_id, ba.longitude, ba.latitude, ba.address_line1, ba.postal_code
|
||||
FROM breweries b
|
||||
JOIN brewery_addresses ba ON ba.brewery_id = b.id;
|
||||
""";
|
||||
@@ -69,6 +69,8 @@ public sealed class PipelineSeedDataReader
|
||||
const int cityIdIndex = 4;
|
||||
const int longitudeIndex = 5;
|
||||
const int latitudeIndex = 6;
|
||||
const int addressLine1Index = 7;
|
||||
const int postalCodeIndex = 8;
|
||||
|
||||
List<BreweryRecord> records = [];
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
@@ -87,6 +89,8 @@ public sealed class PipelineSeedDataReader
|
||||
City = cities[reader.GetInt64(cityIdIndex)],
|
||||
Longitude = reader.GetDouble(longitudeIndex),
|
||||
Latitude = reader.GetDouble(latitudeIndex),
|
||||
AddressLine1 = reader.GetString(addressLine1Index),
|
||||
PostalCode = reader.GetString(postalCodeIndex),
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user