Move brewery reverse geocoding from C# seeding to C++ pipeline (#249)

This commit is contained in:
2026-08-25 21:55:34 -04:00
committed by Aaron Po
parent 2b82d1918b
commit ad1c09f139
25 changed files with 377 additions and 185 deletions

View File

@@ -24,7 +24,7 @@
<ItemGroup>
<None
CopyToOutputDirectory="PreserveNewest"
Include="SeedData\biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite"
Include="SeedData\biergarten_seed_2026-08-25T20-45-50.697244Z.sqlite"
/>
</ItemGroup>
</Project>

View File

@@ -20,5 +20,5 @@ public static class ConnectionStrings
);
public static string SqliteConnectionString =>
"Data Source=SeedData/biergarten_seed_2026-08-24T01-52-09.841693Z.sqlite";
"Data Source=SeedData/biergarten_seed_2026-08-25T20-45-50.697244Z.sqlite";
}

View File

@@ -1,124 +0,0 @@
using System.Globalization;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
namespace Database.Seed.Geocoding;
public sealed record ReverseGeocodeResult(string AddressLine1, string PostalCode);
internal sealed record NominatimAddress
{
[JsonPropertyName("house_number")]
public string? HouseNumber { get; init; }
[JsonPropertyName("road")]
public string? Road { get; init; }
[JsonPropertyName("postcode")]
public string? Postcode { get; init; }
}
internal sealed record NominatimReverseGeocodeResponse
{
[JsonPropertyName("address")]
public NominatimAddress? Address { get; init; }
[JsonPropertyName("error")]
public string? Error { get; init; }
}
/// <summary>
/// Reverse geocodes coordinates into street addresses via the public Nominatim API
/// (https://nominatim.openstreetmap.org), following its usage policy of a single
/// identifying User-Agent and at most one request per second
/// (https://operations.osmfoundation.org/policies/nominatim/).
/// </summary>
public sealed class NominatimReverseGeocoder : IDisposable
{
private const string ReverseEndpoint = "https://nominatim.openstreetmap.org/reverse";
private static readonly TimeSpan MinRequestInterval = TimeSpan.FromSeconds(1);
private readonly HttpClient _httpClient;
private readonly SemaphoreSlim _rateLimitLock = new(1, 1);
private DateTime _lastRequestUtc = DateTime.MinValue;
public NominatimReverseGeocoder(string userAgent)
{
_httpClient = new HttpClient();
_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
}
/// <summary>
/// Reverse geocodes a coordinate pair, returning <c>null</c> if Nominatim has no address
/// for the point or the request fails so callers can fall back to a placeholder address.
/// </summary>
public async Task<ReverseGeocodeResult?> ReverseGeocodeAsync(
double longitude,
double latitude,
CancellationToken cancellationToken = default
)
{
await WaitForRateLimitAsync(cancellationToken);
string url =
$"{ReverseEndpoint}?format=jsonv2"
+ $"&lat={latitude.ToString(CultureInfo.InvariantCulture)}"
+ $"&lon={longitude.ToString(CultureInfo.InvariantCulture)}"
+ "&zoom=18&addressdetails=1";
try
{
NominatimReverseGeocodeResponse? response =
await _httpClient.GetFromJsonAsync<NominatimReverseGeocodeResponse>(
url,
cancellationToken
);
if (response?.Address is null || response.Error is not null)
return null;
string addressLine1 = string.Join(
' ',
new[] { response.Address.HouseNumber, response.Address.Road }.Where(part =>
!string.IsNullOrWhiteSpace(part)
)
);
return string.IsNullOrWhiteSpace(addressLine1)
? null
: new ReverseGeocodeResult(addressLine1, response.Address.Postcode ?? string.Empty);
}
catch (HttpRequestException)
{
return null;
}
catch (TaskCanceledException) when (!cancellationToken.IsCancellationRequested)
{
// HttpClient's own timeout elapsed here.
return null;
}
}
private async Task WaitForRateLimitAsync(CancellationToken cancellationToken)
{
await _rateLimitLock.WaitAsync(cancellationToken);
try
{
TimeSpan elapsed = DateTime.UtcNow - _lastRequestUtc;
if (elapsed < MinRequestInterval)
await Task.Delay(MinRequestInterval - elapsed, cancellationToken);
_lastRequestUtc = DateTime.UtcNow;
}
finally
{
_rateLimitLock.Release();
}
}
public void Dispose()
{
_httpClient.Dispose();
_rateLimitLock.Dispose();
}
}

View File

@@ -7,4 +7,8 @@ public sealed record BreweryAddress
public required double Longitude { get; init; }
public required double Latitude { get; init; }
public required string AddressLine1 { get; init; }
public required string PostalCode { get; init; }
}

View File

@@ -1,6 +1,5 @@
using Database.Connection.DependencyInjection;
using Database.Seed.DatabaseHelpers;
using Database.Seed.Geocoding;
using Database.Seed.PipelineData;
using Database.Seed.Sqlite;
using Domain.Entities;
@@ -17,9 +16,6 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Spectre.Console;
const string NominatimUserAgent =
"TheBiergartenApp-Seeder/1.0 (+https://github.com/aaronpo97/the-biergarten-app)";
return await RunAsync();
static async Task<int> RunAsync()
@@ -96,34 +92,24 @@ static async Task<int> RunAsync()
}
);
List<string> geocodingWarnings = [];
using (NominatimReverseGeocoder geocoder = new(NominatimUserAgent))
{
await AnsiConsole
.Status()
.Spinner(Spinner.Known.Dots)
.SpinnerStyle(Style.Parse("green"))
.StartAsync(
"Loading brewery data into target database...",
async ctx =>
{
await LoadBreweriesIntoDatabaseAsync(
breweryRepository,
locationRepository,
geocoder,
seedData.Breweries,
postedByIds,
geocodingWarnings,
ctx
);
ctx.Status("Brewery data loaded into target database.");
}
);
}
foreach (string warning in geocodingWarnings)
AnsiConsole.MarkupLine($"[yellow]![/] {warning.EscapeMarkup()}");
await AnsiConsole
.Status()
.Spinner(Spinner.Known.Dots)
.SpinnerStyle(Style.Parse("green"))
.StartAsync(
"Loading brewery data into target database...",
async ctx =>
{
await LoadBreweriesIntoDatabaseAsync(
breweryRepository,
locationRepository,
seedData.Breweries,
postedByIds,
ctx
);
ctx.Status("Brewery data loaded into target database.");
}
);
return 0;
}
@@ -175,15 +161,11 @@ static async Task<IReadOnlyList<Guid>> LoadUsersIntoDatabaseAsync(
static async Task LoadBreweriesIntoDatabaseAsync(
IBreweryRepository breweryRepository,
ILocationRepository locationRepository,
NominatimReverseGeocoder geocoder,
IReadOnlyList<BreweryRecord> breweries,
IReadOnlyList<Guid> posterUserIds,
List<string> geocodingWarnings,
StatusContext ctx
)
{
const string placeholderAddressLine1 = "Address unavailable";
if (posterUserIds.Count == 0)
throw new InvalidOperationException("Cannot load breweries without any registered users.");
@@ -191,9 +173,7 @@ static async Task LoadBreweriesIntoDatabaseAsync(
{
BreweryRecord breweryRecord = breweries[i];
ctx.Status(
$"Geocoding and loading brewery {i + 1}/{breweries.Count} into target database..."
);
ctx.Status($"Loading brewery {i + 1}/{breweries.Count} into target database...");
Guid cityId = await locationRepository.GetOrCreateCityIdAsync(
new CityLocation(
@@ -205,18 +185,6 @@ static async Task LoadBreweriesIntoDatabaseAsync(
)
);
ReverseGeocodeResult? geocoded = await geocoder.ReverseGeocodeAsync(
breweryRecord.Address.Longitude,
breweryRecord.Address.Latitude
);
if (geocoded is null)
geocodingWarnings.Add(
$"Could not reverse geocode '{breweryRecord.Brewery.NameEn}' "
+ $"({breweryRecord.Address.Latitude}, {breweryRecord.Address.Longitude}); "
+ "using a placeholder address."
);
await breweryRepository.CreateAsync(
new BreweryPost
{
@@ -229,8 +197,8 @@ static async Task LoadBreweriesIntoDatabaseAsync(
{
BreweryPostLocationId = Guid.NewGuid(),
CityId = cityId,
AddressLine1 = geocoded?.AddressLine1 ?? placeholderAddressLine1,
PostalCode = geocoded?.PostalCode ?? string.Empty,
AddressLine1 = breweryRecord.Address.AddressLine1,
PostalCode = breweryRecord.Address.PostalCode,
},
}
);

View File

@@ -54,7 +54,7 @@ public sealed class PipelineSeedDataReader
{
const string sql = """
SELECT b.name_en, b.description_en, b.name_local, b.description_local,
ba.city_id, ba.longitude, ba.latitude
ba.city_id, ba.longitude, ba.latitude, ba.address_line1, ba.postal_code
FROM breweries b
JOIN brewery_addresses ba ON ba.brewery_id = b.id;
""";
@@ -69,6 +69,8 @@ public sealed class PipelineSeedDataReader
const int cityIdIndex = 4;
const int longitudeIndex = 5;
const int latitudeIndex = 6;
const int addressLine1Index = 7;
const int postalCodeIndex = 8;
List<BreweryRecord> records = [];
while (await reader.ReadAsync(cancellationToken))
@@ -87,6 +89,8 @@ public sealed class PipelineSeedDataReader
City = cities[reader.GetInt64(cityIdIndex)],
Longitude = reader.GetDouble(longitudeIndex),
Latitude = reader.GetDouble(latitudeIndex),
AddressLine1 = reader.GetString(addressLine1Index),
PostalCode = reader.GetString(postalCodeIndex),
},
}
);