Add timeout to wikipedia enrichment to avoid breaking rate limits, add mock enrichment (#224)

* Add timeout for enrichment, refactor json deserialization

* Add location count to application options and as a cli arg

* Add mock enrichment process

tooling/pipeline/src/services/enrichment/wikipedia/fetch_extract.cc

@@ -0,0 +1,112 @@
+/**
+ * @file wikipedia/fetch_extract.cc
+ */
+
+#include <spdlog/spdlog.h>
+
+#include <boost/json.hpp>
+#include <chrono>
+#include <format>
+#include <string>
+#include <string_view>
+#include <thread>
+
+#include "services/enrichment/wikipedia_service.h"
+
+using namespace boost;
+
+std::string WikipediaEnrichmentService::FetchExtract(std::string_view query) {
+
+  const std::string cache_key(query);
+
+  // 1. Cache Lookup
+  if (const auto cache_it = this->extract_cache_.find(cache_key);
+      cache_it != this->extract_cache_.end()) {
+    spdlog::debug("Wikipedia: Cache hit for {}!", cache_key);
+    return cache_it->second;
+  }
+
+  const std::string encoded = this->client_->EncodeURL(cache_key);
+  const std::string url = std::format(
+      "https://en.wikipedia.org/w/"
+      "api.php?action=query&titles={}&prop=extracts&explaintext=1&format=json",
+      encoded);
+
+
+  const std::string body = this->client_->Get(url);
+  {
+    using namespace std::literals::chrono_literals;
+    std::this_thread::sleep_for(1s);
+  }
+
+  // 2. Parse JSON
+  system::error_code ec;
+  json::value doc = json::parse(body, ec);
+
+  if (ec) {
+    spdlog::warn("WikipediaService: JSON parse error for '{}': {}", query,
+                 ec.message());
+    return {};
+  }
+
+  // 3. Safe Extraction
+  const json::object* obj = doc.if_object();
+  if (obj == nullptr) {
+    spdlog::warn("WikipediaService: Expected root object for '{}'", query);
+    return {};
+  }
+
+  const json::value* query_ptr = obj->if_contains("query");
+  const json::value* pages_ptr =
+      ((query_ptr != nullptr) && query_ptr->is_object())
+          ? query_ptr->get_object().if_contains("pages")
+          : nullptr;
+
+  if ((pages_ptr == nullptr) || !pages_ptr->is_object()) {
+    spdlog::warn("WikipediaService: Missing query.pages for '{}'", query);
+    return {};
+  }
+
+  const json::object& pages = pages_ptr->get_object();
+
+  if (pages.empty()) {
+    spdlog::warn("WikipediaService: No pages returned for '{}'", query);
+    this->extract_cache_.emplace(cache_key, "");
+    return {};
+  }
+
+  // Wikipedia returns the page under a dynamic ID key; we just want the first
+  // one
+  const json::value& page_val = pages.begin()->value();
+
+  if (!page_val.is_object()) {
+    spdlog::warn("WikipediaService: Unexpected page format for '{}'", query);
+    return {};
+  }
+
+  const json::object& page = page_val.get_object();
+
+  // Handle 404/Missing status
+  if (page.contains("missing")) {
+    spdlog::warn("WikipediaService: Page '{}' does not exist", query);
+    this->extract_cache_.emplace(cache_key, "");
+    return {};
+  }
+
+  const json::value* extract_ptr = page.if_contains("extract");
+
+  if ((extract_ptr == nullptr) || !extract_ptr->is_string()) {
+    spdlog::warn("WikipediaService: No extract string found for '{}'", query);
+    this->extract_cache_.emplace(cache_key, "");
+    return {};
+  }
+
+  // 4. Success
+  std::string extract(extract_ptr->as_string());
+  spdlog::info("WikipediaService: Fetched {} chars for '{}'", extract.size(),
+                query);
+
+  this->extract_cache_.insert_or_assign(cache_key, extract);
+
+  return extract;
+}

tooling/pipeline/src/services/enrichment/wikipedia/get_summary.cc

@@ -0,0 +1,58 @@
+/**
+ * @file wikipedia/get_summary.cc
+ * @brief WikipediaService::GetLocationContext() implementation.
+ */
+
+#include <spdlog/spdlog.h>
+
+#include <chrono>
+#include <format>
+#include <string>
+#include <thread>
+
+#include "services/enrichment/wikipedia_service.h"
+
+std::string WikipediaEnrichmentService::GetLocationContext(const Location& loc) {
+  using namespace std::literals::chrono_literals;
+  if (!this->client_) {
+    spdlog::warn("Client is nullptr.");
+    return {};
+  }
+
+  std::string result;
+
+  // std::string region_query(loc.city);
+  // if (!loc.country.empty()) {
+  //   region_query += loc.state_province,
+  //   region_query += ", ";
+  //   region_query += loc.country;
+  // }
+
+  constexpr std::string_view brewing_query = "brewing";
+  const std::string location_query =
+      std::format("{}, {}", loc.city, loc.iso3166_2);
+  const std::string beer_query = std::format("beer in {}", loc.country);
+
+  auto append_extract = [&result](const std::string& extract) -> void {
+    if (extract.empty()) {
+      return;
+    }
+    if (!result.empty()) {
+      result += "\n\n";
+    }
+    result += extract;
+  };
+
+  try {
+    append_extract(FetchExtract(brewing_query));
+    append_extract(FetchExtract(beer_query));
+    spdlog::info("Done fetching for {}. Sleeping for 10 seconds.",
+                 location_query);
+    std::this_thread::sleep_for(10s);
+
+  } catch (const std::runtime_error& e) {
+    spdlog::debug("WikipediaService lookup failed for '{}': {}", location_query,
+                  e.what());
+  }
+  return result;
+}

tooling/pipeline/src/services/enrichment/wikipedia/wikipedia_service.cc

@@ -7,5 +7,6 @@
 
 #include <utility>
 
-WikipediaService::WikipediaService(std::unique_ptr<WebClient> client)
+WikipediaEnrichmentService::WikipediaEnrichmentService(
+    std::unique_ptr<WebClient> client)
     : client_(std::move(client)) {}

tooling/pipeline/src/services/wikipedia/fetch_extract.cc

@@ -1,61 +0,0 @@
-/**
- * @file wikipedia/fetch_extract.cc
- * @brief WikipediaService::FetchExtract() implementation.
- */
-
-#include <spdlog/spdlog.h>
-
-#include <boost/json.hpp>
-#include <string>
-#include <string_view>
-
-#include "services/enrichment/wikipedia_service.h"
-
-std::string WikipediaService::FetchExtract(std::string_view query) {
-  const std::string cache_key(query);
-  const auto cache_it = this->extract_cache_.find(cache_key);
-  if (cache_it != this->extract_cache_.end()) {
-    return cache_it->second;
-  }
-
-  const std::string encoded = this->client_->UrlEncode(cache_key);
-  const std::string url =
-      "https://en.wikipedia.org/w/api.php?action=query&titles=" + encoded +
-      "&prop=extracts&explaintext=1&format=json";
-
-  const std::string body = this->client_->Get(url);
-
-  boost::system::error_code parse_error;
-  boost::json::value doc = boost::json::parse(body, parse_error);
-
-  if (!parse_error && doc.is_object()) {
-    try {
-      auto& pages = doc.at("query").at("pages").get_object();
-      if (!pages.empty()) {
-        auto& page = pages.begin()->value().get_object();
-        if (page.contains("extract") && page.at("extract").is_string()) {
-          const std::string_view extract_view = page.at("extract").as_string();
-          std::string extract(extract_view);
-
-          spdlog::debug("WikipediaService fetched {} chars for '{}'",
-                        extract.size(), query);
-
-          this->extract_cache_.emplace(cache_key, extract);
-          return extract;
-        }
-      }
-      this->extract_cache_.emplace(cache_key, std::string{});
-    } catch (const std::exception& e) {
-      spdlog::warn(
-          "WikipediaService: failed to parse response structure for '{}': "
-          "{}",
-          query, e.what());
-      return {};
-    }
-  } else if (parse_error) {
-    spdlog::warn("WikipediaService: JSON parse error for '{}': {}", query,
-                 parse_error.message());
-  }
-
-  return {};
-}

tooling/pipeline/src/services/wikipedia/get_summary.cc

@@ -1,47 +0,0 @@
-/**
- * @file wikipedia/get_summary.cc
- * @brief WikipediaService::GetLocationContext() implementation.
- */
-
-#include <spdlog/spdlog.h>
-
-#include <string>
-
-#include "services/enrichment/wikipedia_service.h"
-
-std::string WikipediaService::GetLocationContext(const Location& loc) {
-  if (!client_) {
-    return {};
-  }
-
-  std::string result;
-
-  std::string region_query(loc.city);
-  if (!loc.country.empty()) {
-    region_query += ", ";
-    region_query += loc.country;
-  }
-
-  const std::string beer_query = "beer in " + loc.country;
-  const std::string city_beer_query = "beer in " + loc.city;
-
-  auto append_extract = [&result](const std::string& extract) -> void {
-    if (extract.empty()) {
-      return;
-    }
-    if (!result.empty()) {
-      result += "\n\n";
-    }
-    result += extract;
-  };
-
-  try {
-    append_extract(FetchExtract(region_query));
-    append_extract(FetchExtract(beer_query));
-    append_extract(FetchExtract(city_beer_query));
-  } catch (const std::runtime_error& e) {
-    spdlog::debug("WikipediaService lookup failed for '{}': {}", region_query,
-                  e.what());
-  }
-  return result;
-}