Pipeline: Remove CURL as a dependency, add new HTTP module (#219)

Rationale: 

HTTP is a supporting concern in the pipeline, used only for Wikipedia enrichment calls. libcurl's C API required significant boilerplate to wrap safely. cpp-httplib is a header-only library that covers the same functionality with far less overhead and no manual resource management.

tooling/pipeline/src/main.cc

@@ -8,12 +8,11 @@
 
 #include <boost/di.hpp>
 #include <boost/program_options.hpp>
-#include <chrono>
-#include <cstdint>
+
 #include <exception>
 #include <memory>
 #include <optional>
-#include <sstream>
+
 #include <string>
 
 #include "biergarten_data_generator.h"
@@ -29,16 +28,22 @@
 #include "services/timer.h"
 #include "services/wikipedia_service.h"
 #include "web_client/curl_web_client.h"
+#include "web_client/http_web_client.h"
 
 namespace di = boost::di;
 
 int main(const int argc, char** argv) {
   try {
     Timer timer;
-    const CurlGlobalState curl_state;
-    const LlamaBackendState llama_backend_state;
     spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%^%l%$] %v");
 
+#ifndef BIERGARTEN_MOCK_ONLY
+    const LlamaBackendState llama_backend_state;
+#endif
+#ifdef DEBUG
+    spdlog::set_level(spdlog::level::debug);
+#endif
+
     const auto parsed_options = ParseArguments(argc, argv);
     if (!parsed_options.has_value()) {
       return 0;
@@ -61,7 +66,7 @@ int main(const int argc, char** argv) {
     }
 
     const auto injector = di::make_injector(
-        di::bind<WebClient>().to<CURLWebClient>(),
+        di::bind<WebClient>().to<HttpWebClient>(),
         di::bind<ApplicationOptions>().to(options),
         di::bind<IEnrichmentService>().to<WikipediaService>(),
         di::bind<IExportService>().to<SqliteExportService>(),
@@ -69,7 +74,7 @@ int main(const int argc, char** argv) {
         di::bind<std::string>().to(model_path),
         di::bind<DataGenerator>().to(
             [options, model_path, sampling, &prompt_directory](
-                const auto& inj) -> std::unique_ptr<DataGenerator> {
+            const auto& inj) -> std::unique_ptr<DataGenerator> {
               if (options.generator.use_mocked) {
                 spdlog::info(
                     "[Generator] Using MockGenerator (no model path provided)");
@@ -101,4 +106,4 @@ int main(const int argc, char** argv) {
     spdlog::critical("Unhandled fatal error in main: {}", exception.what());
     return 1;
   }
-}
+}

tooling/pipeline/src/web_client/curl_global_state.cc

@@ -1,19 +0,0 @@
-/**
- * @file web_client/curl_global_state.cc
- * @brief CurlGlobalState constructor and destructor implementation.
- */
-
-#include <curl/curl.h>
-
-#include <stdexcept>
-
-#include "web_client/curl_web_client.h"
-
-CurlGlobalState::CurlGlobalState() {
-  if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK) {
-    throw std::runtime_error(
-        "[CURLWebClient] Failed to initialize libcurl globally");
-  }
-}
-
-CurlGlobalState::~CurlGlobalState() { curl_global_cleanup(); }

tooling/pipeline/src/web_client/curl_web_client_get.cc

@@ -1,87 +0,0 @@
-/**
- * @file web_client/curl_web_client_get.cc
- * @brief CURLWebClient::Get() implementation.
- */
-
-#include <curl/curl.h>
-
-#include <cstdint>
-#include <limits>
-#include <memory>
-#include <stdexcept>
-#include <string>
-
-#include "web_client/curl_web_client.h"
-
-using CurlHandle = std::unique_ptr<CURL, decltype(&curl_easy_cleanup)>;
-
-static constexpr long kConnectionTimeout = 10;
-static constexpr long kRequestTimeout = 30;
-static constexpr long kMaxRedirects = 5;
-static constexpr int32_t kOkHttpStatus = 200;
-
-static CurlHandle CreateHandle() {
-  CURL* handle = curl_easy_init();
-  if (handle == nullptr) {
-    throw std::runtime_error(
-        "[CURLWebClient] Failed to initialize libcurl handle");
-  }
-  return {handle, &curl_easy_cleanup};
-}
-
-static void SetCommonGetOptions(CURL* curl, const std::string& url) {
-  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-  curl_easy_setopt(curl, CURLOPT_USERAGENT, "biergarten-pipeline/0.1.0");
-  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
-  curl_easy_setopt(curl, CURLOPT_MAXREDIRS, kMaxRedirects);
-  curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, kConnectionTimeout);
-  curl_easy_setopt(curl, CURLOPT_TIMEOUT, kRequestTimeout);
-  curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
-}
-
-// curl write callback that appends response data into a std::string
-static size_t WriteCallbackString(void* contents, const size_t size,
-                                  const size_t nmemb, void* userp) {
-  const size_t real_size = size * nmemb;
-  auto* str = static_cast<std::string*>(userp);
-  str->append(static_cast<char*>(contents), real_size);
-  return real_size;
-}
-
-std::string CURLWebClient::Get(const std::string& url) {
-  const CurlHandle curl = CreateHandle();
-
-  std::string response_string;
-
-  SetCommonGetOptions(curl.get(), url);
-
-  curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, WriteCallbackString);
-  curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response_string);
-
-  CURLcode curl_result = curl_easy_perform(curl.get());
-
-  if (curl_result != CURLE_OK) {
-    const auto error = std::string("[CURLWebClient] GET failed: ") +
-                       curl_easy_strerror(curl_result);
-    throw std::runtime_error(error);
-  }
-
-  long curl_http_code = 0;
-  curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &curl_http_code);
-
-  if (curl_http_code < std::numeric_limits<int32_t>::min() ||
-      curl_http_code > std::numeric_limits<int32_t>::max()) {
-    throw std::runtime_error("[CURLWebClient] Invalid HTTP status code: " +
-                             std::to_string(curl_http_code));
-  }
-
-  const int32_t http_code = static_cast<int32_t>(curl_http_code);
-
-  if (http_code != kOkHttpStatus) {
-    const std::string error = "[CURLWebClient] HTTP error " +
-                              std::to_string(http_code) + " for URL " + url;
-    throw std::runtime_error(error);
-  }
-
-  return response_string;
-}

tooling/pipeline/src/web_client/curl_web_client_url_encode.cc

@@ -1,24 +0,0 @@
-/**
- * @file web_client/curl_web_client_url_encode.cc
- * @brief CURLWebClient::UrlEncode() implementation.
- */
-
-#include <curl/curl.h>
-
-#include <stdexcept>
-#include <string>
-
-#include "web_client/curl_web_client.h"
-
-std::string CURLWebClient::UrlEncode(const std::string& value) {
-  // A NULL handle is fine for UTF-8 encoding according to libcurl docs.
-  char* output = curl_easy_escape(nullptr, value.c_str(), 0);
-
-  if (!output) {
-    throw std::runtime_error("[CURLWebClient] curl_easy_escape failed");
-  }
-
-  std::string result(output);
-  curl_free(output);
-  return result;
-}

tooling/pipeline/src/web_client/http_web_client.cc

@@ -0,0 +1,61 @@
+/**
+* @file web_client/http_web_client.cc
+ * @brief cpp-httplib implementation of WebClient.
+ */
+
+#include "web_client/http_web_client.h"
+
+#include <httplib.h>
+
+#include <regex>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+namespace {
+constexpr time_t kConnectionTimeoutSeconds = 5;
+constexpr time_t kReadTimeoutSeconds = 10;
+
+constexpr int kSuccessMin = 200;
+constexpr int kSuccessMax = 300;
+const std::regex kUrlRegex(
+    R"(^(https?://[^/?#]+)(/[^?#]*(?:\?[^#]*)?(?:#.*)?)?)");
+
+std::pair<std::string, std::string> SplitUrl(const std::string& url) {
+  std::smatch match;
+  if (!std::regex_match(url, match, kUrlRegex)) {
+    throw std::invalid_argument("[HttpWebClient] Malformed URL: " + url);
+  }
+
+  return {match[1].str(), match[2].matched ? match[2].str() : "/"};
+}
+} // namespace
+
+std::string HttpWebClient::Get(const std::string& url) {
+  const auto [origin, path] = SplitUrl(url);
+
+  httplib::Client client(origin);
+  client.set_follow_location(true);
+  client.set_connection_timeout(kConnectionTimeoutSeconds);
+  client.set_read_timeout(kReadTimeoutSeconds);
+
+  const auto result = client.Get(path);
+
+  if (!result) {
+    throw std::runtime_error(
+        "[HttpWebClient] Request failed for URL: " + url +
+        " — " + httplib::to_string(result.error()));
+  }
+
+  if (result->status < kSuccessMin || result->status >= kSuccessMax) {
+    throw std::runtime_error(
+        "[HttpWebClient] HTTP " + std::to_string(result->status) +
+        " for URL: " + url);
+  }
+
+  return result->body;
+}
+
+std::string HttpWebClient::UrlEncode(const std::string& value) {
+  return httplib::encode_uri_component(value);
+}