AaronPo/the-biergarten-app
1
0
Fork 0
mirror of https://github.com/aaronpo97/the-biergarten-app.git synced 2026-07-16 09:37:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pipeline: Add Runpod docker configuration (#222)

Browse Source 
* Begin work on Runpod docker config

* Reduce docker image size

* Create .dockerignore
This commit is contained in:
Aaron Po
2026-05-12 00:44:09 -04:00
committed by GitHub
parent 26635ace84
commit b8ebe03921
17 changed files with 425 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tooling/pipeline/src/data_generation/llama/load.cc
View File

@@ -12,6 +12,7 @@
#include <utility>
#include "data_generation/llama_generator.h"
#include "ggml-backend.h"
#include "llama.h"
// Maximum batch size for decode operations. Capping the batch prevents
@@ -22,7 +23,12 @@ void LlamaGenerator::Load(const std::string& model_path) {
context_.reset();
model_.reset();
const llama_model_params model_params = llama_model_default_params();
// Specifically load dynamic ggml backends (like CUDA) that are provided
// externally before attempting to load a model.
ggml_backend_load_all();
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = n_gpu_layers_;
LlamaGenerator::ModelHandle loaded_model(
llama_model_load_from_file(model_path.c_str(), model_params));
if (!loaded_model) {