vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More requests and threads

Browse source
This commit is contained in:
pudepiedj 2024-02-21 11:39:48 +00:00
parent 760b6d639b
commit f7e29e5248
2 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Llamaserver.py
View file

@ -85,8 +85,8 @@ if __name__ == "__main__":
url = "http://localhost:8080/completion" url = "http://localhost:8080/completion"
num_requests = 40 num_requests = 256
q = Queue(maxsize = 40) q = Queue(maxsize = 256)
threads = [] threads = []
bar = make_empty_bar(num_requests) bar = make_empty_bar(num_requests)

6
examples/server/httplib.h
View file

@ -93,10 +93,10 @@
#ifndef CPPHTTPLIB_COMPRESSION_BUFSIZ #ifndef CPPHTTPLIB_COMPRESSION_BUFSIZ
#define CPPHTTPLIB_COMPRESSION_BUFSIZ size_t(16384u) #define CPPHTTPLIB_COMPRESSION_BUFSIZ size_t(16384u)
#endif #endif
// the value here (8u, 16u, 32u, etc) is what governs max threads at 5126
#ifndef CPPHTTPLIB_THREAD_POOL_COUNT #ifndef CPPHTTPLIB_THREAD_POOL_COUNT
#define CPPHTTPLIB_THREAD_POOL_COUNT \ #define CPPHTTPLIB_THREAD_POOL_COUNT \
((std::max)(32u, std::thread::hardware_concurrency() > 0 \ ((std::max)(128u, std::thread::hardware_concurrency() > 0 \
? std::thread::hardware_concurrency() - 1 \ ? std::thread::hardware_concurrency() - 1 \
: 0)) : 0))
#endif #endif
@ -581,7 +581,7 @@ public:
class ThreadPool : public TaskQueue { class ThreadPool : public TaskQueue {
public: public:
explicit ThreadPool(size_t n) : shutdown_(false) { explicit ThreadPool(size_t n) : shutdown_(false) { // n is CPPHTTPLIB_THREAD_POOL_COUNT
while (n) { while (n) {
threads_.emplace_back(worker(*this)); threads_.emplace_back(worker(*this));
n--; n--;