Sync from device back to host at begin of new prompt.

kompute/src/CMakeLists.txt

@@ -46,8 +46,8 @@ configure_package_config_file(${PROJECT_SOURCE_DIR}/cmake/komputeConfig.cmake.in
     "${PROJECT_BINARY_DIR}/kompute/komputeConfig.cmake"
     INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/kompute)
 
-install(FILES ${PROJECT_BINARY_DIR}/kompute/komputeConfig.cmake
-    ${PROJECT_BINARY_DIR}/kompute/komputeConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/kompute)
+#install(FILES ${PROJECT_BINARY_DIR}/kompute/komputeConfig.cmake
+#    ${PROJECT_BINARY_DIR}/kompute/komputeConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/kompute)
 
 # ####################################################
 # Linking

kompute/src/include/CMakeLists.txt

@@ -29,7 +29,7 @@ target_sources(kompute PRIVATE
     kompute/logger/Logger.hpp
 )
 
-install(DIRECTORY kompute DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+#install(DIRECTORY kompute DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 # ####################################################
 # Logger
@@ -43,4 +43,4 @@ target_sources(kp_logger PRIVATE
     kompute/logger/Logger.hpp
 )
 
-install(DIRECTORY logger DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+#install(DIRECTORY logger DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

llama.cpp

@@ -1889,6 +1889,10 @@ static bool llama_eval_internal(
         ggml_vk_graph_compute(lctx.ctx_kompute, gf);
         ggml_vk_d2h_tensor(lctx.ctx_kompute, res);
     } else {
+        if (lctx.ctx_kompute) {
+            ggml_vk_d2h_tensor(lctx.ctx_kompute, kv_self.k);
+            ggml_vk_d2h_tensor(lctx.ctx_kompute, kv_self.v);
+        }
         ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads);
         if (lctx.ctx_kompute) {
             ggml_vk_h2d_tensor(lctx.ctx_kompute, kv_self.k);