@FIR-1001 - GGML: Tsavorite Performance OPs data

Author: Anoop Kapoor

ggml/include/ggml.h

@@ -343,13 +343,13 @@ extern "C" {
     GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)
     GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);
 
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
     enum ggml_compute_backend_type {
         GGML_COMPUTE_BACKEND_CPU=0,
         GGML_COMPUTE_BACKEND_TSAVORITE,
         GGML_COMPUTE_BACKEND_COUNT
     };
-#endif /* GGML_PERF */
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
 
     enum ggml_status {
         GGML_STATUS_ALLOC_FAILED = -2,
@@ -659,14 +659,15 @@ extern "C" {
         char name[GGML_MAX_NAME];
 
         void * extra; // extra things e.g. for ggml-cuda.cu
-#ifdef GGML_PERF
+
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
         int64_t perf_runs;
         int64_t perf_time_us;
         enum ggml_compute_backend_type ggml_compute_backend;
         char padding[4];
 #else
         char padding[8];
-#endif /* GGML_PERF */
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
     };
 
     static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
@@ -2556,7 +2557,7 @@ extern "C" {
     GGML_API void                          ggml_threadpool_params_init   (struct ggml_threadpool_params * p, int n_threads);
     GGML_API bool                          ggml_threadpool_params_match  (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
 
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
 struct ggml_perf_backend_subtotals {
     int64_t total_us;
     int64_t runs;
@@ -2586,7 +2587,7 @@ void ggml_perf_write_detailed_csv(struct ggml_cgraph * cgraph, FILE *fp);
 void ggml_perf_accumulate(struct ggml_perf_totals totals[GGML_OP_COUNT], struct ggml_cgraph * cgraph);
 const char * ggml_backend_type(enum ggml_compute_backend_type backend);
 
-#endif /* GGML_PERF */
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
 
 #ifdef  __cplusplus
 }

ggml/src/ggml-tsavorite/ggml-tsavorite.cpp

@@ -929,9 +929,9 @@ static enum ggml_status ggml_tsavorite_graph_compute(ggml_backend_t backend,
 
   for (int i = 0; i < cgraph->n_nodes; i++) {
      int32_t kernel_sub_type=-1;
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
     int64_t t_start = ggml_time_us();
-#endif
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
     node = cgraph->nodes[i];
     src0 = node->src[0];
     src1 = node->src[1];
@@ -1279,7 +1279,7 @@ static enum ggml_status ggml_tsavorite_graph_compute(ggml_backend_t backend,
           device->stats.op_run_count[kernel_type].max_num_of_elem < max_num_of_elem)
         device->stats.op_run_count[kernel_type].max_num_of_elem = max_num_of_elem;
     }
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
     int64_t t_end = ggml_time_us();
     node->perf_runs++;
     node->ggml_compute_backend = GGML_COMPUTE_BACKEND_TSAVORITE;
@@ -1289,7 +1289,7 @@ static enum ggml_status ggml_tsavorite_graph_compute(ggml_backend_t backend,
         // Handle wraparound by assuming timer rolls over at max int64_t value
         node->perf_time_us += (INT64_MAX - t_start + t_end + 1);
     }
-#endif
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
   }
 
   // This this need to implement correctly when we have mixture of CPU and accelerator operation

ggml/src/ggml.c

@@ -1020,12 +1020,12 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
     "GLU",
 };
 
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
 static const char * GGML_BACKEND_TYPE[GGML_COMPUTE_BACKEND_COUNT] = {
     "CPU",
     "OPU"
 };
-#endif /* GGML_PERF */
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
 
 static_assert(GGML_OP_COUNT == 90, "GGML_OP_COUNT != 90");
 
@@ -1262,11 +1262,11 @@ const char * ggml_op_name(enum ggml_op op) {
     return GGML_OP_NAME[op];
 }
 
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
 const char * ggml_backend_type(enum ggml_compute_backend_type backend) {
     return GGML_BACKEND_TYPE[backend];
 }
-#endif /* GGML_PERF */
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
 
 const char * ggml_op_symbol(enum ggml_op op) {
     return GGML_OP_SYMBOL[op];
@@ -1692,11 +1692,11 @@ static struct ggml_tensor * ggml_new_tensor_impl(
         /*.data         =*/ obj_alloc_size > 0 ? (void *)(result + 1) : data,
         /*.name         =*/ { 0 },
         /*.extra        =*/ NULL,
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
         /*.perf_runs    =*/ 0,
         /*.perf_time_us =*/ 0,
         /*.ggml_compute_backend =*/ GGML_COMPUTE_BACKEND_CPU,
-#endif /* GGML_PERF */
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
         /*.padding      =*/ { 0 },
     };
 
@@ -7231,7 +7231,7 @@ bool ggml_threadpool_params_match(const struct ggml_threadpool_params * p0, cons
     return memcmp(p0->cpumask, p1->cpumask, GGML_MAX_N_THREADS) == 0;
 }
 
-#ifdef GGML_PERF
+#if defined(GGML_PERF) || defined(GGML_PERF_DETAIL)
 void ggml_perf_accumulate(struct ggml_perf_totals totals[GGML_OP_COUNT], struct ggml_cgraph * cgraph) {
     for (int i = 0; i < cgraph->n_nodes; ++i) {
         struct ggml_tensor * node = cgraph->nodes[i];
@@ -7258,7 +7258,9 @@ void ggml_perf_accumulate(struct ggml_perf_totals totals[GGML_OP_COUNT], struct
         }
     }
 }
+#endif /* GGML_PERF  || GGML_PERF_DETAIL */
 
+#if defined(GGML_PERF_DETAIL)
 FILE * ggml_perf_log_open(const char *filename) {
     // Try to delete existing file, ignore error if it doesn't exist
     remove(filename);
@@ -7326,4 +7328,4 @@ void ggml_perf_write_detailed_csv(struct ggml_cgraph * cgraph, FILE *fp) {
 
     fprintf(fp, "--------------------------------------------------------------------------------------------------------\n\n");
 }
-#endif /* GGML_PERF */
+#endif /* GGML_PERF_DETAIL */

src/llama-context.cpp

@@ -1090,6 +1090,18 @@ int llama_context::decode(const llama_batch & batch_inp) {
         ggml_status status;
         const auto * res = process_ubatch(ubatch, LLM_GRAPH_TYPE_DECODER, mctx.get(), status);
 
+#ifdef GGML_PERF
+        ggml_perf_accumulate(perf_totals, res->get_gf());
+#endif /* GGML_PERF  */
+
+#ifdef GGML_PERF_DETAIL
+        if (perf_all_shape_fp) {
+            ggml_perf_write_detailed_csv(res->get_gf(), perf_all_shape_fp);
+        }
+        ggml_perf_accumulate(perf_totals, res->get_gf());
+#endif /* GGML_PERF_DETAI */
+
+
         if (!res) {
             // the last ubatch failed or was aborted -> remove all positions of that ubatch from the memory module
             llama_pos pos_min[LLAMA_MAX_SEQ];