From a7b816ecdfb9af466ddc2d447f4ecc6c1bfb8bd2 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 20 Nov 2023 11:54:23 -0500 Subject: [PATCH] improving on logging --- ggml-cuda.cu | 6 ++-- ggml.c | 12 ++++++++ grammars/antlr4.gbnf | 66 ++++++++++++++++++++++++++++++++++++++++++++ grammars/python.gbnf | 59 +++++++++++++++++++++++++++++++++++++++ llama.cpp | 4 +-- 5 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 grammars/antlr4.gbnf create mode 100644 grammars/python.gbnf diff --git a/ggml-cuda.cu b/ggml-cuda.cu index b00ee5a4d..65a6890d8 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -7593,10 +7593,12 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1 #endif // debug helpers - printf("JSON { \"data\":{ \"src0\": { \"%s\" :{ \"ne\" : [ %8d, %8d, %8d, %8d ], \"nb\" : [ %8d, %8d, %8d, %8d ], \"contiguous\":\"%d\", \"transposed\":\"%d\", \"type\": \"%s\", \"name\" : \"%s\"}}, \"src1\": { \"%s\" : { \"ne\" : [ %8d, %8d, %8d, %8d ], \"nb\" : [ %8d, %8d, %8d, %8d ], \"contiguous\":\"%d\", \"transposed\":\"%d\", \"type\": \"%s\", \"name\" : \"%s\"}}}}\n", + printf("JSON { \"data\":{ \"src0\": { \"%s\" :{ \"ne\" : [ %8d, %8d, %8d, %8d ], \"nb\" : [ %8d, %8d, %8d, %8d ], \"contiguous\":\"%d\", \"transposed\":\"%d\", \"type\": \"%s\", \"name\" : \"%s\"}}, \"src1\": { \"%s\" :{ \"ne\" : [ %8d, %8d, %8d, %8d ], \"nb\" : [ %8d, %8d, %8d, %8d ], \"contiguous\":\"%d\", \"transposed\":\"%d\", \"type\": \"%s\", \"name\" : \"%s\"}}, \"dst\" : { \"%s\" :{ \"ne\" : [ %8d, %8d, %8d, %8d ], \"nb\" : [ %8d, %8d, %8d, %8d ], \"contiguous\":\"%d\", \"transposed\":\"%d\", \"type\": \"%s\", \"name\" : \"%s\"}}}}\n", src0->name, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name, - src1->name, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name); + src1->name, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name, + dst->name, dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], ggml_is_contiguous(dst), ggml_is_transposed(dst), ggml_type_name(dst->type), dst->name + ); if (!split && all_on_device && !use_tensor_cores && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { // KQ single-batch diff --git a/ggml.c b/ggml.c index 25a7ed7dd..c6cadbc0c 100644 --- a/ggml.c +++ b/ggml.c @@ -13651,7 +13651,19 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm if (tensor->op == GGML_OP_NONE) { return; } + if (tensor->src[1]) { + fprintf(stderr, "JSON: { \"bop\":%d, \"src\":\"%s\",\"src2\":\"%s\",\"dst\":\"%s\"}\n", + tensor->op, + tensor->src[0]->name, + tensor->src[1]->name, + tensor->name); + } else { + fprintf(stderr, "JSON: { \"uop\":%d, \"src\":\"%s\",\"dst\":\"%s\"}\n", + tensor->op, + tensor->src[0]->name, + tensor->name); + } #ifdef GGML_USE_CUBLAS bool skip_cpu = ggml_cuda_compute_forward(params, tensor); if (skip_cpu) { diff --git a/grammars/antlr4.gbnf b/grammars/antlr4.gbnf new file mode 100644 index 000000000..b92a1de4b --- /dev/null +++ b/grammars/antlr4.gbnf @@ -0,0 +1,66 @@ +root ::= grammarSpec +grammarSpec ::= grammarDecl prequelConstruct* rules modeSpec* EOF +grammarDecl ::= grammarType identifier SEMI +grammarType ::= (LEXER GRAMMAR) | (PARSER GRAMMAR) | GRAMMAR +prequelConstruct ::= optionsSpec | delegateGrammars | tokensSpec | channelsSpec | action_ +optionsSpec ::= OPTIONS (option SEMI)* RBRACE +option ::= identifier ASSIGN optionValue +optionValue ::= (identifier (DOT identifier)*) | STRING_LITERAL | actionBlock | INT +delegateGrammars ::= IMPORT delegateGrammar (COMMA delegateGrammar)* SEMI +delegateGrammar ::= (identifier ASSIGN identifier) | identifier +tokensSpec ::= TOKENS idList? RBRACE +channelsSpec ::= CHANNELS idList? RBRACE +idList ::= identifier (COMMA identifier)* COMMA? +action_ ::= AT (actionScopeName COLONCOLON)? identifier actionBlock +actionScopeName ::= identifier | LEXER | PARSER +actionBlock ::= BEGIN_ACTION ACTION_CONTENT* END_ACTION +argActionBlock ::= BEGIN_ARGUMENT ARGUMENT_CONTENT* END_ARGUMENT +modeSpec ::= MODE identifier SEMI lexerRuleSpec* +rules ::= ruleSpec* +ruleSpec ::= parserRuleSpec | lexerRuleSpec +parserRuleSpec ::= ruleModifiers? RULE_REF argActionBlock? ruleReturns? throwsSpec? localsSpec? rulePrequel* COLON ruleBlock SEMI exceptionGroup +exceptionGroup ::= exceptionHandler* finallyClause? +exceptionHandler ::= CATCH argActionBlock actionBlock +finallyClause ::= FINALLY actionBlock +rulePrequel ::= optionsSpec | ruleAction +ruleReturns ::= RETURNS argActionBlock +throwsSpec ::= THROWS identifier (COMMA identifier)* +localsSpec ::= LOCALS argActionBlock +ruleAction ::= AT identifier actionBlock +ruleModifiers ::= ruleModifier+ +ruleModifier ::= PUBLIC | PRIVATE | PROTECTED | FRAGMENT +ruleBlock ::= ruleAltList +ruleAltList ::= labeledAlt (OR labeledAlt)* +labeledAlt ::= alternative (POUND identifier)? +lexerRuleSpec ::= FRAGMENT? TOKEN_REF optionsSpec? COLON lexerRuleBlock SEMI +lexerRuleBlock ::= lexerAltList +lexerAltList ::= lexerAlt (OR lexerAlt)* +lexerAlt ::= (lexerElements lexerCommands?) | () +lexerElements ::= lexerElement+ | () +lexerElement ::= (lexerAtom ebnfSuffix?) | (lexerBlock ebnfSuffix?) | (actionBlock QUESTION?) +lexerBlock ::= LPAREN lexerAltList RPAREN +lexerCommands ::= RARROW lexerCommand (COMMA lexerCommand)* +lexerCommand ::= (lexerCommandName LPAREN lexerCommandExpr RPAREN) | lexerCommandName +lexerCommandName ::= identifier | MODE +lexerCommandExpr ::= identifier | INT +altList ::= alternative (OR alternative)* +alternative ::= (elementOptions? element+) | () +element ::= (labeledElement (ebnfSuffix | ())) | (atom (ebnfSuffix | ())) | ebnf | (actionBlock (QUESTION predicateOptions?)?) +predicateOptions ::= LT predicateOption (COMMA predicateOption)* GT +predicateOption ::= elementOption | (identifier ASSIGN actionBlock) +labeledElement ::= identifier (ASSIGN | PLUS_ASSIGN) (atom | block) +ebnf ::= block blockSuffix? +blockSuffix ::= ebnfSuffix +ebnfSuffix ::= (QUESTION QUESTION?) | (STAR QUESTION?) | (PLUS QUESTION?) +lexerAtom ::= characterRange | terminalDef | notSet | LEXER_CHAR_SET | (DOT elementOptions?) +atom ::= terminalDef | ruleref | notSet | (DOT elementOptions?) +notSet ::= (NOT setElement) | (NOT blockSet) +blockSet ::= LPAREN setElement (OR setElement)* RPAREN +setElement ::= (TOKEN_REF elementOptions?) | (STRING_LITERAL elementOptions?) | characterRange | LEXER_CHAR_SET +block ::= LPAREN (optionsSpec? ruleAction* COLON)? altList RPAREN +ruleref ::= RULE_REF argActionBlock? elementOptions? +characterRange ::= STRING_LITERAL RANGE STRING_LITERAL +terminalDef ::= (TOKEN_REF elementOptions?) | (STRING_LITERAL elementOptions?) +elementOptions ::= LT elementOption (COMMA elementOption)* GT +elementOption ::= identifier | (identifier ASSIGN (identifier | STRING_LITERAL)) +identifier ::= RULE_REF | TOKEN_REF diff --git a/grammars/python.gbnf b/grammars/python.gbnf new file mode 100644 index 000000000..160ba4be1 --- /dev/null +++ b/grammars/python.gbnf @@ -0,0 +1,59 @@ +root ::= (single_input | file_input | eval_input)? EOF +single_input ::= LINE_BREAK | simple_stmt | (compound_stmt LINE_BREAK) +file_input ::= (LINE_BREAK | stmt)+ +eval_input ::= testlist LINE_BREAK* +stmt ::= simple_stmt | compound_stmt +compound_stmt ::= (IF test COLON suite elif_clause* else_clause?) | (WHILE test COLON suite else_clause?) | (ASYNC? FOR exprlist IN testlist COLON suite else_clause?) | (TRY COLON suite ((except_clause+ else_clause? finally_clause?) | finally_clause)) | (ASYNC? WITH with_item (COMMA with_item)* COLON suite) | (decorator* (classdef | funcdef)) +suite ::= simple_stmt | (LINE_BREAK INDENT stmt+ DEDENT) +decorator ::= AT dotted_name (OPEN_PAREN arglist? CLOSE_PAREN)? LINE_BREAK +elif_clause ::= ELIF test COLON suite +else_clause ::= ELSE COLON suite +finally_clause ::= FINALLY COLON suite +with_item ::= test (AS expr)? +except_clause ::= EXCEPT (test ((COMMA name) | (AS name))?)? COLON suite +classdef ::= CLASS name (OPEN_PAREN arglist? CLOSE_PAREN)? COLON suite +funcdef ::= ASYNC? DEF name OPEN_PAREN typedargslist? CLOSE_PAREN (ARROW test)? COLON suite +typedargslist ::= ((def_parameters COMMA)? ((args (COMMA def_parameters)? (COMMA kwargs)?) | kwargs) COMMA?) | (def_parameters COMMA?) +args ::= STAR named_parameter +kwargs ::= POWER named_parameter +def_parameters ::= def_parameter (COMMA def_parameter)* +def_parameter ::= (named_parameter (ASSIGN test)?) | STAR +named_parameter ::= name (COLON test)? +simple_stmt ::= small_stmt (SEMI_COLON small_stmt)* SEMI_COLON? (LINE_BREAK | EOF) +small_stmt ::= (testlist_star_expr assign_part?) | (PRINT ((test (COMMA test)* COMMA?) | (RIGHT_SHIFT test ((COMMA test)+ COMMA?)))) | (DEL exprlist) | PASS | BREAK | CONTINUE | (RETURN testlist?) | (RAISE (test (COMMA test (COMMA test)?)?)? (FROM test)?) | yield_expr | (IMPORT dotted_as_names) | (FROM (((DOT | ELLIPSIS)* dotted_name) | (DOT | ELLIPSIS)+) IMPORT (STAR | (OPEN_PAREN import_as_names CLOSE_PAREN) | import_as_names)) | (GLOBAL name (COMMA name)*) | (EXEC expr (IN test (COMMA test)?)?) | (ASSERT test (COMMA test)?) | (NONLOCAL name (COMMA name)*) +testlist_star_expr ::= (((test | star_expr) COMMA)+ (test | star_expr)?) | testlist +star_expr ::= STAR expr +assign_part ::= (ASSIGN ((testlist_star_expr (ASSIGN testlist_star_expr)* (ASSIGN yield_expr)?) | yield_expr)) | (COLON test (ASSIGN testlist)?) | ((ADD_ASSIGN | SUB_ASSIGN | MULT_ASSIGN | AT_ASSIGN | DIV_ASSIGN | MOD_ASSIGN | AND_ASSIGN | OR_ASSIGN | XOR_ASSIGN | LEFT_SHIFT_ASSIGN | RIGHT_SHIFT_ASSIGN | POWER_ASSIGN | IDIV_ASSIGN) (yield_expr | testlist)) +exprlist ::= expr (COMMA expr)* COMMA? +import_as_names ::= import_as_name (COMMA import_as_name)* COMMA? +import_as_name ::= name (AS name)? +dotted_as_names ::= dotted_as_name (COMMA dotted_as_name)* +dotted_as_name ::= dotted_name (AS name)? +test ::= (logical_test (IF logical_test ELSE test)?) | (LAMBDA varargslist? COLON test) +varargslist ::= ((vardef_parameters COMMA)? ((varargs (COMMA vardef_parameters)? (COMMA varkwargs)?) | varkwargs) COMMA?) | (vardef_parameters COMMA?) +vardef_parameters ::= vardef_parameter (COMMA vardef_parameter)* +vardef_parameter ::= (name (ASSIGN test)?) | STAR +varargs ::= STAR name +varkwargs ::= POWER name +logical_test ::= comparison | (NOT logical_test) | (logical_test AND logical_test) | (logical_test OR logical_test) +comparison ::= (comparison (LESS_THAN | GREATER_THAN | EQUALS | GT_EQ | LT_EQ | NOT_EQ_1 | NOT_EQ_2 | (NOT? IN) | (IS NOT?)) comparison) | expr +expr ::= (AWAIT? atom trailer*) | (expr POWER expr) | ((ADD | MINUS | NOT_OP) expr) | (expr (STAR | DIV | MOD | IDIV | AT) expr) | (expr (ADD | MINUS) expr) | (expr (LEFT_SHIFT | RIGHT_SHIFT) expr) | (expr AND_OP expr) | (expr XOR expr) | (expr OR_OP expr) +atom ::= (OPEN_PAREN (yield_expr | testlist_comp)? CLOSE_PAREN) | (OPEN_BRACKET testlist_comp? CLOSE_BRACKET) | (OPEN_BRACE dictorsetmaker? CLOSE_BRACE) | (REVERSE_QUOTE testlist COMMA? REVERSE_QUOTE) | ELLIPSIS | name | PRINT | EXEC | (MINUS? number) | NONE | STRING+ +dictorsetmaker ::= (((test COLON test) | (POWER expr)) (COMMA ((test COLON test) | (POWER expr)))* COMMA?) | (test COLON test comp_for) | testlist_comp +testlist_comp ::= (test | star_expr) (comp_for | ((COMMA (test | star_expr))* COMMA?)) +testlist ::= test (COMMA test)* COMMA? +dotted_name ::= (dotted_name DOT name) | name +name ::= NAME | TRUE | FALSE +number ::= integer | IMAG_NUMBER | FLOAT_NUMBER +integer ::= DECIMAL_INTEGER | OCT_INTEGER | HEX_INTEGER | BIN_INTEGER +yield_expr ::= YIELD yield_arg? +yield_arg ::= (FROM test) | testlist +trailer ::= (DOT name arguments?) | arguments +arguments ::= (OPEN_PAREN arglist? CLOSE_PAREN) | (OPEN_BRACKET subscriptlist CLOSE_BRACKET) +arglist ::= argument (COMMA argument)* COMMA? +argument ::= (test (comp_for | (ASSIGN test))?) | ((POWER | STAR) test) +subscriptlist ::= subscript (COMMA subscript)* COMMA? +subscript ::= ELLIPSIS | (test (COLON test? sliceop?)?) | (COLON test? sliceop?) +sliceop ::= COLON test? +comp_for ::= FOR exprlist IN logical_test comp_iter? +comp_iter ::= comp_for | (IF test comp_iter?) diff --git a/llama.cpp b/llama.cpp index 3a781c494..7a24be6a6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5291,8 +5291,8 @@ static int llama_decode_internal( // plot the computation graph in dot format (for debugging purposes) //if (n_past%100 == 0) { - // ggml_graph_dump_dot(gf, NULL, "llama.dot"); - //} + //ggml_graph_dump_dot(gf, NULL, "llama.dot"); + //} // extract logits // TODO: do not compute and extract logits if only embeddings are needed