To: vim_dev@googlegroups.com Subject: Patch 8.2.2951 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 8.2.2951 Problem: Vim9: cannot use heredoc in :def function for :python, :lua, etc. Solution: Concatenate the heredoc lines and pass them in the ISN_EXEC_SPLIT instruction. Files: src/userfunc.c, src/vim9compile.c, src/vim9.h, src/vim9execute.c, src/testdir/test_vim9_func.vim, src/testdir/test_vim9_disassemble.vim *** ../vim-8.2.2950/src/userfunc.c 2021-05-24 15:45:25.626174865 +0200 --- src/userfunc.c 2021-06-06 16:09:02.822281560 +0200 *************** *** 631,638 **** --- 631,642 ---- char_u *skip_until = NULL; int ret = FAIL; int is_heredoc = FALSE; + int heredoc_concat_len = 0; + garray_T heredoc_ga; char_u *heredoc_trimmed = NULL; + ga_init2(&heredoc_ga, 1, 500); + // Detect having skipped over comment lines to find the return // type. Add NULL lines to keep the line count correct. sourcing_lnum_off = get_sourced_lnum(eap->getline, eap->cookie); *************** *** 733,738 **** --- 737,756 ---- getline_options = vim9_function ? GETLINE_CONCAT_CONTBAR : GETLINE_CONCAT_CONT; is_heredoc = FALSE; + + if (heredoc_concat_len > 0) + { + // Replace the starting line with all the concatenated + // lines. + ga_concat(&heredoc_ga, theline); + vim_free(((char_u **)(newlines->ga_data))[ + heredoc_concat_len - 1]); + ((char_u **)(newlines->ga_data))[ + heredoc_concat_len - 1] = heredoc_ga.ga_data; + ga_init(&heredoc_ga); + heredoc_concat_len = 0; + theline += STRLEN(theline); // skip the "EOF" + } } } } *************** *** 886,891 **** --- 904,911 ---- skip_until = vim_strnsave(p, skiptowhite(p) - p); getline_options = GETLINE_NONE; is_heredoc = TRUE; + if (eap->cmdidx == CMD_def) + heredoc_concat_len = newlines->ga_len + 1; } // Check for ":cmd v =<< [trim] EOF" *************** *** 928,937 **** if (ga_grow(newlines, 1 + sourcing_lnum_off) == FAIL) goto theend; ! // Copy the line to newly allocated memory. get_one_sourceline() ! // allocates 250 bytes per line, this saves 80% on average. The cost ! // is an extra alloc/free. ! p = vim_strsave(theline); if (p == NULL) goto theend; ((char_u **)(newlines->ga_data))[newlines->ga_len++] = p; --- 948,968 ---- if (ga_grow(newlines, 1 + sourcing_lnum_off) == FAIL) goto theend; ! if (heredoc_concat_len > 0) ! { ! // For a :def function "python << EOF" concatenats all the lines, ! // to be used for the instruction later. ! ga_concat(&heredoc_ga, theline); ! ga_concat(&heredoc_ga, (char_u *)"\n"); ! p = vim_strsave((char_u *)""); ! } ! else ! { ! // Copy the line to newly allocated memory. get_one_sourceline() ! // allocates 250 bytes per line, this saves 80% on average. The ! // cost is an extra alloc/free. ! p = vim_strsave(theline); ! } if (p == NULL) goto theend; ((char_u **)(newlines->ga_data))[newlines->ga_len++] = p; *************** *** 953,958 **** --- 984,990 ---- theend: vim_free(skip_until); vim_free(heredoc_trimmed); + vim_free(heredoc_ga.ga_data); need_wait_return |= saved_wait_return; return ret; } *************** *** 1436,1441 **** --- 1468,1474 ---- cc = name[*lenp]; name[*lenp] = NUL; + v = find_var(name, &ht, no_autoload); name[*lenp] = cc; if (v != NULL) *** ../vim-8.2.2950/src/vim9compile.c 2021-06-04 21:00:27.958234335 +0200 --- src/vim9compile.c 2021-06-06 16:49:32.619920364 +0200 *************** *** 8668,8673 **** --- 8668,8696 ---- return nextcmd; } + /* + * A script command with heredoc, e.g. + * ruby << EOF + * command + * EOF + * Has been turned into one long line with NL characters by + * get_function_body(): + * ruby << EOF commandEOF + */ + static char_u * + compile_script(char_u *line, cctx_T *cctx) + { + if (cctx->ctx_skip != SKIP_YES) + { + isn_T *isn; + + if ((isn = generate_instr(cctx, ISN_EXEC_SPLIT)) == NULL) + return NULL; + isn->isn_arg.string = vim_strsave(line); + } + return (char_u *)""; + } + /* * :s/pat/repl/ *************** *** 9480,9497 **** line = (char_u *)""; break; ! default: ! if (cctx.ctx_skip == SKIP_YES) ! { ! // We don't check for a next command here. ! line = (char_u *)""; ! } ! else ! { ! // Not recognized, execute with do_cmdline_cmd(). ! ea.arg = p; line = compile_exec(line, &ea, &cctx); ! } break; } nextline: --- 9503,9530 ---- line = (char_u *)""; break; ! case CMD_lua: ! case CMD_mzscheme: ! case CMD_perl: ! case CMD_py3: ! case CMD_python3: ! case CMD_python: ! case CMD_pythonx: ! case CMD_ruby: ! case CMD_tcl: ! ea.arg = p; ! if (vim_strchr(line, '\n') == NULL) line = compile_exec(line, &ea, &cctx); ! else ! // heredoc lines have been concatenated with NL ! // characters in get_function_body() ! line = compile_script(line, &cctx); ! break; ! ! default: ! // Not recognized, execute with do_cmdline_cmd(). ! ea.arg = p; ! line = compile_exec(line, &ea, &cctx); break; } nextline: *************** *** 9674,9679 **** --- 9707,9713 ---- { case ISN_DEF: case ISN_EXEC: + case ISN_EXEC_SPLIT: case ISN_LEGACY_EVAL: case ISN_LOADAUTO: case ISN_LOADB: *** ../vim-8.2.2950/src/vim9.h 2021-06-04 21:00:27.958234335 +0200 --- src/vim9.h 2021-06-06 15:54:13.272658509 +0200 *************** *** 14,19 **** --- 14,20 ---- typedef enum { ISN_EXEC, // execute Ex command line isn_arg.string ISN_EXECCONCAT, // execute Ex command from isn_arg.number items on stack + ISN_EXEC_SPLIT, // execute Ex command from isn_arg.string split at NL ISN_LEGACY_EVAL, // evaluate expression isn_arg.string with legacy syntax. ISN_ECHO, // echo isn_arg.echo.echo_count items on top of stack ISN_EXECUTE, // execute Ex commands isn_arg.number items on top of stack *** ../vim-8.2.2950/src/vim9execute.c 2021-06-05 21:36:16.390016558 +0200 --- src/vim9execute.c 2021-06-06 16:15:57.085144404 +0200 *************** *** 1214,1219 **** --- 1214,1250 ---- } /* + * Function passed to do_cmdline() for splitting a script joined by NL + * characters. + */ + static char_u * + get_split_sourceline( + int c UNUSED, + void *cookie, + int indent UNUSED, + getline_opt_T options UNUSED) + { + source_cookie_T *sp = (source_cookie_T *)cookie; + char_u *p; + char_u *line; + + if (*sp->nextline == NUL) + return NULL; + p = vim_strchr(sp->nextline, '\n'); + if (p == NULL) + { + line = vim_strsave(sp->nextline); + sp->nextline += STRLEN(sp->nextline); + } + else + { + line = vim_strnsave(sp->nextline, p - sp->nextline); + sp->nextline = p + 1; + } + return line; + } + + /* * Execute a function by "name". * This can be a builtin function, user function or a funcref. * "iptr" can be used to replace the instruction with a more efficient one. *************** *** 1425,1430 **** --- 1456,1479 ---- } break; + // execute Ex command line split at NL characters. + case ISN_EXEC_SPLIT: + { + source_cookie_T cookie; + + SOURCING_LNUM = iptr->isn_lnum; + CLEAR_FIELD(cookie); + cookie.sourcing_lnum = iptr->isn_lnum - 1; + cookie.nextline = iptr->isn_arg.string; + if (do_cmdline(get_split_sourceline(0, &cookie, 0, 0), + get_split_sourceline, &cookie, + DOCMD_VERBOSE|DOCMD_NOWAIT|DOCMD_KEYTYPED) + == FAIL + || did_emsg) + goto on_error; + } + break; + // Evaluate an expression with legacy syntax, push it onto the // stack. case ISN_LEGACY_EVAL: *************** *** 4536,4541 **** --- 4585,4593 ---- case ISN_EXEC: smsg("%s%4d EXEC %s", pfx, current, iptr->isn_arg.string); break; + case ISN_EXEC_SPLIT: + smsg("%s%4d EXEC_SPLIT %s", pfx, current, iptr->isn_arg.string); + break; case ISN_LEGACY_EVAL: smsg("%s%4d EVAL legacy %s", pfx, current, iptr->isn_arg.string); *** ../vim-8.2.2950/src/testdir/test_vim9_func.vim 2021-06-02 16:47:49.675250216 +0200 --- src/testdir/test_vim9_func.vim 2021-06-06 17:00:08.310222128 +0200 *************** *** 2758,2762 **** --- 2758,2790 ---- call CheckDefAndScriptSuccess(lines) enddef + if has('python3') + def Test_python3_heredoc() + py3 << trim EOF + import vim + vim.vars['didit'] = 'yes' + EOF + assert_equal('yes', g:didit) + + python3 << trim EOF + import vim + vim.vars['didit'] = 'again' + EOF + assert_equal('again', g:didit) + enddef + endif + + " This messes up syntax highlight, keep near the end. + if has('lua') + def Test_lua_heredoc() + g:d = {} + lua << trim EOF + x = vim.eval('g:d') + x['key'] = 'val' + EOF + assert_equal('val', g:d.key) + enddef + endif + " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker *** ../vim-8.2.2950/src/testdir/test_vim9_disassemble.vim 2021-06-04 21:00:27.958234335 +0200 --- src/testdir/test_vim9_disassemble.vim 2021-06-06 16:40:54.121276234 +0200 *************** *** 121,126 **** --- 121,143 ---- res) enddef + if has('python3') + def s:PyHeredoc() + python3 << EOF + print('hello') + EOF + enddef + + def Test_disassemble_python_heredoc() + var res = execute('disass s:PyHeredoc') + assert_match('\d*_PyHeredoc.*' .. + " python3 << EOF^@ print('hello')^@EOF\\_s*" .. + '\d EXEC_SPLIT python3 << EOF^@ print(''hello'')^@EOF\_s*' .. + '\d RETURN 0', + res) + enddef + endif + def s:Substitute() var expr = "abc" :%s/a/\=expr/&g#c *** ../vim-8.2.2950/src/version.c 2021-06-06 15:07:05.624372128 +0200 --- src/version.c 2021-06-06 15:34:43.035903149 +0200 *************** *** 752,753 **** --- 752,755 ---- { /* Add new patch number below this line */ + /**/ + 2951, /**/ -- An SQL statement walks into a bar. He approaches two tables and says, "Mind if I join you?" /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// \\\ \\\ sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///