diff options
Diffstat (limited to 'autoload')
| -rw-r--r-- | autoload/latex-wordcount.vim | 704 |
1 files changed, 0 insertions, 704 deletions
diff --git a/autoload/latex-wordcount.vim b/autoload/latex-wordcount.vim deleted file mode 100644 index a3dca18..0000000 --- a/autoload/latex-wordcount.vim +++ /dev/null @@ -1,704 +0,0 @@ ---[[ -count_latex_words.lua - -Count the number of "real words" in a LaTeX (.tex) document -- i.e. the -words that would actually appear as visible text if the document were -typeset, excluding preamble, command definitions, commands, math, comments, -labels, citations, includes, and similar non-prose content. - -This is a pattern/heuristic-based tool, not a full LaTeX parser, so it will -not be 100% perfect on every document (LaTeX is not a regular language). -It mirrors a Python implementation of the same approach and aims to be a -practical approximation that handles the vast majority of real documents. - -USAGE AS A NEOVIM PLUGIN MODULE --------------------------------- - local wc = require("count_latex_words") - - -- Count the current buffer using the default rules: - local n = wc.count_buffer(0) - - -- Count the current buffer with a custom config file: - local config = wc.default_config() - wc.load_config(config, "/path/to/mycmds.json") - local n = wc.count_buffer(0, config) - - -- Count an arbitrary file or string: - local n = wc.count_file("paper.tex", config) - local n = wc.count_text("\\textbf{hello} world", config) - -If loaded inside Neovim, this module also registers a `:LatexWordCount` -user command (optionally takes a path to a JSON config file as its arg). - -USAGE AS A STANDALONE SCRIPT ------------------------------- - lua count_latex_words.lua paper.tex - lua count_latex_words.lua paper.tex mycmds.json - -CUSTOM COMMANDS (JSON CONFIG) ------------------------------- -Pass a JSON file describing how your own macros should be treated: - - { - "drop_environments": ["myfigureenv"], - "drop_commands": ["mysecretnote"], - "keep_arg_commands": ["myterm"], - "zero_arg_commands": ["companyname"], - "custom_arg_commands": { - "mytranslation": ["drop", "keep"] - } - } - - - drop_environments: environments whose entire contents are discarded. - - drop_commands: commands whose arguments carry no visible words; - command + all {..} args are removed. - - keep_arg_commands: commands wrapping a single word/phrase to keep; - command is stripped, its {..} argument is kept. - - zero_arg_commands: commands with no argument that expand to nothing - countable; just the command token is removed. - - custom_arg_commands: commands with MULTIPLE {..} arguments where each - argument is individually "keep" or "drop", matched - in order, e.g. \mytranslation{fr}{en} -> - ["drop", "keep"] keeps only the English text. - -Config rules extend the built-in defaults. If a command name is listed -under a new category, it is automatically removed from any other category -to avoid ambiguity (so a config file can also *reclassify* a built-in). ---]] - -local M = {} - --- --------------------------------------------------------------------- --- Default rule sets --- --------------------------------------------------------------------- - -local function to_set(list) - local s = {} - for _, v in ipairs(list) do - s[v] = true - end - return s -end - -local DEFAULT_DROP_ENVIRONMENTS = { - "equation", "equation*", "align", "align*", "alignat", "alignat*", - "gather", "gather*", "multline", "multline*", "eqnarray", "eqnarray*", - "verbatim", "verbatim*", "Verbatim", "lstlisting", "minted", "algorithm", - "algorithmic", "tikzpicture", "pgfpicture", "comment", -} - -local DEFAULT_DROP_COMMANDS = { - "label", "ref", "eqref", "pageref", "autoref", "nameref", "vref", - "cite", "citep", "citet", "citeauthor", "citeyear", "citealp", "citealt", - "nocite", "bibliography", "bibliographystyle", - "usepackage", "documentclass", "RequirePackage", - "input", "include", "includeonly", "includegraphics", "includepdf", - "newcommand", "renewcommand", "providecommand", "DeclareMathOperator", - "newenvironment", "renewenvironment", "newcolumntype", "definecolor", - "colorlet", "setlength", "addtolength", "setcounter", "addtocounter", - "hspace", "vspace", "linespread", "pagestyle", - "thispagestyle", "graphicspath", "geometry", "hypersetup", - "bibinput", "newpage", "clearpage", "cleardoublepage", - "pagebreak", "linebreak", "nopagebreak", "nolinebreak", "appendix", -} - -local DEFAULT_KEEP_ARG_COMMANDS = { - "textbf", "textit", "emph", "underline", "texttt", "textsc", "textsf", - "textrm", "textnormal", "text", "mbox", "uline", "uwave", "sout", - "section", "subsection", "subsubsection", "paragraph", "subparagraph", - "chapter", "part", "caption", "captionof", "footnote", "footnotetext", - "title", "author", "date", "thanks", -} - -local DEFAULT_ZERO_ARG_COMMANDS = { - "maketitle", "tableofcontents", "listoffigures", "listoftables", - "noindent", "indent", "par", "newline", "smallskip", "medskip", - "bigskip", "today", "LaTeX", "LaTeXe", "TeX", "item", -} - --- Commands with MULTIPLE {..} arguments, each individually "keep"/"drop". -local DEFAULT_CUSTOM_ARG_COMMANDS = { - textcolor = { "drop", "keep" }, - colorbox = { "drop", "keep" }, - href = { "drop", "keep" }, -} - -function M.default_config() - local custom = {} - for k, v in pairs(DEFAULT_CUSTOM_ARG_COMMANDS) do - custom[k] = { v[1], v[2] } - end - return { - drop_environments = to_set(DEFAULT_DROP_ENVIRONMENTS), - drop_commands = to_set(DEFAULT_DROP_COMMANDS), - keep_arg_commands = to_set(DEFAULT_KEEP_ARG_COMMANDS), - zero_arg_commands = to_set(DEFAULT_ZERO_ARG_COMMANDS), - custom_arg_commands = custom, - } -end - -function M.build_example_config() - return { - drop_environments = { "myfigureenv" }, - drop_commands = { "mysecretnote", "internalcomment" }, - keep_arg_commands = { "myterm", "highlightphrase" }, - zero_arg_commands = { "companyname" }, - custom_arg_commands = { - mytooltip = { "keep", "drop" }, - mytranslation = { "drop", "keep" }, - }, - } -end - --- --------------------------------------------------------------------- --- Minimal JSON decoder (fallback for use outside Neovim; Neovim's --- built-in vim.json.decode / vim.fn.json_decode is preferred when present) --- --------------------------------------------------------------------- - -local json_parse_value -- forward declaration - -local function json_skip_ws(s, i) - while i <= #s and s:sub(i, i):match("%s") do - i = i + 1 - end - return i -end - -local function json_parse_string(s, i) - local j = i + 1 - local buf = {} - local escmap = { ['"'] = '"', ['\\'] = '\\', ['/'] = '/', b = '\b', f = '\f', n = '\n', r = '\r', t = '\t' } - while j <= #s do - local c = s:sub(j, j) - if c == '"' then - return table.concat(buf), j + 1 - elseif c == "\\" then - local nc = s:sub(j + 1, j + 1) - if escmap[nc] then - table.insert(buf, escmap[nc]) - j = j + 2 - elseif nc == "u" then - local hex = s:sub(j + 2, j + 5) - local code = tonumber(hex, 16) or 63 - if code < 0x80 then - table.insert(buf, string.char(code)) - elseif utf8 and utf8.char then - table.insert(buf, utf8.char(code)) - else - table.insert(buf, "?") - end - j = j + 6 - else - table.insert(buf, nc) - j = j + 2 - end - else - table.insert(buf, c) - j = j + 1 - end - end - error("count_latex_words: unterminated string in JSON config") -end - -local function json_parse_number(s, i) - local j = i - while j <= #s and s:sub(j, j):match("[%d%+%-%.eE]") do - j = j + 1 - end - return tonumber(s:sub(i, j - 1)), j -end - -local function json_parse_array(s, i) - local arr = {} - i = json_skip_ws(s, i + 1) - if s:sub(i, i) == "]" then - return arr, i + 1 - end - while true do - local val - val, i = json_parse_value(s, i) - table.insert(arr, val) - i = json_skip_ws(s, i) - local c = s:sub(i, i) - if c == "," then - i = json_skip_ws(s, i + 1) - elseif c == "]" then - return arr, i + 1 - else - error("count_latex_words: malformed JSON array in config") - end - end -end - -local function json_parse_object(s, i) - local obj = {} - i = json_skip_ws(s, i + 1) - if s:sub(i, i) == "}" then - return obj, i + 1 - end - while true do - i = json_skip_ws(s, i) - if s:sub(i, i) ~= '"' then - error("count_latex_words: expected string key in JSON config") - end - local key - key, i = json_parse_string(s, i) - i = json_skip_ws(s, i) - if s:sub(i, i) ~= ":" then - error("count_latex_words: expected ':' in JSON config") - end - i = json_skip_ws(s, i + 1) - local val - val, i = json_parse_value(s, i) - obj[key] = val - i = json_skip_ws(s, i) - local c = s:sub(i, i) - if c == "," then - i = i + 1 - elseif c == "}" then - return obj, i + 1 - else - error("count_latex_words: malformed JSON object in config") - end - end -end - -json_parse_value = function(s, i) - i = json_skip_ws(s, i) - local c = s:sub(i, i) - if c == '"' then - return json_parse_string(s, i) - elseif c == "{" then - return json_parse_object(s, i) - elseif c == "[" then - return json_parse_array(s, i) - elseif s:sub(i, i + 3) == "true" then - return true, i + 4 - elseif s:sub(i, i + 4) == "false" then - return false, i + 5 - elseif s:sub(i, i + 3) == "null" then - return nil, i + 4 - else - return json_parse_number(s, i) - end -end - -function M.json_decode(str) - if vim ~= nil then - if vim.json and vim.json.decode then - local ok, result = pcall(vim.json.decode, str) - if ok then - return result - end - end - if vim.fn and vim.fn.json_decode then - local ok, result = pcall(vim.fn.json_decode, str) - if ok then - return result - end - end - end - local val = json_parse_value(str, 1) - return val -end - --- --------------------------------------------------------------------- --- Config loading --- --------------------------------------------------------------------- - ---- Merge rules from a JSON config file into an existing config table. --- @param config table returned by M.default_config() --- @param path string path to a JSON config file -function M.load_config(config, path) - local f, err = io.open(path, "r") - if not f then - error("count_latex_words: could not open config file: " .. tostring(err)) - end - local content = f:read("*a") - f:close() - - local data = M.json_decode(content) - if type(data) ~= "table" then - return config - end - - local function merge_list(key, target) - if data[key] then - for _, name in ipairs(data[key]) do - target[name] = true - end - end - end - - merge_list("drop_environments", config.drop_environments) - merge_list("drop_commands", config.drop_commands) - merge_list("keep_arg_commands", config.keep_arg_commands) - merge_list("zero_arg_commands", config.zero_arg_commands) - - if data.custom_arg_commands then - for name, pattern in pairs(data.custom_arg_commands) do - config.custom_arg_commands[name] = pattern - end - end - - -- Reclassify: a command named under a new category is removed from any - -- other category so config-file rules take priority over defaults. - local plain_keys = { "drop_commands", "keep_arg_commands", "zero_arg_commands" } - for _, key in ipairs(plain_keys) do - if data[key] then - for _, name in ipairs(data[key]) do - for _, other in ipairs(plain_keys) do - if other ~= key then - config[other][name] = nil - end - end - config.custom_arg_commands[name] = nil - end - end - end - if data.custom_arg_commands then - for name, _ in pairs(data.custom_arg_commands) do - config.drop_commands[name] = nil - config.keep_arg_commands[name] = nil - config.zero_arg_commands[name] = nil - end - end - - return config -end - --- --------------------------------------------------------------------- --- Cleaning pipeline --- --------------------------------------------------------------------- - ---- Remove LaTeX comments (unescaped % to end of line). -function M.strip_comments(text) - local out_lines = {} - for line in (text .. "\n"):gmatch("(.-)\n") do - local buf = {} - local i, n = 1, #line - while i <= n do - local c = line:sub(i, i) - if c == "\\" then - table.insert(buf, c) - if i < n then - table.insert(buf, line:sub(i + 1, i + 1)) - i = i + 2 - else - i = i + 1 - end - elseif c == "%" then - break - else - table.insert(buf, c) - i = i + 1 - end - end - table.insert(out_lines, table.concat(buf)) - end - -- (.."\n"):gmatch("(.-)\n") yields one trailing empty line; drop it if - -- the original text didn't end with a newline itself is not needed here - -- since callers only care about the joined result below. - return table.concat(out_lines, "\n") -end - ---- Keep only the content between \begin{document} and \end{document}. -function M.extract_body(text) - local _, _, body = text:find("\\begin{document}(.-)\\end{document}") - return body or text -end - ---- Escape Lua pattern magic characters in a literal string. -local function escape_pattern(s) - return (s:gsub("[%(%)%.%%%+%-%*%?%[%]%^%$]", "%%%1")) -end -M.escape_pattern = escape_pattern - ---- Remove the entire contents of math/code/graphics-language environments. -function M.drop_environments(text, config) - for env, _ in pairs(config.drop_environments) do - local pat = "\\begin{" .. escape_pattern(env) .. "}.-\\end{" .. escape_pattern(env) .. "}" - text = text:gsub(pat, " ") - end - return text -end - --- Lua patterns have no lookbehind, so a plain gsub can't tell an escaped --- "\$" apart from a real math delimiter. Scan manually instead, treating --- "$$" as display math and a lone "$" as inline math, skipping either kind --- when immediately preceded by a backslash. -local function strip_dollar_math(text) - local out = {} - local i, n = 1, #text - while i <= n do - local c = text:sub(i, i) - if c == "$" and text:sub(i - 1, i - 1) ~= "\\" then - local double = text:sub(i + 1, i + 1) == "$" - local delim = double and "$$" or "$" - local dlen = double and 2 or 1 - local j = i + dlen - local found = nil - while j <= n do - if text:sub(j, j + dlen - 1) == delim and text:sub(j - 1, j - 1) ~= "\\" then - found = j - break - end - j = j + 1 - end - if found then - table.insert(out, " ") - i = found + dlen - else - -- No closing delimiter found; treat as literal and move on. - table.insert(out, c) - i = i + 1 - end - else - table.insert(out, c) - i = i + 1 - end - end - return table.concat(out) -end - ---- Remove inline/display math: $...$, $$...$$, \(...\), \[...\]. -function M.drop_math(text) - text = strip_dollar_math(text) - text = text:gsub("\\%(.-\\%)", " ") - text = text:gsub("\\%[.-\\%]", " ") - return text -end - -local ESCAPED_CHARS = { - { "\\%%", "%%" }, - { "\\&", "&" }, - { "\\%$", "$" }, - { "\\#", "#" }, - { "\\_", "_" }, - { "\\{", "{" }, - { "\\}", "}" }, -} - -function M.replace_escaped_chars(text) - for _, pair in ipairs(ESCAPED_CHARS) do - text = text:gsub(pair[1], pair[2]) - end - return text -end - ---- If text[pos] starts an optional [..] argument, return the position just --- past its matching closing bracket; otherwise return pos unchanged. -function M.strip_optional_arg(text, pos) - if text:sub(pos, pos) == "[" then - local s, e = text:find("^%b[]", pos) - if s then - return e + 1 - end - end - return pos -end - ---- Given text:sub(pos,pos) == "{", return (content, end_index) where --- end_index is the index just past the matching closing brace. Handles --- nesting via Lua's %b{} balanced-match pattern item. -function M.read_braced_group(text, pos) - local s, e = text:find("^%b{}", pos) - if s then - return text:sub(s + 1, e - 1), e + 1 - end - -- Unbalanced braces -- bail out, consume to end of string. - return text:sub(pos + 1), #text + 1 -end - ---- Walk through text once, handling \commandname[opt]{arg}{arg2}... per the --- config's rule sets, with a fallback for unknown commands that keeps the --- first {..} argument's text (most custom macros just wrap visible text). -function M.process_commands(text, config) - local out = {} - local i, n = 1, #text - - while i <= n do - local next_bs = text:find("\\", i, true) - if not next_bs then - table.insert(out, text:sub(i)) - break - end - if next_bs > i then - table.insert(out, text:sub(i, next_bs - 1)) - end - i = next_bs - - local s, e, name = text:find("^\\(%a+)%*?", i) - if s then - local j = e + 1 - - if config.custom_arg_commands[name] then - j = M.strip_optional_arg(text, j) - for _, role in ipairs(config.custom_arg_commands[name]) do - if text:sub(j, j) == "{" then - local content - content, j = M.read_braced_group(text, j) - if role == "keep" then - table.insert(out, M.process_commands(content, config)) - end - else - break - end - end - i = j - elseif config.drop_commands[name] then - j = M.strip_optional_arg(text, j) - while text:sub(j, j) == "{" do - local _, jn = M.read_braced_group(text, j) - j = jn - end - i = j - elseif config.zero_arg_commands[name] then - j = M.strip_optional_arg(text, j) - i = j - elseif config.keep_arg_commands[name] then - j = M.strip_optional_arg(text, j) - if text:sub(j, j) == "{" then - local content - content, j = M.read_braced_group(text, j) - table.insert(out, M.process_commands(content, config)) - end - i = j - elseif name == "begin" or name == "end" then - j = M.strip_optional_arg(text, j) - if text:sub(j, j) == "{" then - local _, jn = M.read_braced_group(text, j) - j = jn - end - i = j - else - -- Unknown command: drop the name, skip an optional [..], keep the - -- first {..} argument's text. - j = M.strip_optional_arg(text, j) - if text:sub(j, j) == "{" then - local content - content, j = M.read_braced_group(text, j) - table.insert(out, M.process_commands(content, config)) - end - i = j - end - else - -- Backslash not followed by a letter: "\\" line break, or stray - -- backslash (already-escaped chars like \% were handled earlier). - if text:sub(i, i + 1) == "\\\\" then - table.insert(out, " ") - i = i + 2 - else - i = i + 1 - end - end - end - - return table.concat(out) -end - ---- Remove stray braces, alignment/table markup, and collapse whitespace. -function M.clean_leftovers(text) - text = text:gsub("{", " ") - text = text:gsub("}", " ") - text = text:gsub("&", " ") - text = text:gsub("~", " ") - text = text:gsub("%-%-%-?", " ") -- en/em dashes (-- or ---) as separators - text = text:gsub("%s+", " ") - text = text:match("^%s*(.-)%s*$") - return text -end - ---- Tokenize cleaned text into words. A "word" must contain at least one --- letter or digit (filters out stray punctuation / leftover markup). -function M.count_words(text) - local words = {} - for tok in text:gmatch("%S+") do - local trimmed = tok:gsub("^[%.,;:!%?%(%)%[%]\"'`]+", "") - trimmed = trimmed:gsub("[%.,;:!%?%(%)%[%]\"'`]+$", "") - if trimmed:match("%w") then - table.insert(words, trimmed) - end - end - return words -end - --- --------------------------------------------------------------------- --- Top-level entry points --- --------------------------------------------------------------------- - ---- Run the full pipeline over a raw LaTeX string. --- @return count, words (table), cleaned_text -function M.count_text(text, config) - config = config or M.default_config() - local t = text - t = M.strip_comments(t) - t = M.extract_body(t) - t = M.drop_environments(t, config) - t = M.drop_math(t) - t = M.replace_escaped_chars(t) - t = M.process_commands(t, config) - t = M.clean_leftovers(t) - local words = M.count_words(t) - return #words, words, t -end - ---- Run the full pipeline over a file on disk. -function M.count_file(path, config) - local f, err = io.open(path, "r") - if not f then - error("count_latex_words: could not open file: " .. tostring(err)) - end - local content = f:read("*a") - f:close() - return M.count_text(content, config) -end - ---- Run the full pipeline over a Neovim buffer's contents. --- @param bufnr number|nil buffer handle (0 or nil = current buffer) -function M.count_buffer(bufnr, config) - if vim == nil then - error("count_latex_words: count_buffer() requires Neovim's 'vim' API") - end - bufnr = bufnr or 0 - local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) - local text = table.concat(lines, "\n") - return M.count_text(text, config) -end - --- --------------------------------------------------------------------- --- Neovim plugin wiring (only active when loaded inside Neovim) --- --------------------------------------------------------------------- - -if vim ~= nil and vim.api ~= nil and vim.api.nvim_create_user_command ~= nil then - vim.api.nvim_create_user_command("LatexWordCount", function(opts) - local config = M.default_config() - if opts.args and opts.args ~= "" then - local ok, err = pcall(M.load_config, config, opts.args) - if not ok then - vim.notify(tostring(err), vim.log.levels.ERROR) - return - end - end - local count = M.count_buffer(0, config) - vim.notify(string.format("Real words: %d", count)) - end, { nargs = "?", complete = "file", desc = "Count real (visible) words in the current LaTeX buffer" }) -end - --- --------------------------------------------------------------------- --- Standalone CLI usage: `lua count_latex_words.lua paper.tex [config.json]` --- --------------------------------------------------------------------- - -if vim == nil and arg ~= nil and arg[0] and arg[1] then - local config = M.default_config() - if arg[2] then - M.load_config(config, arg[2]) - end - local count, _, cleaned = M.count_file(arg[1], config) - if os.getenv("LATEX_WORDS_VERBOSE") then - io.stderr:write("--- Cleaned text ---\n" .. cleaned .. "\n--- End cleaned text ---\n\n") - end - print(string.format("%s: %d words", arg[1], count)) -end - -return M |