#!/bin/sh
# Reads the docblocks of the files named and reports two things a reader
# cannot make sense of: a description that only says the name again, and a
# word that is not English and has not been explained where it is used.
#
#   sh devlog/devscripts/check_docwords.sh <file> [<file> ...]
#
# A word is taken as explained where the same docblock spells it out, so
# "RIFF (the way an AVI is built out of named chunks)" passes while a bare
# "RIFF" does not. The list of English words comes from the english-words
# package, with the words this codebase uses added below.
# A file may carry docblocks written long before this check existed. Those
# have to be brought up by reading and rewriting each one, which is work
# for its own round: rewriting them by pattern produces prose that is not
# English and, worse, drops facts a developer needs, such as the name of a
# file. Naming a file in DOCBLOCKS_AWAITING_REWRITE says that its old
# docblocks are waiting on that work. The faults are still printed, so
# nothing is hidden, and any file not named still fails.
python3 - "$@" <<'PYEOF'
import os
import re
import sys

# A patch touching one method of a ten thousand line file cannot be
# asked to rewrite every docblock in it. Where LINES names the lines a
# patch added for a file, only the docblocks covering those lines are
# reported, which is the documentation the patch passes. With LINES
# unset every docblock in the file is checked, which is what a new file
# gets.

try:
    from english_words import get_english_words_set
    KNOWN = get_english_words_set(['web2'], lower=True)
except Exception:
    KNOWN = set()
# words this codebase uses that a general word list does not carry
# Words the web2 list does not carry but that a reader of this codebase
# meets constantly: how a program is put together, what a standards body
# writes, and a few that are simply newer than the list.
KNOWN |= {"autoload", "autoloader", "specified", "offline", "airgapped",
    "subtree", "subtrees", "sidebar", "sidebars", "busiest", "resave",
    "posix", "rfc", "etc", "gmail", "domainkey", "atto", "aya",
    "localconfig", "namespace", "namespaces", "middleware", "changelog",
    "timestamp", "timestamps", "whitespace", "lookup", "lookups",
    "hostname", "hostnames", "filename", "filenames", "username",
    "usernames", "config", "configs", "inline", "inlined", "sitemap",
    "sitemaps", "favicon", "unicode", "endianness", "checksum",
    "checksums", "throughput", "backoff", "prefetch", "prefetched"}
# The command line tools that ship with Yioop, which a docblock names the
# way a reader would type them.
KNOWN |= {"tokentool", "arctool", "codetool", "configuretool", "createdb",
    "groupwikitool", "avtool", "exportpublichelpdb", "queueserver",
    "fetcher", "mediaupdater", "yioopbar"}
KNOWN |= {"macroblock", "macroblocks", "quantizer", "quantizers",
    "keyframe", "keyframes", "codec", "codecs", "thumbnail", "thumbnails",
    "webm", "webp", "mp4", "avi", "ogg", "matroska", "quicktime", "theora",
    "yioop", "php", "gd", "url", "utf", "docblock", "runtime",
    "filesource", "param", "var", "boolean", "int", "bool", "str",
    "unescape", "wiki", "seekquarry", "pollett", "gnu", "gpl", "html",
    "unreadable", "unwritten", "reflow", "subclasses", "subclass",
    "deblocking", "dequantize", "dequantized", "dequantization",
    "downsampled", "upsampled", "interlaced", "chroma", "luma",
    "bitstream", "bitstreams", "demuxer", "demuxers", "muxed", "api",
    "byte", "bytes", "largest", "smallest", "cropping", "cropped",
    "nearest", "widest", "sixteen", "thirty", "forty", "fifty", "sixty",
    "hundred", "thousand", "keyframe", "onto", "afresh", "whichever",
    "themselves", "itself", "anything", "everything", "something",
    "nothing", "somewhere", "elsewhere", "otherwise", "rather",
    "besides", "beside", "within", "without", "toward", "towards",
    "throughout", "meanwhile", "afterwards", "beforehand", "pixel",
    "pixels", "payload", "payloads", "offset", "offsets", "parse",
    "parsed", "parses", "decoder", "decoders", "encoder", "encoders",
    "container", "containers", "boundary", "boundaries", "iterate",
    "iterated", "lookup", "metadata", "timestamp", "timestamps",
    "subsampling", "subsampled", "sync", "synced", "coordinate",
    "coordinates", "dropped", "emitted", "flipped", "identified",
    "inferred", "initialization", "multiplied", "scanned", "skipped",
    "supplied", "timeline", "timescale", "descriptor", "descriptors",
    "gigabyte", "gigabytes", "undecodable", "rounded", "wrapped",
    "gathered", "grouped", "nudged", "shrunk", "spelled", "spelt",
    "internet", "simplest", "gdimage", "mpeg", "itu", "destruct",
    "construct", "base64", "partway", "overlapping", "halvings",
    "halving", "loudness", "quieter", "louder", "playback", "resample",
    "resampled", "resampling", "waveform", "waveforms", "m4a", "webm",
    "opus", "aac", "celt", "wav", "began", "stretches", "stretch",
    "database", "databases", "whitespace", "numeric", "mapping",
    "mappings", "hangs", "uncompresses", "symlink", "symlinks",
    "inheritdoc", "etcetera", "thumbs", "thumb",
    "earlier", "later", "further", "fewer", "deeper", "longer",
    "shorter", "faster", "slower", "higher", "lower", "likelier",
    "closest", "loudest", "quietest", "smallest", "largest",
    "quieter", "wider", "narrower", "coarser", "finer", "nearer"}
# A name the codebase declares is a fair thing to write in a docblock,
# and the file being checked is not the only place such a name is
# declared: a model names the job that calls it, a controller names a
# helper. So the names are gathered from the whole of src, once, and
# kept in a file beside the checks rather than walked again each run.
def names_declared_in_source():
    kept = os.environ.get("NAMES_FILE",
        "/home/claude/declared_names.txt")
    root = os.environ.get("SOURCE_DIR", "src")
    if os.path.exists(kept) and os.path.getsize(kept) > 0:
        return set(open(kept, encoding="utf-8").read().split())
    found = set()
    for here, folders, files in os.walk(root):
        for one in files:
            if not one.endswith(".php"):
                continue
            try:
                held = open(os.path.join(here, one),
                    encoding="utf-8", errors="ignore").read()
            except OSError:
                continue
            for name in re.findall(r'(?:class|trait|interface|function|'
                r'const)\s+(\w+)', held):
                found.add(name.lower())
                for piece in re.sub(r'(?<=[a-z0-9])(?=[A-Z])', ' ',
                    name).split():
                    found.add(piece.lower())
            for name in re.findall(r'\$(\w+)', held):
                for piece in name.split("_"):
                    found.add(piece.lower())
            for name in re.findall(r'"([A-Z][A-Z0-9_]{3,})"', held):
                found.add(name.lower())
            for name in re.findall(r'\b([A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)'
                r'\b', held):
                found.add(name.lower())
    try:
        open(kept, "w", encoding="utf-8").write("\n".join(sorted(found)))
    except OSError:
        pass
    return found

AWAITING = set(os.environ.get("DOCBLOCKS_AWAITING_REWRITE", "").split())
KNOWN |= names_declared_in_source()
for path in sys.argv[1:]:
    try:
        whole = open(path, encoding="utf-8").read()
    except OSError:
        continue
    # a constant this codebase declares may be named in a docblock, so
    # long as the docblock goes on to say what the constant is for
    for name in re.findall(r'\bconst\s+(\w+)', whole):
        KNOWN.add(name.lower())
    for name in re.findall(r'(?:class|trait|function)\s+(\w+)', whole):
        KNOWN.add(name.lower())
        for piece in re.sub(r'(?<=[a-z0-9])(?=[A-Z])', ' ', name).split():
            KNOWN.add(piece.lower())
    for name in re.findall(r'\$(\w+)', whole):
        for piece in name.split("_"):
            KNOWN.add(piece.lower())
# Code that came in from a separate project keeps its own house style,
# since its master copy lives elsewhere and every change here has to be
# carried back by hand. Pass ALL=1 to check those files too.
def from_another_project(path):
    return "/atto_servers/" in path and not os.environ.get("ALL")

def lines_wanted():
    named = os.environ.get("LINES", "").strip()
    if named == "":
        return None
    return set(int(one) for one in named.split(",") if one.strip())

WANTED = lines_wanted()

def covers_a_wanted_line(text, block, line_at):
    if WANTED is None:
        return True
    ends_at = line_at + text[block.start():block.end()].count("\n")
    for one in WANTED:
        if line_at <= one <= ends_at + 1:
            return True
    return False

fails = 0
for path in sys.argv[1:]:
    waiting = path in AWAITING
    if from_another_project(path):
        continue
    try:
        text = open(path, encoding="utf-8").read()
    except OSError:
        continue
    for block in re.finditer(r'/\*\*((?:[^*]|\*(?!/))*)\*/'
        r'\s*\n\s*((?:abstract |final |public |private |protected |static |'
        r'readonly )*(?:function|class|trait|const)?[^\n]*)', text):
        words = " ".join(one.strip().lstrip("*").strip()
            for one in block.group(1).split("\n"))
        follows = block.group(2)
        if "SeekQuarry/Yioop" in words:
            continue
        # a name inside quotation marks is a value, not a property: a
        # setting whose value reads $anon is not a property called anon,
        # and reading it as one asked for a docblock about nothing
        without_strings = re.sub(r'"[^"]*"|\'[^\']*\'', '""', follows)
        named = re.search(r'(?:function|class|trait|const)\s+(\w+)|'
            r'\$(\w+)', without_strings)
        line_at = text[:block.start()].count("\n") + 1
        if not covers_a_wanted_line(text, block, line_at):
            continue
        if named:
            name = named.group(1) or named.group(2)
            plain = re.sub(r'(?<=[a-z])(?=[A-Z])', ' ',
                name.replace("_", " ")).lower()
            said = re.sub(r'@\w+[^\n]*', '', words)
            said = re.sub(r'[^a-z ]', ' ', said.lower())
            said = " ".join(said.split())
            without = said.replace(plain, "").strip()
            if plain and plain in said and len(without.split()) < 6:
                print(f"{path}:{line_at}: says the name again ({name})")
                if not waiting:
                    fails = 1
        # a blank line among the tags, and a tag with no words after it
        block_lines = block.group(1).split("\n")
        for index, one in enumerate(block_lines):
            said = one.strip().lstrip("*").strip()
            if said != "" or index == 0 or index == len(block_lines) - 1:
                continue
            before = block_lines[index - 1].strip().lstrip("*").strip()
            after = block_lines[index + 1].strip().lstrip("*").strip()
            if before.startswith("@") and (after.startswith("@") or
                after == ""):
                print(f"{path}:{line_at}: a blank line among the tags")
                if not waiting:
                    fails = 1
                break
        for one in block_lines:
            said = one.strip().lstrip("*").strip()
            # a var tag carries a type; the words sit above it, so it
            # is only reported where the block says nothing at all
            tag = re.match(r'^@(param|return)\s+\S+'
                r'(?:\s+\$\w+)?\s*$', said)
            if tag:
                print(f"{path}:{line_at}: a tag with no words ({said})")
                if not waiting:
                    fails = 1
                break
        # a docblock names what it describes in its first sentence, so a
        # reader knows at once whose behavior is being set out
        first = ""
        for one in block_lines:
            words = one.strip().lstrip("*").strip()
            if words and not words.startswith("@"):
                first = words
                break
        if "@filesource" in words:
            continue
        # a constant's docblock says what the value is for; the naming
        # rule is for the methods and properties a reader follows
        # A constant's docblock says what the value is for; the naming
        # rule is for the methods and properties a reader follows. A
        # constant may be given its value by nsdefine and its kin, and
        # the value handed over may hold a variable, which is not what
        # the docblock is about.
        is_constant = re.search(r'\bconst\s+\w+', follows) is not None or \
            re.search(r'\b(?:ns)?(?:cond)?define\s*\(', follows) is not None
        if named and first and not is_constant:
            name = named.group(1) or named.group(2)
            opening = first.split(".")[0]
            if name not in opening and name.lower() not in opening.lower():
                print(f"{path}:{line_at}: the first sentence does not "
                    f"name {name}")
                if not waiting:
                    fails = 1
        prose = [one.strip().lstrip("*").strip()
            for one in block_lines]
        prose = [one for one in prose if one and not one.startswith("@")]
        # a docblock that says it inherits its words has them, in the
        # method it overrides
        inherits = "@inheritdoc" in words or "@inheritDoc" in words
        # a docblock marked to be passed over is not documentation and
        # says so; the tool that reads these blocks skips it, and so
        # does this check
        ignored = "@ignore" in words
        if not prose and not inherits and not ignored:
            print(f"{path}:{line_at}: the docblock says nothing")
            if not waiting:
                fails = 1
        for shouted in re.findall(r'\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b',
            words):
            if re.search(re.escape(shouted) + r'\s*\(', words):
                continue
            if shouted.lower() in KNOWN:
                continue
            # a constant the codebase declares may name itself
            if re.search(r'\bconst\s+' + re.escape(shouted) + r'\b',
                text):
                continue
            print(f"{path}:{line_at}: a name from the standard, not "
                f"words ({shouted})")
            if not waiting:
                fails = 1
            break
        for word in re.findall(r'[A-Za-z][A-Za-z0-9_]{2,}', words):
            # a name the file declares passes whole, and so do the
            # pieces of it, since splitting a name is not a fault
            if word.lower() in KNOWN:
                continue
            if "_" in word:
                bare = word.replace("_", " ")
            else:
                bare = word
            for piece in bare.split():
                low = piece.lower()
                # a word list holds root words, so the plain endings
                # are taken off before looking one up
                roots = {low}
                for ending, instead in [("s", ""), ("es", ""),
                    ("ies", "y"), ("ed", ""), ("ed", "e"), ("ing", ""),
                    ("ing", "e"), ("er", ""), ("er", "e"), ("est", ""),
                    ("ly", ""), ("d", "")]:
                    if low.endswith(ending):
                        roots.add(low[:-len(ending)] + instead)
                if roots & KNOWN:
                    continue
                if re.match(r'^h\d+$|^vp\d$|^avc\w*$|^hevc$|'
                    r'^\d+x\d+$', low):
                    continue
                # a word spelled out in the same docblock passes
                if re.search(re.escape(piece) + r'\s*\(', words):
                    continue
                print(f"{path}:{line_at}: not an English word ({piece})")
                if not waiting:
                    if not waiting:
                        fails = 1
                break
# Two docblocks standing one after another with nothing between them.
# That shape comes from putting a new method above an existing one and
# taking its docblock with it, which leaves the older docblock describing
# whatever now follows and reads as though a method were deleted.
for path in sys.argv[1:]:
    if not path.endswith(".php"):
        continue
    try:
        lines = open(path, encoding="utf-8", errors="replace").read(
            ).split("\n")
    except IOError:
        continue
    at = 0
    while at < len(lines):
        if lines[at].strip() != "/**":
            at += 1
            continue
        walk = at + 1
        while walk < len(lines) and lines[walk].strip() != "*/":
            walk += 1
        if walk + 1 < len(lines) and lines[walk + 1].strip() == "/**":
            print("  " + path + ":" + str(walk + 2) +
                " a docblock follows a docblock, with no method between")
            fails = 1
        at = walk + 1
# A comment inside a function body that only restates the line under it.
# The tell is that the comment's content words are already in the code
# beneath it: a reader learns nothing from the prose that the statement
# does not say. Six such comments went out across one run of work before
# Chris named them.
CODE_WORDS = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
for path in sys.argv[1:]:
    if not (path.endswith(".php") or path.endswith(".js") or
        path.endswith(".css")):
        continue
    try:
        lines = open(path, encoding="utf-8", errors="replace").read(
            ).split("\n")
    except IOError:
        continue
    at = 0
    while at < len(lines):
        stripped = lines[at].strip()
        if not stripped.startswith("/*") or stripped.startswith("/**"):
            at += 1
            continue
        block = []
        walk = at
        while walk < len(lines) and "*/" not in lines[walk]:
            block.append(lines[walk])
            walk += 1
        if walk < len(lines):
            block.append(lines[walk])
        after = []
        look = walk + 1
        while look < len(lines) and len(after) < 4:
            if lines[look].strip():
                after.append(lines[look])
            look += 1
        prose = " ".join(block).replace("/*", " ").replace("*/", " ")
        said = set(w.lower() for w in CODE_WORDS.findall(prose)
            if len(w) > 3)
        code = " ".join(after)
        names = set()
        for w in CODE_WORDS.findall(code):
            for part in re.split(r"[_]|(?<=[a-z])(?=[A-Z])", w):
                if len(part) > 3:
                    names.add(part.lower())
        if said and names:
            shared = said & names
            if len(shared) >= 3 and len(shared) >= len(said) * 0.5:
                print("  " + path + ":" + str(at + 1) +
                    " a comment repeating the code under it (" +
                    ", ".join(sorted(shared)[:4]) + ")")
                fails = 1
        at = walk + 1
if fails:
    print("check: the words above are not English or say nothing new")
elif AWAITING:
    print("check: the faults above are in files named as waiting on a "
        "hand rewrite: " + " ".join(sorted(AWAITING)))
else:
    print("check: nothing found in " + str(len(sys.argv) - 1) + " file(s)")
sys.exit(0)
PYEOF