From b5a05ac43a0cce548020f9453954a7cb666408a3 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 6 Aug 2024 23:51:08 -0700 Subject: [PATCH] initial commit --- .gitignore | 2 + LICENSE | 19 + README.md | 21 + gerrit_linkbot/__init__.py | 230 +++++++++++ gerrit_linkbot/__main__.py | 27 ++ gerrit_linkbot/constants.py | 6 + gerrit_linkbot/forgejo.py | 179 +++++++++ gerrit_linkbot/gerrit.py | 179 +++++++++ gerrit_linkbot/http.py | 4 + gerrit_linkbot/test_gerrit_linkbot.py | 121 ++++++ .../testdata/change-comments-lix~1218.json | 20 + .../testdata/change-comments-lix~1367.json | 72 ++++ .../testdata/change-comments-lix~1369.json | 20 + gerrit_linkbot/testdata/changes.json | 374 ++++++++++++++++++ .../issue-comments-lix-project-lix-361.json | 111 ++++++ module.nix | 84 ++++ package.nix | 35 ++ pyproject.toml | 16 + setup.py | 3 + 19 files changed, 1523 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 gerrit_linkbot/__init__.py create mode 100644 gerrit_linkbot/__main__.py create mode 100644 gerrit_linkbot/constants.py create mode 100644 gerrit_linkbot/forgejo.py create mode 100644 gerrit_linkbot/gerrit.py create mode 100644 gerrit_linkbot/http.py create mode 100644 gerrit_linkbot/test_gerrit_linkbot.py create mode 100644 gerrit_linkbot/testdata/change-comments-lix~1218.json create mode 100644 gerrit_linkbot/testdata/change-comments-lix~1367.json create mode 100644 gerrit_linkbot/testdata/change-comments-lix~1369.json create mode 100644 gerrit_linkbot/testdata/changes.json create mode 100644 gerrit_linkbot/testdata/issue-comments-lix-project-lix-361.json create mode 100644 module.nix create mode 100644 package.nix create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0e28012 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.egg-info/ +/build/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7f8f94e --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4fe1756 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# Gerrit link bot + +This is a bot that polls Gerrit for changes, finds issue references in them, +then goes and posts comments linking back to the CLs. + +## Configuration + +Set FORGEJO_API_KEY in the environment. Currently the endpoints for everything +are hardcoded to the lix.systems ones. + +## Testing locally + +If you want to test locally, go impersonate lix-bot on +https://identity.lix.systems/admin, relogin to Forgejo, then issue yourself a +new personal access token for `lix-bot` with `user:read` and `issue:write`. Set +that in FORGEJO_API_KEY in the environment. + +If you just want to test that it does mostly the right thing, consider either +writing an integration test using the fake APIs, or passing `--fake` on +startup, which will make it operate on fake data and not hit the internet at +all. diff --git a/gerrit_linkbot/__init__.py b/gerrit_linkbot/__init__.py new file mode 100644 index 0000000..6cbbc80 --- /dev/null +++ b/gerrit_linkbot/__init__.py @@ -0,0 +1,230 @@ +import requests +import re +import time +import textwrap +import json +import dataclasses +import logging +import importlib.resources +import importlib.resources.abc +from typing import Literal, Optional +from . import gerrit, forgejo +from .constants import FORGEJO_ACCOUNT, FORGEJO_SERVICE_ACCOUNT, FORGEJO_BASE + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + +fmt = logging.Formatter('{asctime} {levelname} {name}: {message}', + datefmt='%b %d %H:%M:%S', + style='{') + +if not any(isinstance(h, logging.StreamHandler) for h in log.handlers): + hand = logging.StreamHandler() + hand.setFormatter(fmt) + log.addHandler(hand) + +testdata = importlib.resources.files('gerrit_linkbot.testdata') + + +@dataclasses.dataclass +class IssueMatch: + """Result of applying a RegexRule.""" + issue_num: int + project_name: Optional[str] + + +@dataclasses.dataclass +class RegexRule: + """Rule applied on Gerrit CL comments and commit messages to extract issue references.""" + regex: re.Pattern + capture_num: int + project_name_capture_num: Optional[int] = None + + def apply(self, s: str) -> list[IssueMatch]: + return [ + IssueMatch( + int(m.group(self.capture_num)), + m.group(self.project_name_capture_num) + if self.project_name_capture_num else None) + for m in self.regex.finditer(s) + ] + + +RULES = [ + # #123, fj#123 + RegexRule( + re.compile( + r"(?:^|[\s\]\[(){},.;:!?])(fj|lix)?#(\d+)(?=$|[\s\]\[(){},.;:!?])" + ), 2), + # URL to some issue + RegexRule(re.compile( + re.escape(FORGEJO_BASE) + "/" + re.escape(FORGEJO_ACCOUNT) + + r"/([a-zA-Z0-9-]+)/issues/([0-9]+)"), + 2, + project_name_capture_num=1) +] + + +@dataclasses.dataclass +class CLMention: + """One mention of a CL in a Forgejo comment, without mutable information about the CL itself.""" + + backlink: str + number: int + kind: Literal['comment'] | Literal['commit message'] + + @classmethod + def from_messageish(cls, message_ish: gerrit.MessageIsh) -> 'CLMention': + return cls(backlink=message_ish.backlink, + number=message_ish.cl_number, + kind=message_ish.kind) + + @classmethod + def parse(cls, obj) -> 'CLMention': + return cls(backlink=obj['backlink'], + number=obj['number'], + kind=obj['kind']) + + +@dataclasses.dataclass +class CLMeta: + """Mutable metadata on a Gerrit CL.""" + change_title: str + + @classmethod + def parse(cls, obj) -> 'CLMeta': + return cls(change_title=obj['change_title']) + + +@dataclasses.dataclass +class CommentMeta: + """ + The complete set of information that is put into a Forgejo comment. + """ + + cls: list[CLMention] + """CLs mentioned, with unchanging metadata""" + cl_meta: dict[int, CLMeta] + """CL metadata that might change (e.g. title)""" + + def serialize(self): + return { + 'cls': [dataclasses.asdict(x) for x in self.cls], + 'cl_meta': { + num: dataclasses.asdict(x) + for num, x in self.cl_meta.items() + } + } + + @classmethod + def parse(cls, obj) -> 'CommentMeta': + return cls(cls=[CLMention.parse(x) for x in obj['cls']], + cl_meta={ + int(num): CLMeta.parse(x) + for num, x in obj['cl_meta'].items() + }) + + +def parse_comment(comment: str): + LEADER = ' + + This issue was mentioned on Gerrit on the following CLs: + {cls} + """).format(data_json=data_json, cls=cls) + + +class Service: + """ + The top level job in gerrit-linkbot: composes together the GerritWatcher + and taking actual actions with the Forgejo API. + """ + + def __init__(self, + gerrit_api: gerrit.API, + forgejo_api: forgejo.API, + poll_interval=30, + gerrit_per_page=25): + self.gerrit_api = gerrit_api + self.forgejo_api = forgejo_api + self.watcher = gerrit.GerritWatcher(gerrit_api, + self.handle_comment, + per_page=gerrit_per_page) + self.poll_interval = poll_interval + + def handle_hit(self, matched: IssueMatch, comment: gerrit.MessageIsh): + target = [ + FORGEJO_ACCOUNT, matched.project_name or comment.project, + matched.issue_num + ] + try: + comments = list(self.forgejo_api.all_comments(*target)) + except requests.HTTPError as e: + if e.response.status_code in (404, 500): + # The issue doesn't exist, and it may be 500 because of a forgejo bug: + # https://codeberg.org/forgejo/forgejo/issues/4005 + log.warn('Issue %s/%s#%d does not exist', *target) + return + raise + + existing_comment = None + comment_meta = CommentMeta([], {}) + for fj_comment in comments: + if fj_comment.author == FORGEJO_SERVICE_ACCOUNT: + data = parse_comment(fj_comment.text) + if data: + comment_meta = data + existing_comment = (fj_comment.id, fj_comment.text) + break + + # construct the new comment on top of the existing one + comment_meta.cl_meta[comment.cl_number] = CLMeta(comment.change_title) + if not any(cl_mention.number == comment.cl_number + and cl_mention.kind == comment.kind + for cl_mention in comment_meta.cls): + comment_meta.cls.append(CLMention.from_messageish(comment)) + + text = make_comment(comment_meta) + + if existing_comment: + # no change to the comment, no point posting it + if existing_comment[1] == text: + return + + self.forgejo_api.edit_comment( + *target, + existing_comment[0], # type: ignore + text) # type: ignore + else: + self.forgejo_api.post_comment(*target, text) # type: ignore + + def handle_comment(self, comment: gerrit.MessageIsh): + log.debug('Comment: %s', comment) + for rule in RULES: + for res in rule.apply(comment.message): + self.handle_hit(res, comment) + + def step(self): + self.watcher.poll() + + def run(self): + while True: + self.step() + time.sleep(self.poll_interval) diff --git a/gerrit_linkbot/__main__.py b/gerrit_linkbot/__main__.py new file mode 100644 index 0000000..9a0b3b0 --- /dev/null +++ b/gerrit_linkbot/__main__.py @@ -0,0 +1,27 @@ +from . import gerrit, forgejo +import gerrit_linkbot + + +def main(): + import argparse + + ap = argparse.ArgumentParser(description='Gerrit link bot') + ap.add_argument('--fake', action='store_true', help='Fake everything') + ap.add_argument('--gerrit-per-page', type=int, help='Number of results per page (for backfilling gerrit)') + ap.add_argument('--poll-interval', type=int, help='How many seconds between polls of Gerrit', default=30) + + parsed = ap.parse_args() + + if parsed.fake: + gerrit_api = gerrit.FakeAPI(gerrit_linkbot.testdata) + forgejo_api = forgejo.FakeAPI(gerrit_linkbot.testdata) + else: + gerrit_api = gerrit.ConcreteAPI() + forgejo_api = forgejo.ConcreteAPI() + + service = gerrit_linkbot.Service(gerrit_api, forgejo_api, poll_interval=parsed.poll_interval, gerrit_per_page=parsed.gerrit_per_page) + service.run() + + +if __name__ == '__main__': + main() diff --git a/gerrit_linkbot/constants.py b/gerrit_linkbot/constants.py new file mode 100644 index 0000000..f4cb022 --- /dev/null +++ b/gerrit_linkbot/constants.py @@ -0,0 +1,6 @@ + +FORGEJO_ACCOUNT = 'lix-project' +FORGEJO_SERVICE_ACCOUNT = 'lix-bot' + +GERRIT_BASE = 'https://gerrit.lix.systems' +FORGEJO_BASE = 'https://git.lix.systems' diff --git a/gerrit_linkbot/forgejo.py b/gerrit_linkbot/forgejo.py new file mode 100644 index 0000000..040aab9 --- /dev/null +++ b/gerrit_linkbot/forgejo.py @@ -0,0 +1,179 @@ +import dataclasses +from importlib.resources.abc import Traversable +import json +import abc +from typing import Any, Generator +import logging +import os + +from .constants import FORGEJO_SERVICE_ACCOUNT, FORGEJO_BASE +from . import http + +log = logging.getLogger(__name__) + +API_BASE = FORGEJO_BASE + '/api/v1' + + +@dataclasses.dataclass +class Comment: + """One issue comment on Forgejo.""" + author: str + text: str + id: int + + @classmethod + def parse(cls, obj) -> 'Comment': + return cls( + author=obj['user']['username'], + text=obj['body'], + id=obj['id'], + ) + + +class API(metaclass=abc.ABCMeta): + """The Forgejo API, as used by gerrit-linkbot.""" + + @abc.abstractmethod + def all_comments(self, owner: str, repo: str, + issue: int) -> Generator[Comment, None, None]: + pass + + @abc.abstractmethod + def edit_comment(self, owner: str, repo: str, issue: int, id: int, + comment: str) -> Any: + pass + + @abc.abstractmethod + def post_comment(self, owner: str, repo: str, issue: int, + comment: str) -> Any: + pass + + +@dataclasses.dataclass(frozen=True) +class CommentLocator: + """Location that a comment might appear; effectively a fully-qualified issue ID.""" + owner: str + repo: str + issue_id: int + + +@dataclasses.dataclass(frozen=True) +class PostedComment: + """Resulting posted comment in the fake API.""" + locator: CommentLocator + comment: str + comment_id: int + + +@dataclasses.dataclass(frozen=True) +class EditedComment: + locator: CommentLocator + comment: str + edit_comment_id: int + + +class FakeAPI(API): + + def __init__(self, fixtures: Traversable): + super().__init__() + self.fixtures = fixtures + self.posted_comments = [] + + self.mock_id = 10000 + self.comments_by_locator: dict[CommentLocator, list[Comment]] = {} + + def fixture(self, name: str): + return json.loads(self.fixtures.joinpath(name).read_text()) + + def init_comments(self, locator: CommentLocator): + comments = self.comments_by_locator.get(locator, None) + if comments is None: + comments = [ + Comment.parse(c) for c in self.fixture( + f'issue-comments-{locator.owner}-{locator.repo}-{locator.issue_id}.json' + ) + ] + self.comments_by_locator[locator] = comments + + def all_comments(self, owner: str, repo: str, + issue: int) -> Generator[Comment, None, None]: + locator = CommentLocator(owner, repo, issue) + self.init_comments(locator) + yield from self.comments_by_locator[locator] + + def next_id(self) -> int: + id = self.mock_id + self.mock_id += 1 + return id + + def edit_comment(self, owner: str, repo: str, issue: int, id: int, + comment: str): + log.info('Post comment on %s/%s#%d %d: %s', owner, repo, issue, id, + comment) + locator = CommentLocator(owner, repo, issue) + for existing in self.comments_by_locator[locator]: + if existing.id == id: + existing.text = comment + + self.posted_comments.append( + EditedComment(locator=CommentLocator(owner, repo, issue), + edit_comment_id=id, + comment=comment)) + + def post_comment(self, owner: str, repo: str, issue: int, comment: str): + log.info('Post comment on %s/%s#%d: %s', owner, repo, issue, comment) + locator = CommentLocator(owner, repo, issue) + + id = self.next_id() + self.comments_by_locator[locator].append( + Comment(FORGEJO_SERVICE_ACCOUNT, comment, id)) + + self.posted_comments.append( + PostedComment(locator=locator, comment=comment, comment_id=id)) + + +class ConcreteAPI(API): + + def api(self, method, endpoint: str, resp_json=True, **kwargs): + API_KEY = os.environ['FORGEJO_API_KEY'] + + log.info('http %s %s', method, endpoint) + if not endpoint.startswith('https'): + endpoint = API_BASE + endpoint + resp = http.session.request( + method, + endpoint, + headers={'Authorization': f'Bearer {API_KEY}'}, + **kwargs) + resp.raise_for_status() + if resp_json: + return resp.json() + else: + return resp + + def paginate(self, method: str, url: str): + while True: + resp = self.api(method, url, resp_json=False) + yield from resp.json() + next_one = resp.links.get('next') + if not next_one: + return + url = next_one.get('url') + if not url: + return + + def all_comments(self, owner: str, repo: str, + issue: int) -> Generator[Comment, None, None]: + yield from (Comment.parse(c) for c in self.paginate( + 'GET', f'/repos/{owner}/{repo}/issues/{issue}/comments')) + + def edit_comment(self, owner: str, repo: str, issue: int, id: int, + comment: str): + return self.api('PATCH', + f'/repos/{owner}/{repo}/issues/comments/{id}', + json={'body': comment}) + + def post_comment(self, owner: str, repo: str, issue: int, comment: str): + return self.api('POST', + f'/repos/{owner}/{repo}/issues/{issue}/comments', + json={'body': comment}) diff --git a/gerrit_linkbot/gerrit.py b/gerrit_linkbot/gerrit.py new file mode 100644 index 0000000..d8ae872 --- /dev/null +++ b/gerrit_linkbot/gerrit.py @@ -0,0 +1,179 @@ +import dataclasses +from importlib.resources.abc import Traversable +import json +from typing import Callable, Literal +import logging +import abc +from datetime import datetime + +from .constants import GERRIT_BASE +from . import http + +log = logging.getLogger(__name__) + + +@dataclasses.dataclass +class Change: + """Change metadata acquired from the changes query endpoint.""" + subject: str + project: str + id: str + number: int + updated: datetime + commit_msg: str + + @classmethod + def parse(cls, obj: dict) -> 'Change': + commit = obj['current_revision'] + rev_obj = obj['revisions'][commit] + commit_msg = rev_obj['commit']['message'] + + return cls(subject=obj['subject'], + id=obj['id'], + project=obj['project'], + number=obj['_number'], + commit_msg=commit_msg, + updated=datetime.fromisoformat(obj['updated'])) + + +@dataclasses.dataclass +class Comment: + """One comment from the change comments endpoint.""" + message: str + updated: datetime + + @classmethod + def parse(cls, obj) -> 'Comment': + return Comment(message=obj['message'], + updated=datetime.fromisoformat(obj['updated'])) + + +@dataclasses.dataclass +class MessageIsh: + """ + Something kind of like a comment or a commit message. Sufficiently vague + that we can treat both the same. + """ + change_title: str + project: str + message: str + backlink: str + cl_number: int + kind: Literal['comment'] | Literal['commit message'] + + +class API(metaclass=abc.ABCMeta): + """A Gerrit API, as used by gerrit-linkbot.""" + + def __init__(self, base: str): + self.base = base + + @abc.abstractmethod + def changes(self, per_page=25) -> list[Change]: + pass + + @abc.abstractmethod + def change_comments(self, change_id: str) -> dict[str, list[Comment]]: + pass + + def change_link(self, change_id: str) -> str: + """Assumes that change_id is of a format project~id""" + [project, id] = change_id.split('~') + return self.base + f'/c/{project}/+/{id}' + + +class ConcreteAPI(API): + """An actual non-fake Gerrit API.""" + + def __init__(self, base=GERRIT_BASE): + super().__init__(base) + + def get(self, endpoint: str, **kwargs): + log.info('GET %s params=%s', endpoint, kwargs.get('params')) + resp = http.session.get(self.base + endpoint, + headers={'Accept': 'application/json'}, + **kwargs) + resp.raise_for_status() + return json.loads(resp.content.removeprefix(b")]}'")) + + def changes(self, per_page=25) -> list[Change]: + resp = self.get('/changes/', + params={ + 'n': per_page, + 'o': ['CURRENT_REVISION', 'CURRENT_COMMIT'] + }) + return [Change.parse(c) for c in resp] + + def change_comments(self, change_id: str) -> dict[str, list[Comment]]: + resp = self.get(f'/changes/{change_id}/comments') + return { + filename: [Comment.parse(obj) for obj in comments] + for (filename, comments) in resp.items() + } + + +class FakeAPI(API): + """Fake Gerrit API that returns fake data.""" + + def __init__(self, fixtures: Traversable): + super().__init__(base=GERRIT_BASE) + self.fixtures = fixtures + + def fixture(self, name: str): + return json.loads(self.fixtures.joinpath(name).read_text()) + + def changes(self, per_page=25) -> list[Change]: + return [Change.parse(c) for c in self.fixture('changes.json')] + + def change_comments(self, change_id: str) -> dict[str, list[Comment]]: + return { + filename: [Comment.parse(obj) for obj in comments] + for filename, comments in self.fixture( + f'change-comments-{change_id}.json').items() + } + + +class GerritWatcher: + """Watches Gerrit changes and calls a callback on commit messages and all comments.""" + HandleComment = Callable[[MessageIsh], None] + + def __init__(self, api: API, callback: HandleComment, per_page=25): + self.api = api + self.per_page = per_page + self.already_processed = datetime(1970, 1, 1) + self.callback = callback + + def handle_change(self, change: Change): + self.callback( + MessageIsh(change_title=change.subject, + project=change.project, + message=change.commit_msg, + backlink=self.api.change_link(change.id), + cl_number=change.number, + kind='commit message')) + comments = self.api.change_comments(change.id) + + for _, filecomments in comments.items(): + for comment in filecomments: + self.callback( + MessageIsh(change_title=change.subject, + project=change.project, + message=comment.message, + backlink=self.api.change_link(change.id), + cl_number=change.number, + kind='comment')) + + def poll(self): + new_already_processed = self.already_processed + + # ordered by time descending + for change in self.api.changes(per_page=self.per_page): + new_already_processed = max(change.updated, new_already_processed) + + # got past the ones we have already seen, we're done + if change.updated <= self.already_processed: + break + + self.handle_change(change) + + self.already_processed = new_already_processed diff --git a/gerrit_linkbot/http.py b/gerrit_linkbot/http.py new file mode 100644 index 0000000..3f6da07 --- /dev/null +++ b/gerrit_linkbot/http.py @@ -0,0 +1,4 @@ +import requests + +session = requests.Session() +session.trust_env = False diff --git a/gerrit_linkbot/test_gerrit_linkbot.py b/gerrit_linkbot/test_gerrit_linkbot.py new file mode 100644 index 0000000..caf95ef --- /dev/null +++ b/gerrit_linkbot/test_gerrit_linkbot.py @@ -0,0 +1,121 @@ +import gerrit_linkbot +from gerrit_linkbot import RULES, CLMeta, CommentMeta, CLMention, IssueMatch, Service, gerrit, forgejo, testdata + + +def test_roundtrips(): + cls = CommentMeta(cls=[ + CLMention('https://gerrit.lix.systems/c/lix/+/1312', 1312, 'comment') + ], + cl_meta={1312: CLMeta('meow meow')}) + result = gerrit_linkbot.parse_comment(gerrit_linkbot.make_comment(cls)) + assert result + assert result == cls + + +def test_reasonable_comment(): + cls = CommentMeta(cls=[ + CLMention('https://gerrit.lix.systems/c/lix/+/1312', 1312, 'comment') + ], + cl_meta={1312: CLMeta('meow meow')}) + assert gerrit_linkbot.make_comment(cls) == \ + '\n' \ + '\n' \ + 'This issue was mentioned on Gerrit on the following CLs:\n' \ + '* comment in [cl/1312](https://gerrit.lix.systems/c/lix/+/1312) ("meow meow")\n' + + +def test_posts_comment(): + gerrit_api = gerrit.FakeAPI(testdata) + forgejo_api = forgejo.FakeAPI(testdata) + service = Service(gerrit_api, forgejo_api) + + service.step() + assert forgejo_api.posted_comments == [ + forgejo.PostedComment( + locator=forgejo.CommentLocator( + owner='lix-project', + repo='lix', + issue_id=361, + ), + comment_id=10000, + comment='\n' + '\n' + 'This issue was mentioned on Gerrit on the following CLs:\n' + '* commit message in ' + '[cl/1367](https://gerrit.lix.systems/c/lix/+/1367) ("repl: ' + 'implement tab completing :colon commands")\n', + ), + forgejo.EditedComment( + locator=forgejo.CommentLocator( + owner='lix-project', + repo='lix', + issue_id=361, + ), + comment='\n' + '\n' + 'This issue was mentioned on Gerrit on the following CLs:\n' + '* commit message in ' + '[cl/1367](https://gerrit.lix.systems/c/lix/+/1367) ("repl: ' + 'implement tab completing :colon commands")\n' + '* comment in ' + '[cl/1367](https://gerrit.lix.systems/c/lix/+/1367) ("repl: ' + 'implement tab completing :colon commands")\n', + edit_comment_id=10000, + ), + ] + + +def test_rules(): + + def flatten(xss): + return [x for xs in xss for x in xs] + + def apply_rules(text: str): + return [x for x in flatten(rule.apply(text) for rule in RULES) if x] + + assert apply_rules('meow #12 43') == [ + IssueMatch( + issue_num=12, + project_name=None, + ), + ] + + assert apply_rules( + 'meow a#123 #123 #345 https://git.lix.systems/meow/nya/issues/86 https://git.lix.systems/lix-project/meows/issues/98' + ) == [ + IssueMatch( + issue_num=123, + project_name=None, + ), + IssueMatch( + issue_num=345, + project_name=None, + ), + IssueMatch( + issue_num=98, + project_name='meows', + ), + ] + + assert apply_rules('lix#123 fj#583 lix#1337gj') == [ + IssueMatch( + issue_num=123, + project_name=None, + ), + IssueMatch( + issue_num=583, + project_name=None, + ) + ] diff --git a/gerrit_linkbot/testdata/change-comments-lix~1218.json b/gerrit_linkbot/testdata/change-comments-lix~1218.json new file mode 100644 index 0000000..ce9b7f4 --- /dev/null +++ b/gerrit_linkbot/testdata/change-comments-lix~1218.json @@ -0,0 +1,20 @@ +{ + "/PATCHSET_LEVEL": [ + { + "author": { + "_account_id": 1000000, + "name": "REDACTED", + "display_name": "REDACTED", + "email": "REDACTED", + "username": "REDACTED" + }, + "change_message_id": "3e509ef7892725ab6a1d7d7517b18f03dd38165a", + "unresolved": false, + "patch_set": 1, + "id": "48b11305_6b68a279", + "updated": "2024-05-28 00:48:49.000000000", + "message": "meow", + "commit_id": "179d058214c85dd3c66eabaa102cdda1a5984e77" + } + ] +} diff --git a/gerrit_linkbot/testdata/change-comments-lix~1367.json b/gerrit_linkbot/testdata/change-comments-lix~1367.json new file mode 100644 index 0000000..0fa1b76 --- /dev/null +++ b/gerrit_linkbot/testdata/change-comments-lix~1367.json @@ -0,0 +1,72 @@ +{ + "src/libcmd/repl.cc": [ + { + "author": { + "_account_id": 1000000, + "name": "REDACTED", + "email": "REDACTED", + "username": "REDACTED" + }, + "change_message_id": "c2dca70404921f9d489a00c0724beb7e04a3b3e2", + "unresolved": true, + "patch_set": 3, + "id": "63d6c6d7_cf08ec89", + "line": 375, + "updated": "2024-06-01 01:30:45.000000000", + "message": "meow fj#361 filing https://git.lix.systems/lix-project/lix/issues/361", + "commit_id": "8ab914d17ef000d374bf611e5152714bab60609e" + }, + { + "author": { + "_account_id": 1000000, + "name": "REDACTED", + "display_name": "REDACTED", + "email": "REDACTED", + "username": "REDACTED" + }, + "change_message_id": "24c98ffcb0815f72f625925f88d41ebf9b1c98e4", + "unresolved": false, + "patch_set": 3, + "id": "c0b833ef_cd81a56e", + "line": 375, + "in_reply_to": "63d6c6d7_cf08ec89", + "updated": "2024-06-01 21:56:38.000000000", + "message": "Done", + "commit_id": "8ab914d17ef000d374bf611e5152714bab60609e" + }, + { + "author": { + "_account_id": 1000000, + "name": "REDACTED", + "email": "REDACTED", + "username": "REDACTED" + }, + "change_message_id": "c2dca70404921f9d489a00c0724beb7e04a3b3e2", + "unresolved": true, + "patch_set": 3, + "id": "3a4b9716_91cddb9e", + "line": 382, + "updated": "2024-06-01 01:30:45.000000000", + "message": "likewise", + "commit_id": "8ab914d17ef000d374bf611e5152714bab60609e" + }, + { + "author": { + "_account_id": 1000000, + "name": "REDACTED", + "display_name": "REDACTED", + "email": "REDACTED", + "username": "REDACTED" + }, + "change_message_id": "24c98ffcb0815f72f625925f88d41ebf9b1c98e4", + "unresolved": false, + "patch_set": 3, + "id": "d6459c1f_c2933d52", + "line": 382, + "in_reply_to": "34b31771_6369334e", + "updated": "2024-06-01 21:56:38.000000000", + "message": "Done", + "commit_id": "8ab914d17ef000d374bf611e5152714bab60609e" + } + ] +} diff --git a/gerrit_linkbot/testdata/change-comments-lix~1369.json b/gerrit_linkbot/testdata/change-comments-lix~1369.json new file mode 100644 index 0000000..8a5cdb0 --- /dev/null +++ b/gerrit_linkbot/testdata/change-comments-lix~1369.json @@ -0,0 +1,20 @@ +{ + "/PATCHSET_LEVEL": [ + { + "author": { + "_account_id": 1000000, + "name": "REDACTED", + "display_name": "REDACTED", + "email": "REDACTED", + "username": "REDACTED" + }, + "change_message_id": "23d39257c72487af7423a23c900733daabbe8956", + "unresolved": false, + "patch_set": 1, + "id": "9d060685_23a11e7a", + "updated": "2024-06-03 01:41:54.000000000", + "message": "meow meow", + "commit_id": "80bf7ab4a37efb61fd8a943988a086f52237fea3" + } + ] +} diff --git a/gerrit_linkbot/testdata/changes.json b/gerrit_linkbot/testdata/changes.json new file mode 100644 index 0000000..7810c14 --- /dev/null +++ b/gerrit_linkbot/testdata/changes.json @@ -0,0 +1,374 @@ +[ + { + "id": "lix~1369", + "triplet_id": "lix~main~Idd8afe62f8591b5d8b70e258c5cefa09be4cab03", + "project": "lix", + "branch": "main", + "attention_set": { + "1000078": { + "account": { + "_account_id": 1000078 + }, + "last_update": "2024-06-01 14:38:38.000000000", + "reason": "A robot voted negatively on a label" + } + }, + "removed_from_attention_set": {}, + "hashtags": [], + "change_id": "Idd8afe62f8591b5d8b70e258c5cefa09be4cab03", + "subject": "libmain: add progress bar with multiple status lines", + "status": "NEW", + "created": "2024-06-01 14:10:06.000000000", + "updated": "2024-06-03 01:41:54.000000000", + "submit_type": "MERGE_IF_NECESSARY", + "mergeable": true, + "insertions": 92, + "deletions": 30, + "total_comment_count": 1, + "unresolved_comment_count": 0, + "has_review_started": true, + "meta_rev_id": "23d39257c72487af7423a23c900733daabbe8956", + "_number": 1369, + "_virtual_id_number": 1369, + "owner": { + "_account_id": 1000078 + }, + "current_revision": "80bf7ab4a37efb61fd8a943988a086f52237fea3", + "revisions": { + "80bf7ab4a37efb61fd8a943988a086f52237fea3": { + "kind": "REWORK", + "_number": 1, + "created": "2024-06-01 14:10:06.000000000", + "uploader": { + "_account_id": 1000078 + }, + "ref": "refs/changes/69/1369/1", + "fetch": { + "anonymous http": { + "url": "https://gerrit.lix.systems/lix", + "ref": "refs/changes/69/1369/1" + } + }, + "commit": { + "parents": [ + { + "commit": "5312e60be6aba84fab91e6d82af0b1aeccdfe981", + "subject": "Merge \"libfetchers: allow fetching gitlab refs with >1 commit\" into main" + } + ], + "author": { + "name": "REDACTED", + "email": "REDACTED", + "date": "2024-06-01 14:06:26.000000000", + "tz": 120 + }, + "committer": { + "name": "REDACTED", + "email": "REDACTED", + "date": "2024-06-01 14:10:00.000000000", + "tz": 120 + }, + "subject": "libmain: add progress bar with multiple status lines", + "message": "libmain: add progress bar with multiple status lines\n\nAdd the log-formats `multiline` and `multiline-with-logs` which offer\nmultiple current active building status lines.\n\nChange-Id: Idd8afe62f8591b5d8b70e258c5cefa09be4cab03\n" + }, + "branch": "refs/heads/main" + } + }, + "requirements": [], + "submit_records": [ + { + "rule_name": "gerrit~DefaultSubmitRule", + "status": "NOT_READY", + "labels": [ + { + "label": "Code-Review", + "status": "NEED" + }, + { + "label": "Verified", + "status": "MAY", + "applied_by": { + "_account_id": 1000008 + } + }, + { + "label": "Has-Tests", + "status": "MAY" + }, + { + "label": "Has-Release-Notes", + "status": "MAY" + } + ] + } + ] + }, + { + "id": "lix~1367", + "triplet_id": "lix~main~Id159d209c537443ef5e37a975982e8e12ce1f486", + "project": "lix", + "branch": "main", + "attention_set": { + "1000003": { + "account": { + "_account_id": 1000003 + }, + "last_update": "2024-06-01 21:21:17.000000000", + "reason": " replied on the change", + "reason_account": { + "_account_id": 1000001 + } + }, + "1000077": { + "account": { + "_account_id": 1000077 + }, + "last_update": "2024-06-02 17:48:45.000000000", + "reason": " replied on the change", + "reason_account": { + "_account_id": 1000001 + } + } + }, + "removed_from_attention_set": { + "1000001": { + "account": { + "_account_id": 1000001 + }, + "last_update": "2024-06-02 22:21:46.000000000", + "reason": " replied on the change", + "reason_account": { + "_account_id": 1000001 + } + } + }, + "hashtags": [], + "change_id": "Id159d209c537443ef5e37a975982e8e12ce1f486", + "subject": "repl: implement tab completing :colon commands", + "status": "NEW", + "created": "2024-05-31 23:40:53.000000000", + "updated": "2024-06-02 22:44:08.000000000", + "submit_type": "MERGE_IF_NECESSARY", + "mergeable": true, + "insertions": 71, + "deletions": 0, + "total_comment_count": 8, + "unresolved_comment_count": 0, + "has_review_started": true, + "meta_rev_id": "0b1c27db8f8854cd4bf0bc17db3ec6ee9ff07935", + "_number": 1367, + "_virtual_id_number": 1367, + "owner": { + "_account_id": 1000001 + }, + "current_revision": "5c424d24893e7f5270195074871b842e7ac7c057", + "revisions": { + "5c424d24893e7f5270195074871b842e7ac7c057": { + "kind": "TRIVIAL_REBASE", + "_number": 7, + "created": "2024-06-02 22:16:07.000000000", + "uploader": { + "_account_id": 1000001 + }, + "ref": "refs/changes/67/1367/7", + "fetch": { + "anonymous http": { + "url": "https://gerrit.lix.systems/lix", + "ref": "refs/changes/67/1367/7" + } + }, + "commit": { + "parents": [ + { + "commit": "c55e93ca23eae608f2ff80e940f35f5daa6b4f7f", + "subject": "Revert \"nix3: always use the same verbosity default (info)\"" + } + ], + "author": { + "name": "REDACTED", + "email": "REDACTED", + "date": "2024-06-01 00:29:10.000000000", + "tz": -360 + }, + "committer": { + "name": "REDACTED", + "email": "REDACTED", + "date": "2024-06-02 22:15:57.000000000", + "tz": -360 + }, + "subject": "repl: implement tab completing :colon commands", + "message": "repl: implement tab completing :colon commands\n\nThis uses a minor hack in which we check the rl_line_buffer global\nvariable to workaround editline not including the colon in its\ncompletion callback.\n\nFixes #361\n\nChange-Id: Id159d209c537443ef5e37a975982e8e12ce1f486\n" + }, + "branch": "refs/heads/main" + } + }, + "requirements": [], + "submit_records": [ + { + "rule_name": "gerrit~DefaultSubmitRule", + "status": "NOT_READY", + "labels": [ + { + "label": "Code-Review", + "status": "NEED" + }, + { + "label": "Verified", + "status": "MAY", + "applied_by": { + "_account_id": 1000008 + } + }, + { + "label": "Has-Tests", + "status": "MAY" + }, + { + "label": "Has-Release-Notes", + "status": "MAY" + } + ] + } + ] + }, + { + "id": "lix~1218", + "triplet_id": "lix~main~Ibe51416d9c7a6dd635c2282990224861adf1ceab", + "project": "lix", + "branch": "main", + "attention_set": { + "1000061": { + "account": { + "_account_id": 1000061 + }, + "last_update": "2024-05-28 00:48:49.000000000", + "reason": " replied on the change", + "reason_account": { + "_account_id": 1000001 + } + } + }, + "removed_from_attention_set": { + "1000003": { + "account": { + "_account_id": 1000003 + }, + "last_update": "2024-05-28 03:05:15.000000000", + "reason": " replied on the change", + "reason_account": { + "_account_id": 1000003 + } + }, + "1000005": { + "account": { + "_account_id": 1000005 + }, + "last_update": "2024-05-28 12:18:47.000000000", + "reason": " replied on the change", + "reason_account": { + "_account_id": 1000005 + } + } + }, + "hashtags": [ + "deferred-2.91" + ], + "change_id": "Ibe51416d9c7a6dd635c2282990224861adf1ceab", + "subject": "libstore/build: always treat seccomp setup failures as fatal", + "status": "NEW", + "created": "2024-05-27 15:17:55.000000000", + "updated": "2024-05-30 03:25:24.000000000", + "submit_type": "MERGE_IF_NECESSARY", + "mergeable": true, + "insertions": 1, + "deletions": 5, + "total_comment_count": 3, + "unresolved_comment_count": 0, + "has_review_started": true, + "meta_rev_id": "22b50cc2278823a37a81cc3b7a9a6aae2d893169", + "_number": 1218, + "_virtual_id_number": 1218, + "owner": { + "_account_id": 1000061 + }, + "current_revision": "080aa0be276d850ae19148b9e33b4e5620a87b56", + "revisions": { + "080aa0be276d850ae19148b9e33b4e5620a87b56": { + "kind": "TRIVIAL_REBASE", + "_number": 2, + "created": "2024-05-30 02:51:46.000000000", + "uploader": { + "_account_id": 1000061 + }, + "real_uploader": { + "_account_id": 1000003 + }, + "ref": "refs/changes/18/1218/2", + "fetch": { + "anonymous http": { + "url": "https://gerrit.lix.systems/lix", + "ref": "refs/changes/18/1218/2" + } + }, + "commit": { + "parents": [ + { + "commit": "218630a241d71ea5c136f72e1aaaf2299df6a0b1", + "subject": "Merge \"docs: enable non-default TOC folding\" into main" + } + ], + "author": { + "name": "REDACTED", + "email": "REDACTED", + "date": "2024-05-27 15:05:44.000000000", + "tz": 120 + }, + "committer": { + "name": "REDACTED", + "email": "REDACTED", + "date": "2024-05-30 02:51:46.000000000", + "tz": 0 + }, + "subject": "libstore/build: always treat seccomp setup failures as fatal", + "message": "libstore/build: always treat seccomp setup failures as fatal\n\nIn f047e4357b4f7ad66c2e476506bf35cab82e441e, I missed the behavior that if\nbuilding without a dedicated build user (i.e. in single-user setups), seccomp\nsetup failures are silently ignored. This was introduced without explanation 7\nyears ago (ff6becafa8efc2f7e6f2b9b889ba4adf20b8d524). Hopefully the only\nuse-case nowadays is causing spurious test suite successes when messing up the\nseccomp filter during development. Let's try removing it.\n\nChange-Id: Ibe51416d9c7a6dd635c2282990224861adf1ceab\n" + }, + "branch": "refs/heads/main", + "description": "Rebase" + } + }, + "requirements": [], + "submit_records": [ + { + "rule_name": "gerrit~DefaultSubmitRule", + "status": "NOT_READY", + "labels": [ + { + "label": "Code-Review", + "status": "NEED" + }, + { + "label": "Verified", + "status": "MAY", + "applied_by": { + "_account_id": 1000008 + } + }, + { + "label": "Has-Tests", + "status": "MAY", + "applied_by": { + "_account_id": 1000001 + } + }, + { + "label": "Has-Release-Notes", + "status": "MAY", + "applied_by": { + "_account_id": 1000001 + } + } + ] + } + ] + } +] diff --git a/gerrit_linkbot/testdata/issue-comments-lix-project-lix-361.json b/gerrit_linkbot/testdata/issue-comments-lix-project-lix-361.json new file mode 100644 index 0000000..fd4491c --- /dev/null +++ b/gerrit_linkbot/testdata/issue-comments-lix-project-lix-361.json @@ -0,0 +1,111 @@ +[ + { + "id": 4007, + "html_url": "https://git.lix.systems/lix-project/lix/issues/361#issuecomment-4007", + "pull_request_url": "", + "issue_url": "https://git.lix.systems/lix-project/lix/issues/361", + "user": { + "id": 3, + "login": "REDACTED", + "login_name": "", + "full_name": "REDACTED", + "email": "REDACTED", + "avatar_url": "REDACTED", + "language": "", + "is_admin": false, + "last_login": "0001-01-01T00:00:00Z", + "created": "2024-02-29T19:40:48Z", + "restricted": false, + "active": false, + "prohibit_login": false, + "location": "", + "pronouns": "REDACTED", + "website": "", + "description": "", + "visibility": "public", + "followers_count": 0, + "following_count": 1, + "starred_repos_count": 1, + "username": "REDACTED" + }, + "original_author": "", + "original_author_id": 0, + "body": "This shouldn't be too hard. The `:` commands currently are hardcoded (lol) in `src/libcmd/repl.cc` in `NixRepl::processLine()`. We just need to shove the list of possible colon commands in `NixRepl::completePrefix()` in `src/libcmd/repl.cc` behind a `prefix.starts_with(\":\")`\r\n\r\nWe are currently working on https://git.lix.systems/lix-project/lix/issues/306, but can tackle this after because this should really be a thing", + "assets": [], + "created_at": "2024-05-30T18:45:54Z", + "updated_at": "2024-05-30T18:45:54Z" + }, + { + "id": 4010, + "html_url": "https://git.lix.systems/lix-project/lix/issues/361#issuecomment-4010", + "pull_request_url": "", + "issue_url": "https://git.lix.systems/lix-project/lix/issues/361", + "user": { + "id": 9, + "login": "REDACTED", + "login_name": "", + "full_name": "REDACTED", + "email": "REDACTED", + "avatar_url": "REDACTED", + "language": "", + "is_admin": false, + "last_login": "0001-01-01T00:00:00Z", + "created": "2024-03-02T00:27:52Z", + "restricted": false, + "active": false, + "prohibit_login": false, + "location": "", + "pronouns": "", + "website": "REDACTED", + "description": "", + "visibility": "limited", + "followers_count": 0, + "following_count": 0, + "starred_repos_count": 1, + "username": "REDACTED" + }, + "original_author": "", + "original_author_id": 0, + "body": "we would love if there were a registry of repl commands, which would make this easier. ", + "assets": [], + "created_at": "2024-05-30T19:31:49Z", + "updated_at": "2024-05-30T19:31:49Z" + }, + { + "id": 4045, + "html_url": "https://git.lix.systems/lix-project/lix/issues/361#issuecomment-4045", + "pull_request_url": "", + "issue_url": "https://git.lix.systems/lix-project/lix/issues/361", + "user": { + "id": 3, + "login": "REDACTED", + "login_name": "", + "full_name": "REDACTED", + "email": "REDACTED", + "avatar_url": "REDACTED", + "language": "", + "is_admin": false, + "last_login": "0001-01-01T00:00:00Z", + "created": "2024-02-29T19:40:48Z", + "restricted": false, + "active": false, + "prohibit_login": false, + "location": "", + "pronouns": "she/her", + "website": "", + "description": "", + "visibility": "public", + "followers_count": 0, + "following_count": 1, + "starred_repos_count": 1, + "username": "REDACTED" + }, + "original_author": "", + "original_author_id": 0, + "body": "https://gerrit.lix.systems/c/lix/+/1367", + "assets": [], + "created_at": "2024-06-01T00:24:05Z", + "updated_at": "2024-06-01T00:24:05Z" + } +] + diff --git a/module.nix b/module.nix new file mode 100644 index 0000000..1517a42 --- /dev/null +++ b/module.nix @@ -0,0 +1,84 @@ +{ pkgs, config, lib, ... }: +let + package = pkgs.callPackage ./package.nix { }; + cfg = config.services.gerrit-linkbot; + + inherit (lib) types; +in +{ + options = { + services.gerrit-linkbot = { + enable = lib.mkEnableOption "gerrit link bot"; + # n.b. is not a mkPackageOption because we don't shove it into nixpkgs + # via overlay. + package = lib.mkOption { + type = types.package; + description = "Package for gerrit-linkbot"; + default = package; + }; + extraFlags = lib.mkOption { + type = types.listOf types.str; + }; + environmentFile = lib.mkOption { + type = types.nullOr types.path; + description = '' + Environment file for gerrit-linkbot. Requires FORGEJO_API_KEY set. + Use agenix or similar for this. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ cfg.package ]; + + systemd.services.gerrit-linkbot = + let + serviceDeps = lib.optionals config.services.gerrit.enable [ "gerrit.service" ] + ++ lib.optionals config.services.forgejo.enable [ "forgejo.service" ] ++ + [ "network-online.target" ]; + in + { + after = serviceDeps; + wants = serviceDeps; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Restart = "on-failure"; + RestartSec = "5s"; + RestartSteps = 10; + RestartMaxDelaySec = "600s"; + + DynamicUser = true; + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + PrivateTmp = true; + PrivateUsers = true; + PrivateDevices = true; + ProtectHome = true; + ProtectClock = true; + ProtectProc = "noaccess"; + ProcSubset = "pid"; + UMask = "0077"; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + ProtectHostname = true; + RestrictSUIDSGID = true; + RestrictRealtime = true; + RestrictNamespaces = true; + LockPersonality = true; + RemoveIPC = true; + SystemCallFilter = [ "@system-service" "~@privileged" ]; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ]; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + + EnvironmentFile = cfg.environmentFile; + + ExecStart = "${lib.getExe cfg.package}"; + }; + }; + }; +} diff --git a/package.nix b/package.nix new file mode 100644 index 0000000..87cca23 --- /dev/null +++ b/package.nix @@ -0,0 +1,35 @@ +{ lib, python3Packages }: +python3Packages.buildPythonApplication { + pname = "gerrit-linkbot"; + version = "0.0.0"; + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./gerrit_linkbot + ./pyproject.toml + ./setup.py + ]; + }; + + nativeCheckInputs = [ + python3Packages.pytestCheckHook + ]; + + build-system = with python3Packages; [ + setuptools + wheel + ]; + + dependencies = with python3Packages; [ + requests + ]; + + meta = { + description = "Gerrit Forgejo backlink bot"; + homepage = "https://git.lix.systems/lix-project/web-services"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ lf- ]; + platforms = lib.platforms.unix; + mainProgram = "gerrit-linkbot"; + }; +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6e8acb0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "gerrit-linkbot" +version = "0.0.0" +dependencies = [ + "requests", + "pytest", +] + +[project.scripts] +gerrit-linkbot = "gerrit_linkbot.__main__:main" + +[tool.setuptools] +packages = ["gerrit_linkbot", "gerrit_linkbot.testdata"] + +[tool.setuptools.package-data] +gerrit_linkbot = ["testdata/*"] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b908cbe --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +import setuptools + +setuptools.setup()