feat: Gerrit HTTP API and partial VCS implementation

This implements the Gerrit surface of the VCS API async API via HTTP.

TODO:

- Event streamer should go somewhere else
- We need to replace the missing features

There's some impedence mismatch on IDs but this can be solved by harder
refactors.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
This commit is contained in:
raito 2024-11-05 23:05:08 +01:00
parent 34de912604
commit 2db0eff088
8 changed files with 359 additions and 103 deletions

4
Cargo.lock generated
View file

@ -1114,9 +1114,9 @@ dependencies = [
[[package]]
name = "hashbrown"
version = "0.15.0"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3"
[[package]]
name = "heck"

View file

@ -2,8 +2,11 @@ use std::env;
use std::error::Error;
use std::path::Path;
use std::process;
use std::sync::RwLock;
use async_std::task;
use ofborg::config::VCSConfig;
use ofborg::tasks::evaluate::SupportedVCS;
use tracing::{error, info};
use ofborg::checkout;
@ -51,11 +54,16 @@ fn main() -> Result<(), Box<dyn Error>> {
no_wait: false,
})?;
let vcs_data = match cfg.vcs {
VCSConfig::GitHub => SupportedVCS::GitHub(RwLock::new(cfg.github_app_vendingmachine())),
VCSConfig::Gerrit => SupportedVCS::Gerrit,
};
let handle = easylapin::WorkerChannel(chan).consume(
tasks::evaluate::EvaluationWorker::new(
cloner,
&nix,
cfg.github_app_vendingmachine(),
vcs_data,
cfg.acl(),
cfg.runner.identity.clone(),
events,

View file

@ -12,6 +12,12 @@ use hubcaps::{Credentials, Github, InstallationTokenGenerator, JWTCredentials};
use serde::de::{self, Deserialize, Deserializer};
use tracing::{debug, error, info, warn};
#[derive(Serialize, Deserialize, Debug)]
pub enum VCSConfig {
GitHub,
Gerrit,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Config {
pub runner: RunnerConfig,
@ -19,10 +25,16 @@ pub struct Config {
pub checkout: CheckoutConfig,
pub nix: NixConfig,
pub rabbitmq: RabbitMqConfig,
pub github: Option<GithubConfig>,
pub github_app: Option<GithubAppConfig>,
pub vcs: VCSConfig,
pub pastebin: PastebinConfig,
pub log_storage: Option<LogStorage>,
// GitHub-specific configuration if vcs == GitHub.
pub github: Option<GithubConfig>,
pub github_app: Option<GithubAppConfig>,
// Gerrit-specific configuration if vcs == Gerrit.
pub gerrit: Option<GerritConfig>,
}
#[derive(Serialize, Deserialize, Debug)]
@ -58,6 +70,20 @@ pub struct NixConfig {
pub initial_heap_size: Option<String>,
}
const fn default_gerrit_ssh_port() -> u16 {
29418
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct GerritConfig {
// For all requests.
#[serde(deserialize_with = "deserialize_and_expand_pathbuf")]
pub ssh_private_key_file: PathBuf,
pub instance_uri: String,
#[serde(default = "default_gerrit_ssh_port")]
pub ssh_port: u16,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct GithubConfig {
pub token_file: PathBuf,

View file

@ -11,6 +11,7 @@ use crate::tasks::eval;
use crate::utils::pastebin::PersistedPastebin;
use crate::vcs::commit_status::{CommitStatus, CommitStatusError};
use crate::vcs::generic::{Issue, IssueState, State, VersionControlSystemAPI};
use crate::vcs::gerrit::http::GerritHTTPApi;
use crate::vcs::github::compat::GitHubAPI;
use crate::worker;
@ -21,10 +22,15 @@ use std::time::Instant;
use tracing::{debug_span, error, info, warn};
pub enum SupportedVCS {
GitHub(RwLock<GithubAppVendingMachine>),
Gerrit,
}
pub struct EvaluationWorker<E> {
cloner: checkout::CachedCloner,
nix: nix::Nix,
github_vend: RwLock<GithubAppVendingMachine>,
vcs: SupportedVCS,
acl: Acl,
identity: String,
events: E,
@ -35,7 +41,7 @@ impl<E: stats::SysEvents> EvaluationWorker<E> {
pub fn new(
cloner: checkout::CachedCloner,
nix: &nix::Nix,
github_vend: GithubAppVendingMachine,
vcs: SupportedVCS,
acl: Acl,
identity: String,
events: E,
@ -43,7 +49,7 @@ impl<E: stats::SysEvents> EvaluationWorker<E> {
EvaluationWorker {
cloner,
nix: nix.without_limited_supported_systems(),
github_vend: RwLock::new(github_vend),
vcs,
acl,
identity,
events,
@ -81,20 +87,21 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for EvaluationWorker<E>
let span = debug_span!("job", change_id = ?job.change.number);
let _enter = span.enter();
// TODO: introduce dynamic dispatcher instantiation here for the VCS API.
let mut vending_machine = self
.github_vend
.write()
.expect("Failed to get write lock on github vending machine");
let github_client = vending_machine
.for_repo(&job.repo.owner, &job.repo.name)
.expect("Failed to get a github client token");
let github_api = Rc::new(GitHubAPI::new(github_client.clone()));
let vcs_api: Rc<dyn VersionControlSystemAPI> = match self.vcs {
SupportedVCS::GitHub(ref vending_machine) => {
let mut vending_machine = vending_machine
.write()
.expect("Failed to get write lock on github vending machine");
let github_client = vending_machine
.for_repo(&job.repo.owner, &job.repo.name)
.expect("Failed to get a github client token");
Rc::new(GitHubAPI::new(github_client.clone()))
}
SupportedVCS::Gerrit => Rc::new(GerritHTTPApi),
};
OneEval::new(
github_api,
vcs_api,
&self.nix,
&self.acl,
&mut self.events,

View file

@ -4,50 +4,65 @@ use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug)]
pub struct Account {
pub name: Option<String>, // User's full name, if configured
pub email: Option<String>, // User's preferred email address
pub username: Option<String>, // User's username, if configured
pub name: Option<String>, // User's full name, if configured
pub email: Option<String>, // User's preferred email address
pub username: Option<String>, // User's username, if configured
}
impl From<Vec<Account>> for crate::vcs::generic::ChangeReviewers {
fn from(value: Vec<Account>) -> Self {
// FIXME: I don't think Gerrit remembers when we added a group instead of entities.
crate::vcs::generic::ChangeReviewers {
entity_reviewers: value
.into_iter()
// FIXME: this is a… quite the assumption, let's relax it by having at _least_ one
// identity identifier.
.map(|a| a.username.expect("Expected username"))
.collect(),
team_reviewers: vec![],
}
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Approval {
pub r#type: String, // Internal name of the approval
pub description: String, // Human-readable category of the approval
pub value: i32, // Value assigned by the approval (usually a numerical score)
pub r#type: String, // Internal name of the approval
pub description: String, // Human-readable category of the approval
pub value: i32, // Value assigned by the approval (usually a numerical score)
#[serde(rename = "oldValue")]
pub old_value: Option<i32>, // Previous approval score, if present
pub old_value: Option<i32>, // Previous approval score, if present
#[serde(rename = "grantedOn")]
pub granted_on: u64, // Time in seconds since the UNIX epoch
pub by: Account, // Reviewer of the patch set
pub granted_on: u64, // Time in seconds since the UNIX epoch
pub by: Account, // Reviewer of the patch set
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "UPPERCASE")]
pub enum ChangeType {
Added, // The file is being created/introduced by this patch
Modified, // The file already exists and has updated content
Deleted, // The file existed, but is being removed by this patch
Renamed, // The file is renamed
Copied, // The file is copied from another file
Rewrite, // The file was rewritten
Added, // The file is being created/introduced by this patch
Modified, // The file already exists and has updated content
Deleted, // The file existed, but is being removed by this patch
Renamed, // The file is renamed
Copied, // The file is copied from another file
Rewrite, // The file was rewritten
}
#[derive(Serialize, Deserialize, Debug)]
pub struct PatchFile {
pub file: String, // Name of the file, or new name if renamed
pub file: String, // Name of the file, or new name if renamed
#[serde(rename = "fileOld")]
pub file_old: Option<String>, // Old name of the file, if renamed
pub r#type: ChangeType, // Type of change (ADDED, MODIFIED, DELETED, etc.)
pub insertions: i64, // Number of insertions in the patch
pub deletions: i64, // Number of deletions in the patch
pub file_old: Option<String>, // Old name of the file, if renamed
pub r#type: ChangeType, // Type of change (ADDED, MODIFIED, DELETED, etc.)
pub insertions: i64, // Number of insertions in the patch
pub deletions: i64, // Number of deletions in the patch
}
#[derive(Serialize, Deserialize, Debug)]
pub struct PatchSetComment {
pub file: String, // Name of the file on which the comment was added
pub line: u64, // Line number at which the comment was added
pub reviewer: Account, // Account that added the comment
pub message: String, // Comment text
pub file: String, // Name of the file on which the comment was added
pub line: u64, // Line number at which the comment was added
pub reviewer: Account, // Account that added the comment
pub message: String, // Comment text
}
#[derive(Serialize, Deserialize, Debug)]
@ -63,30 +78,30 @@ pub enum Kind {
#[derive(Serialize, Deserialize, Debug)]
pub struct PatchSet {
pub number: u64, // Patchset number
pub revision: String, // Git commit for this patchset
pub parents: Vec<String>, // List of parent revisions
pub r#ref: String, // Git reference pointing at the revision
pub uploader: Account, // Uploader of the patchset
pub author: Account, // Author of the patchset
pub number: u64, // Patchset number
pub revision: String, // Git commit for this patchset
pub parents: Vec<String>, // List of parent revisions
pub r#ref: String, // Git reference pointing at the revision
pub uploader: Account, // Uploader of the patchset
pub author: Account, // Author of the patchset
#[serde(rename = "createdOn")]
pub created_on: u64, // Time in seconds since the UNIX epoch
pub kind: Kind, // Kind of change ("REWORK", "TRIVIAL_REBASE", etc.)
pub approvals: Vec<Approval>, // Approvals granted
pub comments: Vec<PatchSetComment>, // All comments for this patchset
pub files: Vec<String>, // All changed files in this patchset
pub created_on: u64, // Time in seconds since the UNIX epoch
pub kind: Kind, // Kind of change ("REWORK", "TRIVIAL_REBASE", etc.)
pub approvals: Vec<Approval>, // Approvals granted
pub comments: Vec<PatchSetComment>, // All comments for this patchset
pub files: Vec<String>, // All changed files in this patchset
#[serde(rename = "sizeInsertions")]
pub size_insertions: i64, // Size of insertions
pub size_insertions: i64, // Size of insertions
#[serde(rename = "sizeDeletions")]
pub size_deletions: i64, // Size of deletions
pub size_deletions: i64, // Size of deletions
}
#[derive(Serialize, Deserialize, Debug)]
pub struct ReviewerMessage {
#[serde(rename = "message")]
pub comment_text: String, // Comment text added by the reviewer
pub timestamp: u64, // Time in seconds since the UNIX epoch when this comment was added
pub reviewer: Account, // Account of the reviewer who added the comment
pub comment_text: String, // Comment text added by the reviewer
pub timestamp: u64, // Time in seconds since the UNIX epoch when this comment was added
pub reviewer: Account, // Account of the reviewer who added the comment
}
#[derive(Serialize, Deserialize, Debug)]
@ -97,10 +112,10 @@ pub struct TrackingId {
#[derive(Serialize, Deserialize, Debug)]
pub struct ChangeDependency {
pub id: String, // Change identifier
pub number: u64, // Change number
pub revision: String, // Patchset revision
pub r#ref: String, // Ref name
pub id: String, // Change identifier
pub number: u64, // Change number
pub revision: String, // Patchset revision
pub r#ref: String, // Ref name
#[serde(rename = "isCurrentPatchSet")]
pub is_current_patch_set: bool, // If the revision is the current patchset of the change
}
@ -108,49 +123,49 @@ pub struct ChangeDependency {
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "UPPERCASE")]
pub enum ChangeStatus {
New, // Change is still being reviewed
Merged, // Change has been merged to its branch
Abandoned, // Change was abandoned by its owner or administrator
New, // Change is still being reviewed
Merged, // Change has been merged to its branch
Abandoned, // Change was abandoned by its owner or administrator
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "UPPERCASE")]
pub enum SubmitStatus {
Ok, // The change is ready for submission or already submitted
NotReady, // The change is missing a required label
RuleError, // An internal server error occurred preventing computation
Ok, // The change is ready for submission or already submitted
NotReady, // The change is missing a required label
RuleError, // An internal server error occurred preventing computation
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Label {
pub label: String, // Name of the label
pub status: LabelStatus, // Status of the label
pub by: Account, // The account that applied the label
pub label: String, // Name of the label
pub status: LabelStatus, // Status of the label
pub by: Account, // The account that applied the label
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "UPPERCASE")]
pub enum LabelStatus {
Ok, // This label provides what is necessary for submission
Reject, // This label prevents the change from being submitted
Need, // The label is required for submission but has not been satisfied
May, // The label may be set but isnt necessary for submission
Impossible, // The label is required but impossible to complete
Ok, // This label provides what is necessary for submission
Reject, // This label prevents the change from being submitted
Need, // The label is required for submission but has not been satisfied
May, // The label may be set but isnt necessary for submission
Impossible, // The label is required but impossible to complete
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Requirement {
#[serde(rename = "fallbackText")]
pub fallback_text: String, // Human-readable description of the requirement
pub r#type: String, // Alphanumerical string identifying the requirement
pub fallback_text: String, // Human-readable description of the requirement
pub r#type: String, // Alphanumerical string identifying the requirement
pub data: Option<HashMap<String, String>>, // Additional key-value data linked to this requirement
}
#[derive(Serialize, Deserialize, Debug)]
pub struct SubmitRecord {
pub status: SubmitStatus, // Current submit status
pub labels: Option<Vec<Label>>, // State of each code review label attribute
pub requirements: Vec<Requirement>, // Requirements for submission
pub status: SubmitStatus, // Current submit status
pub labels: Option<Vec<Label>>, // State of each code review label attribute
pub requirements: Vec<Requirement>, // Requirements for submission
}
#[derive(Serialize, Deserialize, Debug)]
@ -173,7 +188,7 @@ pub struct Change {
pub open: bool,
pub status: ChangeStatus, // "NEW", "MERGED", or "ABANDONED"
pub private: bool,
pub wip: bool, // Work in progress
pub wip: bool, // Work in progress
pub comments: Vec<ReviewerMessage>, // Inline/file comments
#[serde(rename = "trackingIds")]
pub tracking_ids: Vec<TrackingId>, // Links to issue tracking systems
@ -191,14 +206,26 @@ pub struct Change {
pub all_reviewers: Vec<Account>, // List of all reviewers
}
impl From<Change> for crate::message::Change {
fn from(value: Change) -> Self {
Self {
target_branch: Some(value.branch),
// While the change number is deprecated, we actually need it.
// FIXME: enforce type level checking of this.
number: value.change_number.unwrap(),
head_sha: value.current_patch_set.revision,
}
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct RefUpdate {
#[serde(rename = "oldRev")]
pub old_rev: String, // The old value of the ref, prior to the update
pub old_rev: String, // The old value of the ref, prior to the update
#[serde(rename = "newRev")]
pub new_rev: String, // The new value the ref was updated to
pub ref_name: String, // Full ref name within the project
pub project: String, // Project path in Gerrit
pub new_rev: String, // The new value the ref was updated to
pub ref_name: String, // Full ref name within the project
pub project: String, // Project path in Gerrit
}
#[derive(Serialize, Deserialize, Debug)]
@ -209,25 +236,25 @@ pub enum GerritStreamEvent {
patch_set: PatchSet,
abandoner: Account,
reason: String,
event_created_on: u64
event_created_on: u64,
},
ChangeDeleted {
change: Change,
deleter: Account
deleter: Account,
},
ChangeMerged {
change: Change,
patch_set: PatchSet,
submitter: Account,
new_rev: String,
event_created_on: u64
event_created_on: u64,
},
ChangeRestored {
change: Change,
patch_set: PatchSet,
restorer: Account,
reason: String,
event_created_on: u64
event_created_on: u64,
},
CommentAdded {
change: Change,
@ -235,7 +262,7 @@ pub enum GerritStreamEvent {
author: Account,
approvals: Vec<Approval>,
comment: String,
event_created_on: u64
event_created_on: u64,
},
DroppedOutput,
HashtagsChanged {
@ -249,30 +276,30 @@ pub enum GerritStreamEvent {
ProjectCreated {
project_name: String,
project_head: String,
event_created_on: u64
event_created_on: u64,
},
PatchSetCreated {
change: Change,
patch_set: PatchSet,
uploader: Account,
event_created_on: u64
event_created_on: u64,
},
RefUpdated {
submitter: Account,
ref_update: RefUpdate,
event_created_on: u64
event_created_on: u64,
},
BatchRefUpdated {
submitter: Account,
ref_updates: Vec<RefUpdate>,
event_created_on: u64
event_created_on: u64,
},
ReviewerAdded {
change: Change,
patch_set: PatchSet,
reviewer: Account,
adder: Account,
event_created_on: u64
event_created_on: u64,
},
ReviewerDeleted {
change: Change,
@ -288,19 +315,19 @@ pub enum GerritStreamEvent {
old_topic: Option<String>,
new_topic: Option<String>,
changer: Account,
event_created_on: u64
event_created_on: u64,
},
WorkInProgressStateChanged {
change: Change,
patch_set: PatchSet,
changer: Account,
event_created_on: u64
event_created_on: u64,
},
PrivateStateChanged {
change: Change,
patch_set: PatchSet,
changer: Account,
event_created_on: u64
event_created_on: u64,
},
VoteDeleted {
change: Change,
@ -313,6 +340,6 @@ pub enum GerritStreamEvent {
ProjectHeadUpdate {
old_head: String,
new_head: String,
event_created_on: u64
}
event_created_on: u64,
},
}

View file

@ -0,0 +1,49 @@
//! REST API bindings for Gerrit
//! TODO:
//! - trace IDs support
//! - label support
use super::data_structures::{Account, Change};
pub struct GerritHTTPApi;
impl GerritHTTPApi {
// async fn get_project(&self, project_name: &str) -> Project {}
/// Fetches all changes according to the query and the given limit.
/// This will default to 60 changes by default.
pub(crate) async fn list_changes(&self, _query: &str, _limit: Option<u64>) -> Vec<Change> {
Default::default()
}
/// Fetch the latest change ID for a given project and CL number.
pub(crate) async fn get_change_id(&self, _project_name: &str, _cl_number: u64) -> String {
"".to_owned()
}
/// Fetch a given change according to the change ID (not the CL number).
pub(crate) async fn get_change(&self, _change_id: &str) -> Option<Change> {
Default::default()
}
/// Set additional and remove certain hashtags for a given change ID (not the CL number).
pub(crate) async fn set_hashtags(
&self,
_change_id: &str,
_add: &[String],
_remove: &[String],
) -> Vec<String> {
Default::default()
}
/// List all reviewers on a given change ID (not the CL number).
pub(crate) async fn list_reviewers(&self, _change_id: &str) -> Vec<Account> {
Default::default()
}
/// Set reviewers and a message on a given change ID (not the CL number).
pub(crate) async fn set_reviewers(
&self,
_change_id: &str,
_message: &str,
_reviewers: Vec<Account>,
) {
}
}

View file

@ -0,0 +1,137 @@
//! Implementation of the VCS API for Gerrit
//! This uses the HTTP API.
use futures_util::FutureExt;
use crate::vcs::generic::VersionControlSystemAPI;
use super::{data_structures::Account, http::GerritHTTPApi};
impl VersionControlSystemAPI for GerritHTTPApi {
// The next three APIs are todo!() because they cannot be implemented in Gerrit.
// Gerrit does not offer any way to get this information out.
// GerritHTTPApi needs to return something like Unsupported
// and we need to compose a GerritHTTPApi with a GerritForge which contains an implementation
// of check statuses and commit statuses and an issue tracker.
fn create_check_statuses(
&self,
_repo: &crate::message::Repo,
_checks: Vec<crate::vcs::generic::CheckRunOptions>,
) -> futures_util::future::BoxFuture<()> {
todo!();
}
fn create_commit_statuses(
&self,
_repo: &crate::message::Repo,
_sha: String,
_state: crate::vcs::generic::State,
_context: String,
_description: String,
_target_url: String,
) -> futures_util::future::BoxFuture<Result<(), crate::vcs::commit_status::CommitStatusError>>
{
todo!();
}
fn get_issue(
&self,
_repo: &crate::message::Repo,
_number: u64,
) -> futures_util::future::BoxFuture<Result<crate::vcs::generic::Issue, String>> {
todo!();
}
fn get_repository(&self, _repo: &crate::message::Repo) -> crate::vcs::generic::Repository {
todo!();
}
fn get_changes(
&self,
repo: &crate::message::Repo,
) -> futures_util::future::BoxFuture<Vec<crate::message::Change>> {
let repo_name = repo.name.to_owned();
async move {
self.list_changes(&format!("project:{}", &repo_name), None)
.await
.into_iter()
.map(|c| c.into())
.collect()
}
.boxed()
}
fn get_change(
&self,
repo: &crate::message::Repo,
number: u64,
) -> futures_util::future::BoxFuture<Option<crate::message::Change>> {
let repo_name = repo.name.to_owned();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
GerritHTTPApi::get_change(&self, &change_id)
.await
.map(|c| c.into())
}
.boxed()
}
fn update_labels(
&self,
repo: &crate::message::Repo,
number: u64,
add: &[String],
remove: &[String],
) -> futures_util::future::BoxFuture<()> {
let add = add.to_owned();
let remove = remove.to_owned();
let repo_name = repo.name.to_owned();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
self.set_hashtags(&change_id, &add, &remove).await;
}
.boxed()
}
fn get_existing_reviewers(
&self,
repo: &crate::message::Repo,
number: u64,
) -> futures_util::future::BoxFuture<crate::vcs::generic::ChangeReviewers> {
let repo_name = repo.name.to_owned();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
self.list_reviewers(&change_id).await.into()
}
.boxed()
}
fn request_reviewers(
&self,
repo: &crate::message::Repo,
number: u64,
entity_reviewers: Vec<String>,
// FIXME: support group reviews
_team_reviewers: Vec<String>,
) -> futures_util::future::BoxFuture<()> {
let repo_name = repo.name.to_owned();
async move {
let change_id = self.get_change_id(&repo_name, number).await;
self.set_reviewers(
&change_id,
"Automatic reviewer request",
entity_reviewers
.into_iter()
.map(|reviewer| Account {
username: Some(reviewer),
email: None,
name: None,
})
.collect(),
)
.await
}
.boxed()
}
}

View file

@ -1,3 +1,5 @@
pub mod checks;
pub mod data_structures;
pub mod http;
pub mod r#impl;
// pub mod events;