diff --git a/ofborg/src/maintainers.rs b/ofborg/src/maintainers.rs
index 4642ce9..0d257d9 100644
--- a/ofborg/src/maintainers.rs
+++ b/ofborg/src/maintainers.rs
@@ -1,20 +1,22 @@
 use ofborg::nix::Nix;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::io::Write;
 use std::path::Path;
 use tempfile::NamedTempFile;
 
 #[derive(Deserialize, Debug, Eq, PartialEq)]
 pub struct ImpactedMaintainers(HashMap<Maintainer, Vec<Package>>);
-#[derive(Deserialize, Debug, Eq, PartialEq, Hash)]
-struct Maintainer(String);
+pub struct MaintainersByPackage(pub HashMap<Package, HashSet<Maintainer>>);
+
+#[derive(Deserialize, Clone, Debug, Eq, PartialEq, Hash)]
+pub struct Maintainer(String);
 impl<'a> From<&'a str> for Maintainer {
     fn from(name: &'a str) -> Maintainer {
         Maintainer(name.to_owned())
     }
 }
-#[derive(Deserialize, Debug, Eq, PartialEq, Hash)]
-struct Package(String);
+#[derive(Deserialize, Clone, Debug, Eq, PartialEq, Hash)]
+pub struct Package(String);
 impl<'a> From<&'a str> for Package {
     fn from(name: &'a str) -> Package {
         Package(name.to_owned())
@@ -80,6 +82,22 @@ impl ImpactedMaintainers {
             .map(|(maintainer, _)| maintainer.0.clone())
             .collect()
     }
+
+    pub fn maintainers_by_package(&self) -> MaintainersByPackage {
+        let mut bypkg = MaintainersByPackage(HashMap::new());
+
+        for (maintainer, packages) in self.0.iter() {
+            for package in packages.iter() {
+                bypkg
+                    .0
+                    .entry(package.clone())
+                    .or_insert_with(HashSet::new)
+                    .insert(maintainer.clone());
+            }
+        }
+
+        bypkg
+    }
 }
 
 impl std::fmt::Display for ImpactedMaintainers {
diff --git a/ofborg/src/tagger.rs b/ofborg/src/tagger.rs
index 9982e49..3d3bc81 100644
--- a/ofborg/src/tagger.rs
+++ b/ofborg/src/tagger.rs
@@ -1,3 +1,4 @@
+use crate::maintainers::{Maintainer, MaintainersByPackage};
 use ofborg::outpathdiff::PackageArch;
 use ofborg::tasks;
 use std::collections::HashMap;
@@ -269,15 +270,27 @@ impl MaintainerPRTagger {
         Default::default()
     }
 
-    pub fn record_maintainer(&mut self, pr_submitter: &str, identified_maintainers: &[String]) {
-        let mut compare_to: Vec<String> = identified_maintainers.to_vec().clone();
-        compare_to.sort();
-        compare_to.dedup();
+    pub fn record_maintainer(
+        &mut self,
+        pr_submitter: &str,
+        identified_maintainers: &MaintainersByPackage,
+    ) {
+        let submitter = Maintainer::from(pr_submitter);
 
-        if compare_to.len() == 1 && compare_to.contains(&pr_submitter.to_string()) {
-            self.selected
-                .push(String::from("11.by: package-maintainer"));
+        if identified_maintainers.0.is_empty() {
+            // No packages -> not from the maintainer
+            return;
         }
+
+        for (_package, maintainers) in identified_maintainers.0.iter() {
+            if !maintainers.contains(&submitter) {
+                // One of the packages is not maintained by this submitter
+                return;
+            }
+        }
+
+        self.selected
+            .push(String::from("11.by: package-maintainer"));
     }
 
     pub fn tags_to_add(&self) -> Vec<String> {
diff --git a/ofborg/src/tasks/massrebuilder.rs b/ofborg/src/tasks/massrebuilder.rs
index 02a9ead..bc592f1 100644
--- a/ofborg/src/tasks/massrebuilder.rs
+++ b/ofborg/src/tasks/massrebuilder.rs
@@ -572,7 +572,8 @@ impl<E: stats::SysEvents + 'static> worker::SimpleWorker for MassRebuildWorker<E
                     if let Ok(ref maint) = m {
                         request_reviews(&maint, &pull);
                         let mut maint_tagger = MaintainerPRTagger::new();
-                        maint_tagger.record_maintainer(&issue.user.login, &maint.maintainers());
+                        maint_tagger
+                            .record_maintainer(&issue.user.login, &maint.maintainers_by_package());
                         update_labels(
                             &issue_ref,
                             &maint_tagger.tags_to_add(),