Normalize source URLs (#274)

* Normalize source URLs

* Fix URL normalisation

Author: senden9

src/sources/cfg.rs

@@ -1,4 +1,4 @@
-use super::OrgType;
+use super::{normalize_url, OrgType};
 use crate::{cfg, diag::FileId, LintLevel, Spanned};
 use serde::Deserialize;
 
@@ -113,7 +113,8 @@ impl cfg::UnvalidatedConfig for Config {
             .chain(self.allow_git.into_iter())
         {
             match url::Url::parse(aurl.as_ref()) {
-                Ok(url) => {
+                Ok(mut url) => {
+                    normalize_url(&mut url);
                     allowed_sources.push(UrlSpan {
                         value: url,
                         span: aurl.span,

src/sources/mod.rs

@@ -5,6 +5,7 @@ use crate::{
     diag::{Check, Diagnostic, Label, Pack, Severity},
     LintLevel,
 };
+use url::Url;
 
 pub fn check(ctx: crate::CheckCtx<'_, ValidConfig>, sender: crossbeam::channel::Sender<Pack>) {
     use bitvec::prelude::*;
@@ -34,6 +35,7 @@ pub fn check(ctx: crate::CheckCtx<'_, ValidConfig>, sender: crossbeam::channel::
             let mut url = source.url().clone();
             url.set_query(None);
             url.set_fragment(None);
+            normalize_url(&mut url);
             url
         };
 
@@ -219,6 +221,23 @@ pub fn check(ctx: crate::CheckCtx<'_, ValidConfig>, sender: crossbeam::channel::
     }
 }
 
+fn normalize_url(url: &mut Url) {
+    // Normalizes the URL so that different represatations can be compared to each other.
+    // At the moment we just remove a tailing `.git` but there are more possible optimisations.
+    // See https://github.com/rust-lang/cargo/blob/1f6c6bd5e7bbdf596f7e88e6db347af5268ab113/src/cargo/util/canonical_url.rs#L31-L57
+    // for what cargo does
+
+    let git_extension = ".git";
+    let needs_chopping = url.path().ends_with(&git_extension);
+    if needs_chopping {
+        let last = {
+            let last = url.path_segments().unwrap().last().unwrap();
+            last[..last.len() - git_extension.len()].to_owned()
+        };
+        url.path_segments_mut().unwrap().pop().push(&last);
+    }
+}
+
 #[derive(PartialEq, Debug)]
 pub enum OrgType {
     Github,

tests/cfg/sources.toml

@@ -6,7 +6,7 @@ allow-registry = [
     "https://sekretz.com/registry/index",
 ]
 allow-git = [
-    "https://notgithub.com/orgname/reponame",
+    "https://notgithub.com/orgname/reponame.git",
 ]
 [sources.allow-org]
 github = [