optimize IDNumber selection, avoid conflicts, remove site vars, rewrite custom mapping, add tests

.gitignore

@@ -1,6 +1,3 @@
-# don't track custom user mappings
-resources/custom_user_mappings/*.csv
-
 # vscode project files
 .vscode/
 
@@ -14,6 +11,10 @@ composer.lock
 
 # don't track site configs
 deployment/*
+!deployment/overrides/
+!deployment/overrides/phpunit/
+!deployment/overrides/phpunit/config/
+!deployment/overrides/phpunit/config/config.ini
 !deployment/**/README.md
 
 .phpunit.result.cache

README.md

@@ -62,6 +62,7 @@ See the Docker Compose environment (`tools/docker-dev/`) for an (unsafe for prod
     * Make sure this file is not world readable!
 1. If using mulitple domains, create `deployment/overrides/<domain>/config/config.ini`
 1. If using custom UIDNumber/GIDNumber mappings, create `deployment/custom_user_mappings/*.csv`
+    * The 1st column is UID, the 2nd column is both UIDNumber and GIDNumber
 1. Add logos to `webroot/assets/footer_logos/`
 
 ## Integration
@@ -105,6 +106,15 @@ rm "$prod" && ln -s "$old" "$prod"
 
 ### Version-specific update instructions:
 
+### 1.2 -> 1.3
+* SQL:
+    * remove the `sitevars` table
+* `defaults/config.ini.default` has some new fields that need to be overriden:
+    * `offset_UIDGID`
+    * `offset_PIGID`
+    * `offset_ORGGID`
+* `custom_user_mappings` can no longer match with just the 1st segment of the logged in user's UID, an exact match is required
+
 ### 1.2.0 -> 1.2.1
 * SQL:
     * Add new columns to the `requests` table:

defaults/config.ini.default

@@ -18,6 +18,7 @@ account_policy_url = "https://github.com" ; this can be external or a portal pag
 uri = "ldap://identity"  ; URI of remote LDAP server
 user = "cn=admin,dc=unityhpc,dc=test"  ; Admin bind DN LDAP user
 pass = "password"  ; Admin bind password
+custom_user_mappings_dir = "deployment/custom_user_mappings" ; for internal use only
 basedn = "dc=unityhpc,dc=test"  ; Base search DN
 user_ou = "ou=users,dc=unityhpc,dc=test"  ; User organizational unit (may contain more than user group)
 user_group = "cn=unityusers,dc=unityhpc,dc=test" ; User group
@@ -26,6 +27,9 @@ pigroup_ou = "ou=pi_groups,dc=unityhpc,dc=test"  ; PI Group organizational unit
 orggroup_ou = "ou=org_groups,dc=unityhpc,dc=test"  ; ORG group organizational unit
 admin_group = "cn=web_admins,dc=unityhpc,dc=test"  ; admin dn (members of this group are admins on the web portal)
 def_user_shell = "/bin/bash"  ; Default shell for new users
+offset_UIDGID = 1000000 ; start point when allocating new UID/GID pairs for a new user
+offset_PIGID = 2000000 ; start point when allocating new GID for a new PI group
+offset_ORGGID = 3000000 ; start point when allocating new GID for a new org group
 
 [sql]
 host = "sql"  ; mariadb hostname

deployment/overrides/phpunit/config/config.ini

@@ -0,0 +1,2 @@
+[ldap]
+custom_user_mappings_dir = "test/custom_user_mappings"

resources/lib/UnityGroup.php

@@ -464,7 +464,7 @@ private function init()
     {
         $owner = $this->getOwner();
         assert(!$this->entry->exists());
-        $nextGID = $this->LDAP->getNextPiGIDNumber($this->SQL);
+        $nextGID = $this->LDAP->getNextPIGIDNumber();
         $this->entry->setAttribute("objectclass", UnityLDAP::POSIX_GROUP_CLASS);
         $this->entry->setAttribute("gidnumber", strval($nextGID));
         $this->entry->setAttribute("memberuid", array($owner->uid));

resources/lib/UnityLDAP.php

@@ -24,7 +24,11 @@ class UnityLDAP extends ldapConn
     "top"
     );
 
-    private $custom_mappings_path = __DIR__ . "/../../deployment/custom_user_mappings";
+    private $custom_mappings_path = CONFIG["ldap"]["custom_user_mappings_dir"];
+    private $def_user_shell = CONFIG["ldap"]["def_user_shell"];
+    private $offset_UIDGID = CONFIG["ldap"]["offset_UIDGID"];
+    private $offset_PIGID = CONFIG["ldap"]["offset_PIGID"];
+    private $offset_ORGGID = CONFIG["ldap"]["offset_ORGGID"];
 
     // Instance vars for various ldapEntry objects
     private $baseOU;
@@ -34,7 +38,6 @@ class UnityLDAP extends ldapConn
     private $org_groupOU;
     private $adminGroup;
     private $userGroup;
-    private $def_user_shell;
 
     public function __construct()
     {
@@ -46,7 +49,6 @@ public function __construct()
         $this->org_groupOU = $this->getEntry(CONFIG["ldap"]["orggroup_ou"]);
         $this->adminGroup = $this->getEntry(CONFIG["ldap"]["admin_group"]);
         $this->userGroup = $this->getEntry(CONFIG["ldap"]["user_group"]);
-        $this->def_user_shell = CONFIG["ldap"]["def_user_shell"];
     }
 
     public function getUserOU()
@@ -84,104 +86,100 @@ public function getDefUserShell()
         return $this->def_user_shell;
     }
 
-    public function getNextUIDNumber($UnitySQL)
+    public function getNextUIDGIDNumber($uid)
     {
-        $max_uid = $UnitySQL->getSiteVar('MAX_UID');
-        $new_uid = $max_uid + 1;
-
-        while ($this->IDNumInUse($new_uid)) {
-            $new_uid++;
+        $IDNumsInUse = array_merge($this->getAllUIDNumbersInUse(), $this->getAllGIDNumbersInUse());
+        $start = $this->offset_UIDGID;
+        $customIDMappings = $this->getCustomIDMappings();
+        $customMappedID = $customIDMappings[$uid] ?? null;
+        if (!is_null($customMappedID) && !in_array($customMappedID, $IDNumsInUse)) {
+            return $customMappedID;
         }
-
-        $UnitySQL->updateSiteVar('MAX_UID', $new_uid);
-
-        return $new_uid;
+        if (!is_null($customMappedID) && in_array($customMappedID, $IDNumsInUse)) {
+            UnitySite::errorLog(
+                "warning",
+                "user '$uid' has a custom mapped IDNumber $customMappedID but it's already in use!",
+            );
+        }
+        return $this->getNextIDNumber($start, $IDNumsInUse);
     }
 
-    public function getNextPiGIDNumber($UnitySQL)
+    public function getNextPIGIDNumber()
     {
-        $max_pigid = $UnitySQL->getSiteVar('MAX_PIGID');
-        $new_pigid = $max_pigid + 1;
-
-        while ($this->IDNumInUse($new_pigid)) {
-            $new_pigid++;
-        }
-
-        $UnitySQL->updateSiteVar('MAX_PIGID', $new_pigid);
-
-        return $new_pigid;
+        $IDNumsInUse = $this->getAllGIDNumbersInUse();
+        $start = $this->offset_PIGID;
+        return $this->getNextIDNumber($start, $IDNumsInUse);
     }
 
-    public function getNextOrgGIDNumber($UnitySQL)
+    public function getNextOrgGIDNumber()
     {
-        $max_gid = $UnitySQL->getSiteVar('MAX_GID');
-        $new_gid = $max_gid + 1;
-
-        while ($this->IDNumInUse($new_gid)) {
-            $new_gid++;
-        }
-
-        $UnitySQL->updateSiteVar('MAX_GID', $new_gid);
-
-        return $new_gid;
+        $IDNumsInUse = array_values($this->getCustomIDMappings());
+        $start = $this->offset_ORGGID;
+        return $this->getNextIDNumber($start, $IDNumsInUse);
     }
 
-    private function IDNumInUse($id)
+    private function isIDNumberForbidden($id)
     {
         // 0-99 are probably going to be used for local system accounts instead of LDAP accounts
         // 100-999, 60000-64999 are reserved for debian packages
-        if (($id <= 999) || ($id >= 60000 && $id <= 64999)) {
-            return true;
-        }
-        $users = $this->userOU->getChildrenArray([], true);
-        foreach ($users as $user) {
-            if ($user["uidnumber"][0] == $id) {
-                return true;
-            }
-        }
-        $pi_groups = $this->pi_groupOU->getChildrenArray(["gidnumber"], true);
-        foreach ($pi_groups as $pi_group) {
-            if ($pi_group["gidnumber"][0] == $id) {
-                return true;
-            }
-        }
-        $groups = $this->groupOU->getChildrenArray(["gidnumber"], true);
-        foreach ($groups as $group) {
-            if ($group["gidnumber"][0] == $id) {
-                return true;
-            }
-        }
+        return (($id <= 999) || ($id >= 60000 && $id <= 64999));
+    }
 
-        return false;
+    private function getNextIDNumber($start, $IDNumsInUse)
+    {
+        // custom ID mappings are considered both UIDs and GIDs
+        $IDNumsInUse = array_merge($IDNumsInUse, array_values($this->getCustomIDMappings()));
+        $new_id = $start;
+        while ($this->isIDNumberForbidden($new_id) || in_array($new_id, $IDNumsInUse)) {
+            $new_id++;
+        }
+        return $new_id;
     }
 
-    public function getUnassignedID($uid, $UnitySQL)
+    private function getCustomIDMappings()
     {
-        $netid = strtok($uid, "_");  // extract netid
-        // scrape all files in custom folder
+        $output = [];
         $dir = new \DirectoryIterator($this->custom_mappings_path);
         foreach ($dir as $fileinfo) {
+            $filename = $fileinfo->getFilename();
+            if ($fileinfo->isDot() || ($filename == "README.md")) {
+                continue;
+            }
             if ($fileinfo->getExtension() == "csv") {
-                // found csv file
                 $handle = fopen($fileinfo->getPathname(), "r");
-                while (($data = fgetcsv($handle, 1000, ",")) !== false) {
-                    $netid_match = $data[0];
-                    $uid_match = $data[1];
-
-                    if ($uid == $netid_match || $netid == $netid_match) {
-                        // found a match
-                        if (!$this->IDNumInUse($uid_match)) {
-                            return $uid_match;
-                        }
-                    }
+                while (($row = fgetcsv($handle, null, ",")) !== false) {
+                    array_push($output, $row);
                 }
+            } else {
+                UnitySite::errorLog(
+                    "warning",
+                    "custom ID mapping file '$filename' ignored, extension != .csv",
+                );
             }
         }
+        $output_map = [];
+        foreach ($output as [$uid, $uidNumber_str]) {
+            $output_map[$uid] = intval($uidNumber_str);
+        }
+        return $output_map;
+    }
 
-        // didn't find anything from existing mappings, use next available
-        $next_uid = $this->getNextUIDNumber($UnitySQL);
+    private function getAllUIDNumbersInUse()
+    {
+        // use baseOU for awareness of externally managed entries
+        return array_map(
+            fn($x) => $x["uidnumber"][0],
+            $this->baseOU->getChildrenArray(["uidNumber"], true, "(objectClass=posixAccount)"),
+        );
+    }
 
-        return $next_uid;
+    private function getAllGIDNumbersInUse()
+    {
+        // use baseOU for awareness of externally managed entries
+        return array_map(
+            fn($x) => $x["gidnumber"][0],
+            $this->baseOU->getChildrenArray(["gidNumber"], true, "(objectClass=posixGroup)"),
+        );
     }
 
     public function getAllUsersUIDs()
@@ -235,7 +233,7 @@ public function getAllUsersAttributes($attributes)
         $user_attributes = $this->baseOU->getChildrenArray(
             $attributes,
             true, // recursive
-            "objectClass=posixAccount"
+            "(objectClass=posixAccount)"
         );
         foreach ($user_attributes as $i => $attributes) {
             if (!in_array($attributes["uid"][0], $include_uids)) {

resources/lib/UnitySQL.php

@@ -11,7 +11,6 @@ class UnitySQL
     private const TABLE_PAGES = "pages";
     private const TABLE_AUDIT_LOG = "audit_log";
     private const TABLE_ACCOUNT_DELETION_REQUESTS = "account_deletion_requests";
-    private const TABLE_SITEVARS = "sitevars";
     private const TABLE_GROUP_ROLES = "groupRoles";
     private const TABLE_GROUP_TYPES = "groupTypes";
     private const TABLE_GROUP_ROLE_ASSIGNMENTS = "groupRoleAssignments";
@@ -319,29 +318,6 @@ public function deleteAccountDeletionRequest($uid)
         $stmt->execute();
     }
 
-    public function getSiteVar($name)
-    {
-        $stmt = $this->conn->prepare(
-            "SELECT * FROM " . self::TABLE_SITEVARS . " WHERE name=:name"
-        );
-        $stmt->bindParam(":name", $name);
-
-        $stmt->execute();
-
-        return $stmt->fetchAll()[0]['value'];
-    }
-
-    public function updateSiteVar($name, $value)
-    {
-        $stmt = $this->conn->prepare(
-            "UPDATE " . self::TABLE_SITEVARS . " SET value=:value WHERE name=:name"
-        );
-        $stmt->bindParam(":name", $name);
-        $stmt->bindParam(":value", $value);
-
-        $stmt->execute();
-    }
-
     public function getRole($uid, $group)
     {
         $table = self::TABLE_GROUP_ROLE_ASSIGNMENTS;

resources/lib/UnityUser.php

@@ -60,7 +60,7 @@ public function __toString()
     public function init($firstname, $lastname, $email, $org, $send_mail = true)
     {
         $ldapGroupEntry = $this->getGroupEntry();
-        $id = $this->LDAP->getUnassignedID($this->uid, $this->SQL);
+        $id = $this->LDAP->getNextUIDGIDNumber($this->uid);
         assert(!$ldapGroupEntry->exists());
         $ldapGroupEntry->setAttribute("objectclass", UnityLDAP::POSIX_GROUP_CLASS);
         $ldapGroupEntry->setAttribute("gidnumber", strval($id));

test/custom_user_mappings/test.csv

@@ -0,0 +1,6 @@
+user2001_org998_test,555
+foobar0,1000000
+foobar1,1000001
+foobar2,1000002
+foobar3,1000003
+foobar4,1000004