dev up to date with hot fixes in main

README.md

@@ -1,4 +1,4 @@
-# Augur NEW Release v0.86.0
+# Augur NEW Release v0.86.1
 
 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! 
 The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). 
@@ -11,7 +11,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o
 ## NEW RELEASE ALERT!
 **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**.
 
-Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.86.0).
+Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.86.1).
 
 
 - The `main` branch is a stable version of our new architecture, which features:

augur/application/db/lib.py

@@ -1,3 +1,4 @@
+import re
 import time
 import random
 import logging
@@ -17,7 +18,7 @@
 
 logger = logging.getLogger("db_lib")
 
 def convert_type_of_value(config_dict, logger=None):
 
 
     data_type = config_dict["type"]
@@ -196,7 +197,7 @@
 
     try:
         working_commits = fetchall_data_from_sql_text(query)
     except:
         working_commits = []
 
     return working_commits
@@ -212,7 +213,7 @@
 
     try:
         missing_commit_hashes = fetchall_data_from_sql_text(fetch_missing_hashes_sql)
     except:
         missing_commit_hashes = []
 
     return missing_commit_hashes
@@ -232,7 +233,7 @@
         return session.query(CollectionStatus).filter(getattr(CollectionStatus,f"{collection_type}_status" ) == CollectionState.COLLECTING.value).count()
 
 
 def facade_bulk_insert_commits(logger, records):
 
     with get_session() as session:
 
@@ -243,6 +244,7 @@
                 )
             session.commit()
         except Exception as e:
+            session.rollback()
 
             if len(records) > 1:
                 logger.error(f"Ran into issue when trying to insert commits \n Error: {e}")
@@ -257,7 +259,14 @@
                 commit_record = records[0]
                 #replace incomprehensible dates with epoch.
                 #2021-10-11 11:57:46 -0500
-                placeholder_date = "1970-01-01 00:00:15 -0500"
+
+                # placeholder_date = "1970-01-01 00:00:15 -0500"
+                placeholder_date = commit_record['author_timestamp']
+
+                # Reconstruct timezone portion of the date string to UTC
+                placeholder_date = re.split("[-+]", placeholder_date)
+                placeholder_date.pop()
+                placeholder_date = "-".join(placeholder_date) + "+0000"
 
                 #Check for improper utc timezone offset
                 #UTC timezone offset should be between -14:00 and +14:00
@@ -274,7 +283,7 @@
                 raise e
 
 
 def batch_insert_contributors(logger, data: Union[List[dict], dict]) -> Optional[List[dict]]:
 
     batch_size = 1000
 
@@ -285,7 +294,7 @@
 
 
 
 def bulk_insert_dicts(logger, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]:
 
     if isinstance(data, list) is False:
 

augur/tasks/github/util/github_data_access.py

@@ -10,11 +10,11 @@
 
 class RatelimitException(Exception):
 
-    def __init__(self, response, message="Github Rate limit exceeded") -> None:
+    def __init__(self, response, keys_used, message="Github Rate limit exceeded") -> None:
 
         self.response = response
 
-        super().__init__(message)
+        super().__init__(f"{message}. Keys used: {keys_used}")
 
 class UrlNotFoundException(Exception):
     pass
@@ -29,6 +29,7 @@ def __init__(self, key_manager, logger: logging.Logger):
         self.logger = logger
         self.key_client = KeyClient("github_rest", logger)
         self.key = None
+        self.expired_keys_for_request = []
 
     def get_resource_count(self, url):
 
@@ -108,7 +109,8 @@ def make_request(self, url, method="GET", timeout=100):
             response = client.request(method=method, url=url, headers=headers, timeout=timeout, follow_redirects=True)
 
             if response.status_code in [403, 429]:
-                raise RatelimitException(response)
+                self.expired_keys_for_request.append(self.key)
+                raise RatelimitException(response, self.expired_keys_for_request[-5:])
 
             if response.status_code == 404:
                 raise UrlNotFoundException(f"Could not find {url}")
@@ -120,7 +122,8 @@ def make_request(self, url, method="GET", timeout=100):
 
             try:
                 if "X-RateLimit-Remaining" in response.headers and int(response.headers["X-RateLimit-Remaining"]) < GITHUB_RATELIMIT_REMAINING_CAP:
-                    raise RatelimitException(response)
+                    self.expired_keys_for_request.append(self.key)
+                    raise RatelimitException(response, self.expired_keys_for_request[-5:])
             except ValueError:
                 self.logger.warning(f"X-RateLimit-Remaining was not an integer. Value: {response.headers['X-RateLimit-Remaining']}")
 
@@ -147,12 +150,16 @@ def __make_request_with_retries(self, url, method="GET", timeout=100):
         """
 
         try:
-            return self.make_request(url, method, timeout)
+            result = self.make_request(url, method, timeout)
+            self.expired_keys_for_request = []
+            return result
         except RatelimitException as e:
             self.__handle_github_ratelimit_response(e.response)
             raise e
         except NotAuthorizedException as e:
+            self.expired_keys_for_request = []
             self.__handle_github_not_authorized_response()
+            raise e
 
     def __handle_github_not_authorized_response(self):
 
@@ -162,6 +169,7 @@ def __handle_github_not_authorized_response(self):
     def __handle_github_ratelimit_response(self, response):
 
         headers = response.headers
+        previous_key = self.key
 
         if "Retry-After" in headers:
 
@@ -184,6 +192,9 @@ def __handle_github_ratelimit_response(self, response):
         else:
             self.key = self.key_client.expire(self.key, time.time() + 60)
 
+        if previous_key == self.key:
+            self.logger.error(f"The same key was returned after a request to expire it was sent (key: {self.key[-5:]})")
+
     def __add_query_params(self, url: str, additional_params: dict) -> str:
         """Add query params to a url.
 

docker/backend/Dockerfile

@@ -17,7 +17,7 @@ RUN go install github.com/ossf/scorecard/v5@v5.1.1 \
 FROM python:3.11-slim-bullseye
 
 LABEL maintainer="outdoors@acm.org"
-LABEL version="0.86.0"
+LABEL version="0.86.1"
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PATH="/usr/bin/:/usr/local/bin:/usr/lib:${PATH}"

docker/database/Dockerfile

@@ -2,11 +2,11 @@
 FROM postgres:16
 
 LABEL maintainer="outdoors@acm.org"
-LABEL version="0.86.0"
+LABEL version="0.86.1"
 
 ENV POSTGRES_DB "test"
 ENV POSTGRES_USER "augur"
 ENV POSTGRES_PASSWORD "augur"
 
 EXPOSE 5432
 

docker/rabbitmq/Dockerfile

@@ -1,10 +1,10 @@
 FROM rabbitmq:3.12-management-alpine
 
 LABEL maintainer="574/augur@simplelogin.com"
-LABEL version="0.86.0"
+LABEL version="0.86.1"
 
 ARG RABBIT_MQ_DEFAULT_USER=augur
 ARG RABBIT_MQ_DEFAULT_PASSWORD=password123
 ARG RABBIT_MQ_DEFAULT_VHOST=augur_vhost
 
 COPY --chown=rabbitmq:rabbitmq ./docker/rabbitmq/augur.conf /etc/rabbitmq/conf.d/

metadata.py

@@ -5,8 +5,8 @@
 
 __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection"
 
-__version__ = "0.86.0"
-__release__ = "v0.86.0 (Pod People)"
+__version__ = "0.86.1"
+__release__ = "v0.86.1 (Pod People)"
 
 __license__ = "MIT"
 __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Sean Goggins, Brian Warner & Augurlabs 2025"