From e07efae6c6cf734c9da957173879e85e20b451f6 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Wed, 5 Jun 2024 21:29:39 +0300 Subject: [PATCH 1/5] adds initial embed topics and content logic --- .../automation/utils/appnexus/base.py | 19 ++--- .../automation/utils/appnexus/errors.py | 15 ++-- .../contentcuration/utils/recommendations.py | 77 ++++++++++++++++--- 3 files changed, 84 insertions(+), 27 deletions(-) diff --git a/contentcuration/automation/utils/appnexus/base.py b/contentcuration/automation/utils/appnexus/base.py index efbc41f100..0998753bd8 100644 --- a/contentcuration/automation/utils/appnexus/base.py +++ b/contentcuration/automation/utils/appnexus/base.py @@ -1,9 +1,9 @@ -import time import logging -import requests +import time from abc import ABC from abc import abstractmethod -from builtins import NotImplementedError + +import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry @@ -15,7 +15,7 @@ class SessionWithMaxConnectionAge(requests.Session): Session with a maximum connection age. If the connection is older than the specified age, it will be closed and a new one will be created. The age is specified in seconds. """ - def __init__(self, age = 100): + def __init__(self, age=100): super().__init__() self.age = age self.last_used = time.time() @@ -56,7 +56,8 @@ def __init__( class BackendResponse(object): """ Class that should be inherited by specific backend for its responses""" - def __init__(self, **kwargs): + def __init__(self, error=None, **kwargs): + self.error = error for key, value in kwargs.items(): setattr(self, key, value) @@ -67,8 +68,8 @@ class Backend(ABC): session = None base_url = None connect_endpoint = None - max_retries=1 - backoff_factor=0.3 + max_retries = 1 + backoff_factor = 0.3 def __new__(cls, *args, **kwargs): if not isinstance(cls._instance, cls): @@ -156,14 +157,14 @@ def _make_request(self, request): ) as e: logging.exception(e) raise errors.InvalidResponse(f"Invalid response from {url}") - + def connect(self, **kwargs): """ Establishes a connection to the backend service. """ try: request = BackendRequest(method="GET", path=self.connect_endpoint, **kwargs) self._make_request(request) return True - except Exception as e: + except Exception: return False def make_request(self, request): diff --git a/contentcuration/automation/utils/appnexus/errors.py b/contentcuration/automation/utils/appnexus/errors.py index f9166678ab..34ef92f749 100644 --- a/contentcuration/automation/utils/appnexus/errors.py +++ b/contentcuration/automation/utils/appnexus/errors.py @@ -1,15 +1,18 @@ - class ConnectionError(Exception): - pass + pass + class TimeoutError(Exception): - pass + pass + class HttpError(Exception): - pass + pass + class InvalidRequest(Exception): - pass + pass + class InvalidResponse(Exception): - pass + pass diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py index 8fd551da15..1786c0c585 100644 --- a/contentcuration/contentcuration/utils/recommendations.py +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -1,38 +1,56 @@ +from typing import Any +from typing import Dict +from typing import List from typing import Union +from automation.utils.appnexus import errors from automation.utils.appnexus.base import Adapter from automation.utils.appnexus.base import Backend from automation.utils.appnexus.base import BackendFactory from automation.utils.appnexus.base import BackendRequest from automation.utils.appnexus.base import BackendResponse +from contentcuration.models import ContentNode + class RecommendationsBackendRequest(BackendRequest): - pass + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) -class RecommedationsRequest(RecommendationsBackendRequest): - def __init__(self) -> None: - super().__init__() +class RecommendationsRequest(RecommendationsBackendRequest): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) class EmbeddingsRequest(RecommendationsBackendRequest): - def __init__(self) -> None: - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) class RecommendationsBackendResponse(BackendResponse): - pass + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) class RecommendationsResponse(RecommendationsBackendResponse): - def __init__(self) -> None: - pass + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class EmbedTopicsRequest(RecommendationsBackendRequest): + path = '/embed-topics' + method = 'POST' + + +class EmbedContentRequest(RecommendationsBackendRequest): + path = '/embed-content' + method = 'POST' class EmbeddingsResponse(RecommendationsBackendResponse): - def __init__(self) -> None: - pass + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) class RecommendationsBackendFactory(BackendFactory): @@ -67,9 +85,44 @@ def cache_embeddings(self, embeddings_list) -> bool: return True def get_recommendations(self, embedding) -> RecommendationsResponse: - request = RecommedationsRequest(embedding) + request = RecommendationsRequest(embedding) return self.backend.make_request(request) + def embed_topics(self, topics: Dict[str, Any]) -> EmbeddingsResponse: + + if not self.backend.connect(): + raise errors.ConnectionError("Connection to the backend failed") + + try: + embed_topics_request = EmbedTopicsRequest(json=topics) + return self.backend.make_request(embed_topics_request) + except Exception as e: + return EmbeddingsResponse(error=e) + + def embed_content(self, nodes: List[ContentNode]) -> EmbeddingsResponse: + + resources = [] + + if not self.backend.connect(): + raise errors.ConnectionError("Connection to the backend failed") + + try: + for node in nodes: + resource = self.extract_content(node) + resources.append(resource) + + json = { + 'resources': resources, + 'metadata': {} + } + embed_content_request = EmbedContentRequest(json=json) + return self.backend.make_request(embed_content_request) + except Exception as e: + return EmbeddingsResponse(error=e) + + def extract_content(self, node: ContentNode) -> Dict[str, Any]: + pass + class Recommendations(Backend): From 7b43a37ec15edd688c0bce0568e237f6b7d5e089 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Thu, 6 Jun 2024 17:14:05 +0300 Subject: [PATCH 2/5] Refactors code --- .../contentcuration/utils/recommendations.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py index 1786c0c585..ed4d1f50b5 100644 --- a/contentcuration/contentcuration/utils/recommendations.py +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -101,16 +101,11 @@ def embed_topics(self, topics: Dict[str, Any]) -> EmbeddingsResponse: def embed_content(self, nodes: List[ContentNode]) -> EmbeddingsResponse: - resources = [] - if not self.backend.connect(): raise errors.ConnectionError("Connection to the backend failed") try: - for node in nodes: - resource = self.extract_content(node) - resources.append(resource) - + resources = [self.__extract_content(node) for node in nodes] json = { 'resources': resources, 'metadata': {} @@ -120,8 +115,8 @@ def embed_content(self, nodes: List[ContentNode]) -> EmbeddingsResponse: except Exception as e: return EmbeddingsResponse(error=e) - def extract_content(self, node: ContentNode) -> Dict[str, Any]: - pass + def __extract_content(self, node: ContentNode) -> Dict[str, Any]: + return {} class Recommendations(Backend): From d44e5a78c7757f0f0150c391700067c05b79a0fd Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Thu, 6 Jun 2024 17:14:50 +0300 Subject: [PATCH 3/5] Refactors code --- contentcuration/contentcuration/utils/recommendations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/utils/recommendations.py b/contentcuration/contentcuration/utils/recommendations.py index ed4d1f50b5..1c015d2050 100644 --- a/contentcuration/contentcuration/utils/recommendations.py +++ b/contentcuration/contentcuration/utils/recommendations.py @@ -105,7 +105,7 @@ def embed_content(self, nodes: List[ContentNode]) -> EmbeddingsResponse: raise errors.ConnectionError("Connection to the backend failed") try: - resources = [self.__extract_content(node) for node in nodes] + resources = [self.extract_content(node) for node in nodes] json = { 'resources': resources, 'metadata': {} @@ -115,7 +115,7 @@ def embed_content(self, nodes: List[ContentNode]) -> EmbeddingsResponse: except Exception as e: return EmbeddingsResponse(error=e) - def __extract_content(self, node: ContentNode) -> Dict[str, Any]: + def extract_content(self, node: ContentNode) -> Dict[str, Any]: return {} From da2942bcbd3692846b4c24e6cd71d9dac72fdf43 Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Fri, 7 Jun 2024 01:00:09 +0300 Subject: [PATCH 4/5] adds tests for embed_topics method --- .../tests/utils/test_recommendations.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/contentcuration/contentcuration/tests/utils/test_recommendations.py b/contentcuration/contentcuration/tests/utils/test_recommendations.py index a4f711033a..4f5ceb8ff1 100644 --- a/contentcuration/contentcuration/tests/utils/test_recommendations.py +++ b/contentcuration/contentcuration/tests/utils/test_recommendations.py @@ -1,6 +1,11 @@ +from automation.utils.appnexus import errors from django.test import TestCase +from mock import MagicMock +from mock import patch +from contentcuration.utils.recommendations import EmbeddingsResponse from contentcuration.utils.recommendations import Recommendations +from contentcuration.utils.recommendations import RecommendationsAdapter class RecommendationsTestCase(TestCase): @@ -8,3 +13,51 @@ def test_backend_initialization(self): recomendations = Recommendations() self.assertIsNotNone(recomendations) self.assertIsInstance(recomendations, Recommendations) + + +class RecommendationsAdapterTestCase(TestCase): + def setUp(self): + self.adapter = RecommendationsAdapter(MagicMock()) + self.topic = { + 'id': 'topic_id', + 'title': 'topic_title', + 'description': 'topic_description', + 'language': 'en', + 'ancestors': [ + { + 'id': 'ancestor_id', + 'title': 'ancestor_title', + 'description': 'ancestor_description', + } + ] + } + + def test_adapter_initialization(self): + self.assertIsNotNone(self.adapter) + self.assertIsInstance(self.adapter, RecommendationsAdapter) + + @patch('contentcuration.utils.recommendations.EmbedTopicsRequest') + def test_embed_topics_backend_connect_success(self, embed_topics_request_mock): + self.adapter.backend.connect.return_value = True + self.adapter.backend.make_request.return_value = MagicMock(spec=EmbeddingsResponse) + response = self.adapter.embed_topics(self.topic) + self.adapter.backend.connect.assert_called_once() + self.adapter.backend.make_request.assert_called_once() + self.assertIsInstance(response, EmbeddingsResponse) + + def test_embed_topics_backend_connect_failure(self): + self.adapter.backend.connect.return_value = False + with self.assertRaises(errors.ConnectionError): + self.adapter.embed_topics(self.topic) + self.adapter.backend.connect.assert_called_once() + self.adapter.backend.make_request.assert_not_called() + + @patch('contentcuration.utils.recommendations.EmbedTopicsRequest') + def test_embed_topics_make_request_exception(self, embed_topics_request_mock): + self.adapter.backend.connect.return_value = True + self.adapter.backend.make_request.side_effect = Exception("Mocked exception") + response = self.adapter.embed_topics(self.topic) + self.adapter.backend.connect.assert_called_once() + self.adapter.backend.make_request.assert_called_once() + self.assertIsInstance(response, EmbeddingsResponse) + self.assertEqual(str(response.error), "Mocked exception") From 34bbe9e0da0e705e39f9714f56ed99a9c3548d5d Mon Sep 17 00:00:00 2001 From: Samson Akol Date: Fri, 7 Jun 2024 01:16:19 +0300 Subject: [PATCH 5/5] adds tests for embed_content method --- .../tests/utils/test_recommendations.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/contentcuration/contentcuration/tests/utils/test_recommendations.py b/contentcuration/contentcuration/tests/utils/test_recommendations.py index 4f5ceb8ff1..3ef11e43e5 100644 --- a/contentcuration/contentcuration/tests/utils/test_recommendations.py +++ b/contentcuration/contentcuration/tests/utils/test_recommendations.py @@ -3,6 +3,7 @@ from mock import MagicMock from mock import patch +from contentcuration.models import ContentNode from contentcuration.utils.recommendations import EmbeddingsResponse from contentcuration.utils.recommendations import Recommendations from contentcuration.utils.recommendations import RecommendationsAdapter @@ -31,6 +32,9 @@ def setUp(self): } ] } + self.resources = [ + MagicMock(spec=ContentNode), + ] def test_adapter_initialization(self): self.assertIsNotNone(self.adapter) @@ -61,3 +65,29 @@ def test_embed_topics_make_request_exception(self, embed_topics_request_mock): self.adapter.backend.make_request.assert_called_once() self.assertIsInstance(response, EmbeddingsResponse) self.assertEqual(str(response.error), "Mocked exception") + + @patch('contentcuration.utils.recommendations.EmbedContentRequest') + def test_embed_content_backend_connect_success(self, embed_content_request_mock): + self.adapter.backend.connect.return_value = True + self.adapter.backend.make_request.return_value = MagicMock(spec=EmbeddingsResponse) + response = self.adapter.embed_content(self.resources) + self.adapter.backend.connect.assert_called_once() + self.adapter.backend.make_request.assert_called_once() + self.assertIsInstance(response, EmbeddingsResponse) + + def test_embed_content_backend_connect_failure(self): + self.adapter.backend.connect.return_value = False + with self.assertRaises(errors.ConnectionError): + self.adapter.embed_content(self.resources) + self.adapter.backend.connect.assert_called_once() + self.adapter.backend.make_request.assert_not_called() + + @patch('contentcuration.utils.recommendations.EmbedContentRequest') + def test_embed_content_make_request_exception(self, embed_content_request_mock): + self.adapter.backend.connect.return_value = True + self.adapter.backend.make_request.side_effect = Exception("Mocked exception") + response = self.adapter.embed_content(self.resources) + self.adapter.backend.connect.assert_called_once() + self.adapter.backend.make_request.assert_called_once() + self.assertIsInstance(response, EmbeddingsResponse) + self.assertEqual(str(response.error), "Mocked exception")