diff --git a/archivo/utils/async_rdf_retrieval.py b/archivo/utils/async_rdf_retrieval.py index 151b461..55cb66a 100644 --- a/archivo/utils/async_rdf_retrieval.py +++ b/archivo/utils/async_rdf_retrieval.py @@ -8,7 +8,7 @@ def chunk_list(lst, size): for i in range(0, len(lst), size): - yield lst[i : i + size] + yield lst[i: i + size] async def fetch_one_nt_resource( diff --git a/archivo/utils/inspectVocabs.py b/archivo/utils/inspectVocabs.py index 96cfaca..ba69783 100644 --- a/archivo/utils/inspectVocabs.py +++ b/archivo/utils/inspectVocabs.py @@ -39,6 +39,9 @@ def getGraphOfVocabFile(filepath, logger=None): def get_graph_of_string(rdf_string, content_type): + + if rdf_string is None: + return None graph = rdflib.Graph() graph.parse(data=rdf_string, format=header_rdflib_mapping.get(content_type, "xml")) return graph @@ -406,7 +409,24 @@ def hackyShaclInspection(shaclURL): return "OK" +def inspect_shacl_string(shaclString): + + if shaclString is None: + return None + + if "sh:resultSeverity sh:Violation" in shaclString: + return "VIOLATION" + elif "sh:resultSeverity sh:Warning" in shaclString: + return "WARNING" + elif "sh:resultSeverity sh:Info" in shaclString: + return "INFO" + else: + return "OK" + + def interpretShaclGraph(graph): + if graph is None: + return None violationRef = URIRef("http://www.w3.org/ns/shacl#Violation") warningRef = URIRef("http://www.w3.org/ns/shacl#Warning") infoRef = URIRef("http://www.w3.org/ns/shacl#Info") diff --git a/archivo/utils/queryDatabus.py b/archivo/utils/queryDatabus.py index 0232216..be7bde2 100644 --- a/archivo/utils/queryDatabus.py +++ b/archivo/utils/queryDatabus.py @@ -1,4 +1,4 @@ -import requests, sys, requests, traceback, rdflib +import sys, requests, traceback, rdflib from SPARQLWrapper import SPARQLWrapper, JSON from io import StringIO from urllib.error import URLError @@ -6,6 +6,10 @@ from utils.stringTools import generateStarString from datetime import datetime, timedelta import csv +import asyncio +import aiohttp +from utils.archivoLogs import webservice_logger +import json databusRepoUrl = "https://databus.dbpedia.org/repo/sparql" @@ -15,56 +19,6 @@ mod_endpoint = "https://mods.tools.dbpedia.org/sparql" -def getLatestMetaFile(group, artifact): - databusLink = f"https://databus.dbpedia.org/ontologies/{group}/{artifact}" - latestMetaFileQuery = ( - "PREFIX dcat: \n" - "PREFIX dct: \n" - "PREFIX dataid-cv: \n" - "PREFIX dataid: \n" - "PREFIX databus: \n" - "SELECT DISTINCT ?versionURL ?file WHERE\n" - "{\n" - "?dataset dataid:account databus:ontologies .\n" - f"?dataset dataid:artifact <{databusLink}>.\n" - "?dataset dcat:distribution ?distribution .\n" - "?dataset dataid:version ?versionURL .\n" - "?distribution 'meta'^^ .\n" - "?distribution dcat:downloadURL ?file .\n" - "?dataset dct:hasVersion ?latestVersion .\n" - "{ SELECT DISTINCT ?art (MAX(?v) as ?latestVersion) WHERE {\n" - "?dataset dataid:account databus:ontologies .\n" - "?dataset dataid:artifact ?art.\n" - "?dataset dcat:distribution ?distribution .\n" - "?distribution 'meta'^^ .\n" - "?dataset dct:hasVersion ?v .\n" - "} GROUP BY ?art #HAVING (?v = MAX(?v))\n" - "}\n" - "}" - ) - # query = latestMetaFileQuery.format(group, artifact) - sparql = SPARQLWrapper(databusRepoUrl) - sparql.setQuery(latestMetaFileQuery) - sparql.setReturnFormat(JSON) - try: - results = sparql.query().convert() - except URLError: - return ( - False, - databusLink, - "", - "There was an Error querying the databus. Probably an error on the databus side.", - ) - - try: - metaFileUri = results["results"]["bindings"][0]["file"]["value"] - databusVersionUri = results["results"]["bindings"][0]["versionURL"]["value"] - req = requests.get(metaFileUri) - return True, databusLink, databusVersionUri, req.json() - except KeyError: - return False, databusLink, "", "There was an error finding the metadata-file" - - def getInfoForArtifact(group, artifact): """Returns the info for a given group and artifact: Returns a tuple (title, comment, version_infos) with: @@ -186,7 +140,6 @@ def getInfoForArtifact(group, artifact): goodLicenseURL = binding.get("goodLicense", {"value": ""})["value"] lodeShaclURL = binding.get("lode", {"value": ""})["value"] consistencyURL = binding["consistencyFile"]["value"] - try: archivo_test_url = binding["archivoCheck"]["value"] archiv_test_severity = inspectVocabs.hackyShaclInspection(archivo_test_url) @@ -247,53 +200,226 @@ def getInfoForArtifact(group, artifact): return title, comment, version_infos -def getLatestParsedOntology(group, artifact): +async def artifact_info_async(group, artifact): + """Returns the info for a given group and artifact: + Returns a tuple (title, comment, version_infos) with: + title -> being the lates title on the databus + comment -> being the latest comment (short description) on the databus + version_info -> being a list of dicts + which contain: + { + "minLicense": { + "conforms": metadata["test-results"]["License-I"], + "url": minLicenseURL, + }, + "goodLicense": { + "conforms": metadata["test-results"]["License-II"], + "url": goodLicenseURL, + }, + "lode": { + "severity": inspectVocabs.hackyShaclInspection(lodeShaclURL), + "url": lodeShaclURL, + }, + "archivo": { + "severity": inspectVocabs.hackyShaclInspection(lodeShaclURL), + "url": lodeShaclURL, + }, + "version": {"label": version, "url": versionURL}, + "consistent": { + "conforms": isConsistent(metadata["test-results"]["consistent"]), + "url": consistencyURL, + }, + "triples": metadata["ontology-info"]["triples"], + "parsing": { + "conforms": parsing, + "errors": metadata["logs"]["rapper-errors"], + }, + "semversion": metadata["ontology-info"]["semantic-version"], + "stars": stars, + "docuURL": docuURL, + }""" databusLink = f"https://databus.dbpedia.org/ontologies/{group}/{artifact}" - query = "\n".join( - ( - "PREFIX dataid: ", - "PREFIX dct: ", - "PREFIX dcat: ", - "PREFIX db: ", - "PREFIX rdf: ", - "PREFIX rdfs: ", - "PREFIX dataid-cv: ", - "PREFIX xsd: ", - "", - "SELECT DISTINCT ?file WHERE {", - "VALUES ?art { <%s> } ." % databusLink, - " ?dataset dataid:account db:ontologies .", - " ?dataset dataid:artifact ?art .", - " ?dataset dcat:distribution ?distribution .", - " ?distribution dataid-cv:type 'parsed'^^xsd:string .", - " ?distribution dataid:formatExtension 'ttl'^^xsd:string .", - " ?distribution dcat:downloadURL ?file .", - " ?dataset dct:hasVersion ?latestVersion .", - "{", - " SELECT DISTINCT ?art (MAX(?v) as ?latestVersion) WHERE {", - " ?dataset dataid:account db:ontologies .", - " ?dataset dataid:artifact ?art .", - " ?dataset dct:hasVersion ?v .", - "}", - "}", - "}", - ) + + query = ( + """PREFIX dataid: +PREFIX dct: +PREFIX dcat: +PREFIX db: +PREFIX rdf: +PREFIX rdfs: +PREFIX dataid-cv: +PREFIX xsd: +SELECT DISTINCT ?title ?comment ?versionURL ?version ?metafile ?minLicense ?goodLicense ?lode ?archivoCheck ?consistencyFile ?docuURL ?pylodeURL WHERE { + VALUES ?art { <%s> } . + ?dataset dataid:account db:ontologies . + ?dataset dataid:artifact ?art . + ?dataset dcat:distribution ?metaDst . + ?metaDst dataid-cv:type 'meta'^^xsd:string . + ?metaDst dcat:downloadURL ?metafile . + ?dataset dcat:distribution ?shaclMinLicense . + ?dataset dcat:distribution ?consistencyReport . + ?consistencyReport dataid-cv:type 'pelletConsistency'^^xsd:string . + ?consistencyReport dataid-cv:imports 'FULL'^^xsd:string . + ?consistencyReport dcat:downloadURL ?consistencyFile . + ?shaclMinLicense dataid-cv:type 'shaclReport'^^xsd:string . + ?shaclMinLicense dataid-cv:validates 'minLicense'^^xsd:string . + ?shaclMinLicense dcat:downloadURL ?minLicense . + ?dataset dcat:distribution ?shaclGoodLicense . + ?shaclGoodLicense dataid-cv:type 'shaclReport'^^xsd:string . + ?shaclGoodLicense dataid-cv:validates 'goodLicense'^^xsd:string . + ?shaclGoodLicense dcat:downloadURL ?goodLicense . + ?dataset dcat:distribution ?shaclLode . + ?shaclLode dataid-cv:type 'shaclReport'^^xsd:string . + ?shaclLode dataid-cv:validates 'lodeMetadata'^^xsd:string . + ?shaclLode dcat:downloadURL ?lode . + OPTIONAL { ?dataset dcat:distribution ?docuDst . + ?docuDst dataid-cv:type 'generatedDocu'^^xsd:string . + ?docuDst dcat:downloadURL ?docuURL . + } + OPTIONAL { ?dataset dcat:distribution ?pylodeDocDst . + ?pylodeDocDst dataid-cv:type 'pyLodeDoc'^^xsd:string . + ?pylodeDocDst dcat:downloadURL ?pylodeURL . + } + OPTIONAL { + ?dataset dcat:distribution ?shaclArchivo . + ?shaclArchivo dataid-cv:type 'shaclReport'^^xsd:string . + ?shaclArchivo dataid-cv:validates 'archivoMetadata'^^xsd:string . + ?shaclArchivo dcat:downloadURL ?archivoCheck . + } + ?dataset dataid:version ?versionURL . + ?dataset dct:hasVersion ?version . ?dataset dct:title ?title . + ?dataset rdfs:comment ?comment . + } + """ + % databusLink ) - # query = latestMetaFileQuery.format(group, artifact) sparql = SPARQLWrapper(databusRepoUrl) sparql.setQuery(query) sparql.setReturnFormat(JSON) results = sparql.query().convert() + version_infos = [] try: - parsedOntoUri = results["results"]["bindings"][0]["file"]["value"] - req = requests.get(parsedOntoUri) - graph = rdflib.Graph() - graph.parse(StringIO(req.text), format="turtle") - return graph + results = results["results"]["bindings"] except KeyError: - traceback.print_exc(file=sys.stdout) - except Exception: - traceback.print_exc(file=sys.stdout) + return False, version_infos, f"No data found for {databusLink}" + try: + title = sorted( + results, key=lambda binding: binding["version"]["value"], reverse=True + )[0]["title"]["value"] + comment = sorted( + results, key=lambda binding: binding["version"]["value"], reverse=True + )[0]["comment"]["value"] + except Exception as e: + return None, None, None + + async with aiohttp.ClientSession() as session: + for binding in results: + url_fetch_keys = [ + "metafile", + "minLicense", + "goodLicense", + "lode", + "consistencyFile", + "archivoCheck", + ] + + version = binding.get("version", {"value": ""})["value"] + versionURL = binding.get("versionURL", {"value": ""})["value"] + + id_url_mapping = {} + + for key in url_fetch_keys: + id_url_mapping[key] = binding.get(key, {"value": ""})["value"] + + # async fetch content and map to id + content_mapping = {} + tasks = [] + + for key in url_fetch_keys: + task = asyncio.ensure_future(fetch_content_async(session, id_url_mapping[key], key)) + tasks.append(task) + + result_tuples = await asyncio.gather(*tasks) + + for key, text in result_tuples: + content_mapping[key] = text + + metadata = json.loads(content_mapping["metafile"]) + # select docu url, pref pylode doc + docuURL = binding.get("pylodeURL", {}).get("value", None) + if docuURL is None: + docuURL = binding.get("docuURL", {}).get("value", None) + + parsing = ( + True + if metadata["logs"]["rapper-errors"] == [] + or metadata["logs"]["rapper-errors"] == "" + else False + ) + stars = ontoFiles.stars_from_meta_dict(metadata) + + def is_consistent(s): + if s == "Yes": + return True + else: + return False + version_infos.append( + { + "minLicense": { + "conforms": metadata["test-results"]["License-I"], + "url": id_url_mapping["minLicense"], + "content": inspectVocabs.interpretShaclGraph( + inspectVocabs.get_graph_of_string( + content_mapping["minLicense"], "text/turtle" + ) + ), + }, + "goodLicense": { + "conforms": metadata["test-results"]["License-II"], + "url": id_url_mapping["goodLicense"], + "content": inspectVocabs.interpretShaclGraph( + inspectVocabs.get_graph_of_string( + content_mapping["goodLicense"], "text/turtle" + ) + ), + }, + "lode": { + "severity": inspectVocabs.inspect_shacl_string( + content_mapping["lode"] + ), + "url": id_url_mapping["lode"], + "content": inspectVocabs.interpretShaclGraph( + inspectVocabs.get_graph_of_string(content_mapping["lode"], "text/turtle") + ), + }, + "archivoCheck": { + "severity": inspectVocabs.inspect_shacl_string( + content_mapping["archivoCheck"] + ), + "url": id_url_mapping["archivoCheck"], + "content": inspectVocabs.interpretShaclGraph( + inspectVocabs.get_graph_of_string( + content_mapping["archivoCheck"], "text/turtle" + ) + ), + }, + "version": {"label": version, "url": versionURL}, + "consistencyFile": { + "conforms": is_consistent(metadata["test-results"]["consistent"]), + "url": id_url_mapping["consistencyFile"], + "content": content_mapping["consistencyFile"], + }, + "triples": metadata["ontology-info"]["triples"], + "parsing": { + "conforms": parsing, + "content": "\n".join(metadata["logs"]["rapper-errors"]), + }, + "semversion": metadata["ontology-info"]["semantic-version"], + "stars": stars, + "docuURL": docuURL, + } + ) + return title, comment, version_infos def getDownloadURL(group, artifact, fileExt="owl", version=None): @@ -345,74 +471,6 @@ def getDownloadURL(group, artifact, fileExt="owl", version=None): return None -def allLatestParsedTurtleFiles(): - query = "\n".join( - ( - "PREFIX dataid: ", - "PREFIX dct: ", - "PREFIX dcat: ", - "PREFIX db: ", - "PREFIX rdf: ", - "PREFIX rdfs: ", - "PREFIX dataid-cv: ", - "PREFIX xsd: ", - "SELECT DISTINCT ?art ?title ?latestVersion ?ttlFile ?owlFile ?ntFile ?metafile WHERE {", - "?dataset dataid:account db:ontologies .", - "?dataset dataid:artifact ?art .", - "?dataset dct:title ?title .", - "?dataset dcat:distribution ?turtleDst .", - "?turtleDst dataid-cv:type 'parsed'^^xsd:string .", - "?turtleDst dataid:formatExtension 'ttl'^^xsd:string .", - "?turtleDst dcat:downloadURL ?ttlFile .", - "?dataset dcat:distribution ?owlDst .", - "?owlDst dataid-cv:type 'parsed'^^xsd:string .", - "?owlDst dataid:formatExtension 'owl'^^xsd:string .", - "?owlDst dcat:downloadURL ?owlFile .", - "?dataset dcat:distribution ?ntDst .", - "?ntDst dataid-cv:type 'parsed'^^xsd:string .", - "?ntDst dataid:formatExtension 'nt'^^xsd:string .", - "?ntDst dcat:downloadURL ?ntFile .", - "?dataset dcat:distribution ?metaDst .", - "?metaDst dataid-cv:type 'meta'^^xsd:string .", - "?metaDst dcat:downloadURL ?metafile .", - "?dataset dct:hasVersion ?latestVersion .", - "{", - "SELECT DISTINCT ?art (MAX(?v) as ?latestVersion) WHERE {", - "?dataset dataid:account db:ontologies .", - "?dataset dataid:artifact ?art .", - "?dataset dct:hasVersion ?v .", - "}", - "}", - "}", - ) - ) - sparql = SPARQLWrapper(databusRepoUrl) - sparql.setQuery(query) - sparql.setReturnFormat(JSON) - try: - ontdata = sparql.query().convert() - except URLError: - return None - result = {} - for binding in ontdata["results"]["bindings"]: - try: - databusUri = binding["art"]["value"] - title = binding["title"]["value"] - if databusUri not in result: - result[databusUri] = { - "title": title, - "ttlFile": binding["ttlFile"]["value"], - "owlFile": binding["owlFile"]["value"], - "ntFile": binding["ntFile"]["value"], - "meta": binding["metafile"]["value"], - "version": binding["latestVersion"]["value"], - } - except KeyError: - continue - - return result - - def latestNtriples(): """Returns a dict with the NIR being the key and value being another dict with entries: ntFile -> URL of the parsed ntriples of the ontology @@ -472,173 +530,6 @@ def latestNtriples(): return result -def getLatestInfoForAll(): - query = "\n".join( - ( - "PREFIX dataid: ", - "PREFIX dct: ", - "PREFIX dcat: ", - "PREFIX db: ", - "PREFIX rdf: ", - "PREFIX rdfs: ", - "PREFIX dataid-cv: ", - "PREFIX xsd: ", - "SELECT DISTINCT ?art ?title ?comment ?latestVersion ?metafile ?minLicense ?goodLicense ?lode ?consistencyFile ?docuURL WHERE {", - "?dataset dataid:account db:ontologies .", - "?dataset dataid:artifact ?art .", - "?dataset dcat:distribution ?metaDst .", - "?metaDst dataid-cv:type 'meta'^^xsd:string .", - "?metaDst dcat:downloadURL ?metafile .", - "?dataset dcat:distribution ?shaclMinLicense .", - "?dataset dcat:distribution ?consistencyReport .", - "?consistencyReport dataid-cv:type 'pelletConsistency'^^xsd:string .", - "?consistencyReport dataid-cv:imports 'FULL'^^xsd:string .", - "?consistencyReport dcat:downloadURL ?consistencyFile .", - "?shaclMinLicense dataid-cv:type 'shaclReport'^^xsd:string .", - "?shaclMinLicense dataid-cv:validates 'minLicense'^^xsd:string .", - "?shaclMinLicense dcat:downloadURL ?minLicense .", - "?dataset dcat:distribution ?shaclGoodLicense .", - "?shaclGoodLicense dataid-cv:type 'shaclReport'^^xsd:string .", - "?shaclGoodLicense dataid-cv:validates 'goodLicense'^^xsd:string .", - "?shaclGoodLicense dcat:downloadURL ?goodLicense .", - "?dataset dcat:distribution ?shaclLode .", - "?shaclLode dataid-cv:type 'shaclReport'^^xsd:string .", - "?shaclLode dataid-cv:validates 'lodeMetadata'^^xsd:string .", - "?shaclLode dcat:downloadURL ?lode .", - "OPTIONAL {" "?dataset dcat:distribution ?docuDst .", - "?docuDst dataid-cv:type 'generatedDocu'^^xsd:string .", - "?docuDst dcat:downloadURL ?docuURL .", - "}", - "{", - "SELECT DISTINCT ?art (MAX(?v) as ?latestVersion) WHERE {", - "?dataset dataid:account db:ontologies .", - "?dataset dataid:artifact ?art .", - "?dataset dct:hasVersion ?v .", - "}", - "}", - "?dataset dct:hasVersion ?latestVersion.", - "?dataset dct:title ?title .", - "?dataset rdfs:comment ?comment .", - "}", - ) - ) - sparql = SPARQLWrapper(databusRepoUrl) - sparql.setQuery(query) - sparql.setReturnFormat(JSON) - results = sparql.query().convert() - - info_dict = {} - - try: - results = results["results"]["bindings"] - except KeyError: - return False, version_infos, f"No data found for {databusLink}" - - for binding in results: - - artifactURL = binding.get("art", {"value": ""})["value"] - info_dict[artifactURL] = {} - - info_dict[artifactURL]["title"] = binding.get("title", {"value": ""})["value"] - info_dict[artifactURL]["version"] = binding.get("latestVersion", {"value": ""})[ - "value" - ] - info_dict[artifactURL]["metafile"] = binding.get("metafile", {"value": ""})[ - "value" - ] - info_dict[artifactURL]["minLicenseURL"] = binding.get( - "minLicense", {"value": ""} - )["value"] - info_dict[artifactURL]["goodLicenseURL"] = binding.get( - "goodLicense", {"value": ""} - )["value"] - lodeShaclURL = binding.get("lode", {"value": ""})["value"] - consistencyURL = binding["consistencyFile"]["value"] - try: - docuURL = binding["docuURL"]["value"] - except KeyError: - docuURL = None - - parsing = True if metadata["logs"]["rapper-errors"] == "" else False - stars = ontoFiles.measureStars( - metadata["logs"]["rapper-errors"], - metadata["test-results"]["License-I"], - metadata["test-results"]["consistent"], - metadata["test-results"]["consistent-without-imports"], - metadata["test-results"]["License-II"], - ) - isConsistent = lambda s: True if s == "Yes" else False - version_infos.append( - { - "minLicense": { - "conforms": metadata["test-results"]["License-I"], - "url": minLicenseURL, - }, - "goodLicense": { - "conforms": metadata["test-results"]["License-II"], - "url": goodLicenseURL, - }, - "lode": { - "severity": inspectVocabs.hackyShaclInspection(lodeShaclURL), - "url": lodeShaclURL, - }, - "version": {"label": version, "url": versionURL}, - "consistent": { - "conforms": isConsistent(metadata["test-results"]["consistent"]), - "url": consistencyURL, - }, - "triples": metadata["ontology-info"]["triples"], - "parsing": { - "conforms": parsing, - "errors": metadata["logs"]["rapper-errors"], - }, - "semversion": metadata["ontology-info"]["semantic-version"], - "stars": stars, - "docuURL": docuURL, - } - ) - return title, comment, version_infos - - -def loadLastIndex(): - query = "\n".join( - ( - "PREFIX dataid: ", - "PREFIX dct: ", - "PREFIX dcat: ", - "PREFIX db: ", - "PREFIX rdf: ", - "PREFIX rdfs: ", - "SELECT DISTINCT ?downloadURL WHERE {", - "VALUES ?art { }", - "?dataset dataid:artifact ?art .", - "?dataset dct:hasVersion ?latestVersion .", - "?dataset dcat:distribution ?dst .", - "?dst dcat:downloadURL ?downloadURL ." "{", - "SELECT DISTINCT ?art (MAX(?v) as ?latestVersion) WHERE {", - "?dataset dataid:artifact ?art .", - "?dataset dct:hasVersion ?v .", - "}", - "}", - "}", - ) - ) - sparql = SPARQLWrapper(databusRepoUrl) - sparql.setQuery(query) - sparql.setReturnFormat(JSON) - results = sparql.query().convert() - - try: - downloadURL = results["results"]["bindings"][0]["downloadURL"]["value"] - except (KeyError, IndexError): - return None - - csvString = requests.get(downloadURL).text - csvIO = StringIO(csvString) - - return [tp for tp in csv.reader(csvIO, delimiter=",")] - - def get_last_official_index(): query = "\n".join( ( @@ -813,5 +704,24 @@ def get_VOID_URIs(): return [binding["URI"]["value"] for binding in results["results"]["bindings"]] +async def fetch_content_async( + session: aiohttp.ClientSession, + uri: str, + id: str, + logger=webservice_logger, + **kwargs, +): + if uri is None or uri == "": + return id, None + + try: + async with session.request("GET", url=uri, **kwargs) as resp: + text = await resp.text() + return id, text + except Exception: + logger.exception("Error during async content retrieval:") + return id, None + + if __name__ == "__main__": getDownloadURL("datashapes.org", "dash", fileExt="ttl", version="2020.07.16-115603") diff --git a/archivo/webservice/routes.py b/archivo/webservice/routes.py index 1eb07db..f32d325 100644 --- a/archivo/webservice/routes.py +++ b/archivo/webservice/routes.py @@ -13,6 +13,7 @@ import dbUtils from urllib.error import HTTPError, URLError import json +import asyncio # small hack for the correct path archivoPath = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] @@ -201,6 +202,91 @@ def vocabInfo(): title=f"Archivo - Ontology Info", ) +@app.route("/newinfo/", methods=["GET", "POST"]) +@app.route("/newinfo", methods=["GET", "POST"]) +@accept("text/html") +def newvocabInfo(): + args = request.args + ontoUri = args["o"] if "o" in args else "" + ontoUri = unquote(ontoUri) + isDev = True if "dev" in args else False + form = InfoForm() + allOntos = [ont.uri for ont in db.session.query(dbModels.OfficialOntology).all()] + if form.validate_on_submit(): + uri = form.uris.data.strip() + return redirect(f"/newinfo?o={uri}") + if ontoUri != "": + foundUri = stringTools.get_uri_from_index(ontoUri, allOntos) + if foundUri is None: + abort(status=404) + ont = ( + db.session.query(dbModels.OfficialOntology).filter_by(uri=foundUri).first() + ) + general_info = {} + general_info["hasDev"] = True if ont.devel is not None else False + group, artifact = stringTools.generateGroupAndArtifactFromUri( + foundUri, dev=isDev + ) + try: + title, comment, versions_info = asyncio.run(queryDatabus.artifact_info_async(group, artifact)) + except HTTPError as e: + general_info[ + "message" + ] = f"There seems to be a problem with the databus, please try it again later! {str(e)}" + return render_template( + "info.html", + general_info=general_info, + form=form, + title=f"Archivo - Info about {foundUri}", + ) + if isDev: + ont = ont.devel + general_info["sourceURI"] = ont.uri + general_info["source"] = ont.source + general_info["isDev"] = isDev + general_info["archivement"] = ont.accessDate + general_info["title"] = title + general_info["comment"] = comment + general_info[ + "databusArtifact" + ] = f"https://databus.dbpedia.org/ontologies/{group}/{artifact}" + general_info["nir"] = {"regular": foundUri, "encoded": quote(foundUri)} + # check latest crawling status + if ont.crawling_status: + general_info["access"] = {"status": ont.crawling_status, "message": ""} + elif ont.crawling_status is None: + general_info["access"] = { + "status": True, + "message": "No database entry -> no crawling happened", + } + else: + latestFallout = ( + db.session.query(dbModels.Fallout) + .filter_by(ontology=ont.uri) + .order_by(dbModels.Fallout.date.desc()) + .first() + ) + general_info["access"] = { + "status": ont.crawling_status, + "message": latestFallout.error, + } + for v in versions_info: + v["stars"] = stringTools.generateStarString(v["stars"]) + return render_template( + "newinfo.html", + versions_info=sorted( + versions_info, key=lambda d: d["version"]["label"], reverse=True + ), + general_info=general_info, + form=form, + title=f"Archivo - Info about {title}", + ) + return render_template( + "newinfo.html", + general_info={}, + form=form, + title=f"Archivo - Ontology Info", + ) @vocabInfo.support("text/turtle") def turtleInfo(): diff --git a/archivo/webservice/templates/list.html b/archivo/webservice/templates/list.html index 6a45c1e..0be6058 100644 --- a/archivo/webservice/templates/list.html +++ b/archivo/webservice/templates/list.html @@ -4,7 +4,17 @@

Archivo - Ontology Archive

Archivo automatically discovers OWL ontologies on the web and checks them every 8 hours. When changes are detected, Archivo downloads and rates and archives the latest snapshot persistently on the Databus. See the about page for details (paper & video).

- + +

News

+
+
+
Vocarnival at the SEMANTiCS 2021!
+ +

Submit your ontology to Archivo and prepare it for the Vocarnival at the SEMANTiCS 2021!

+ Ontology Submission + SEMANTiCS 2021 +
+

Status

At this moment Archivo contains {{ontoNumber}} Ontologies. See all available Ontologies

diff --git a/archivo/webservice/templates/newinfo.html b/archivo/webservice/templates/newinfo.html new file mode 100644 index 0000000..c37aa2a --- /dev/null +++ b/archivo/webservice/templates/newinfo.html @@ -0,0 +1,323 @@ +{% extends "index.html" %} {% block content %} +
+ + + +

+ + {% if general_info.title %} + +

given title: +

{{general_info.title}}

+ +

given comment: {{general_info.comment}} +

+

{% if general_info.hasDev %} Switch: + Official + Develop {% endif %} +

+

+ + + + + {% if general_info.sourceURI %} + + {% endif %} + + + + + + + + + + {% if general_info.sourceURI %} + + {% endif %} + + + + + + +
Ontology URISource URIFirst DiscoveryDiscovery SourceDatabus ArtifactAccessability? +
{{general_info.nir.regular}}{{ general_info.sourceURI }}{{general_info.archivement}}{{general_info.source}}Link + {% if general_info.access.status %} + + {% else %} + {% endif %} + +
+

+

+

+
+ + +
+ +

+
+ +{% else %} {% if general_info.message %} {{ general_info.message }} {% else %} +Ontology missing? Try adding it at the Archivo Suggestion Feature. {% endif %} {% endif %} + + + +{% endblock %} \ No newline at end of file