Improve Navigation Bar

.ARCHIVE/ETL/events/event.json

@@ -1,3 +1,3 @@
-{
-    "name": "collect_parkingsensor"
+{
+    "name": "collect_parkingsensor"
 }

.ARCHIVE/ETL/logic/data.py

@@ -1,17 +1,17 @@
-import botocore
-import pandas as pd
-
-
-def get_csv(bucket, key, s3_resource, fallback=None):
-    try:
-        # try and get the csv from s3 if it exists
-        s3_resource.Object(bucket, key).load()
-        read_file = s3_resource.Object(
-            bucket, key).get()  # note key is on line 27
-        df = pd.read_csv(read_file['Body'])
-        return df  # return the s3 csv
-    except botocore.exceptions.ClientError as e:
-        if e.response['Error']['Code'] == "404":
-            return fallback  # if the key does not exist then we return the fallback csv
-        else:
-            raise  # throw error because something else is wrong
+import botocore
+import pandas as pd
+
+
+def get_csv(bucket, key, s3_resource, fallback=None):
+    try:
+        # try and get the csv from s3 if it exists
+        s3_resource.Object(bucket, key).load()
+        read_file = s3_resource.Object(
+            bucket, key).get()  # note key is on line 27
+        df = pd.read_csv(read_file['Body'])
+        return df  # return the s3 csv
+    except botocore.exceptions.ClientError as e:
+        if e.response['Error']['Code'] == "404":
+            return fallback  # if the key does not exist then we return the fallback csv
+        else:
+            raise  # throw error because something else is wrong

.ARCHIVE/ETL/logic/parking_availability.py

@@ -1,89 +1,89 @@
-import datetime
-from io import StringIO
-
-import boto3
-import pandas as pd
-from sodapy import Socrata
-
-from .data import get_csv
-
-""" Given a schedule -
- will update the csv file for the corresponding day with the parking sensor statuses for the given day
-"""
-
-
-def update_daily_parking():
-    bucket = 'opendataplayground.deakin'
-    # get existing dataframe from csv on S3
-    s3_resource = boto3.resource('s3')
-
-    a = datetime.datetime.today().replace(microsecond=0)
-    ts = pd.Timestamp(a, tz="UTC")
-    d = ts.tz_convert(tz='Australia/Victoria')
-    key = f'parkingsensor/daily/{d.date()}.csv'
-
-    # get the csv for the key above, if it can't fetch the csv will return the fallback (empty df)
-    df = get_csv(
-        bucket,
-        key,
-        s3_resource,
-        fallback=pd.DataFrame(
-            {'bay_id': [], 'st_marker_id': [], 'status': [], 'datetime': []})
-    )
-    client = Socrata(
-        "data.melbourne.vic.gov.au",
-        # app token, just used to reduce throttling, not authentication
-        "EC65cHicC3xqFXHHvAUICVXEr",
-        timeout=120
-    )
-
-    # add another number to the csv
-    df1 = pd.DataFrame(client.get("vh2v-4nfs", limit=200000))
-    df1.drop(columns=['location', 'lat', 'lon',
-             ':@computed_region_evbi_jbp8'], inplace=True)
- #  df1['datetime'] = datetime.datetime.today().replace(microsecond=0) + datetime.timedelta(hours = 10) #Melbourne Time
-    # This is changing it to Melbourne Timezone won't be affect by DST
-    df1['datetime'] = d
-    df = df.append(df1)  # append the new data to the dataframe
-
-    # write the csv to a buffer
-    csv_buffer = StringIO()
-    df.to_csv(csv_buffer, index=False)
-    s3_resource.Object(
-        bucket, f'parkingsensor/daily/{d.date()}.csv').put(Body=csv_buffer.getvalue())
-    return f"CSV now has {len(df)} rows"
-
-
-def collect_parkingsensor():
-    import datetime as dt
-    current_time = dt.datetime.now()
-
-    bucket = 'opendataplayground.deakin'
-    # get existing dataframe from csv on S3
-    s3_resource = boto3.resource('s3')
-
-    def get_daily_csv(filename): return get_csv(
-        bucket,
-        filename,
-        s3_resource,
-        fallback=pd.DataFrame(
-            {'bay_id': [], 'st_marker_id': [], 'status': [], 'datetime': []})
-    )
-
-    # collect data from previous 29 days
-    dataframe = None
-    for day in range(1, 29):
-        d = current_time - dt.timedelta(days=day)
-        daily_filename = f'parkingsensor/daily/{d.date()}.csv'
-        csv = get_daily_csv(
-            daily_filename)
-
-        if dataframe is None:
-            dataframe = csv
-        else:
-            dataframe = dataframe.append(csv)
-
-    csv_buffer = StringIO()
-    dataframe.to_csv(csv_buffer, index=False)
-    s3_resource.Object(
-        bucket, f'parkingsensor/parkingsensor.csv').put(Body=csv_buffer.getvalue())
+import datetime
+from io import StringIO
+
+import boto3
+import pandas as pd
+from sodapy import Socrata
+
+from .data import get_csv
+
+""" Given a schedule -
+ will update the csv file for the corresponding day with the parking sensor statuses for the given day
+"""
+
+
+def update_daily_parking():
+    bucket = 'opendataplayground.deakin'
+    # get existing dataframe from csv on S3
+    s3_resource = boto3.resource('s3')
+
+    a = datetime.datetime.today().replace(microsecond=0)
+    ts = pd.Timestamp(a, tz="UTC")
+    d = ts.tz_convert(tz='Australia/Victoria')
+    key = f'parkingsensor/daily/{d.date()}.csv'
+
+    # get the csv for the key above, if it can't fetch the csv will return the fallback (empty df)
+    df = get_csv(
+        bucket,
+        key,
+        s3_resource,
+        fallback=pd.DataFrame(
+            {'bay_id': [], 'st_marker_id': [], 'status': [], 'datetime': []})
+    )
+    client = Socrata(
+        "data.melbourne.vic.gov.au",
+        # app token, just used to reduce throttling, not authentication
+        "EC65cHicC3xqFXHHvAUICVXEr",
+        timeout=120
+    )
+
+    # add another number to the csv
+    df1 = pd.DataFrame(client.get("vh2v-4nfs", limit=200000))
+    df1.drop(columns=['location', 'lat', 'lon',
+             ':@computed_region_evbi_jbp8'], inplace=True)
+ #  df1['datetime'] = datetime.datetime.today().replace(microsecond=0) + datetime.timedelta(hours = 10) #Melbourne Time
+    # This is changing it to Melbourne Timezone won't be affect by DST
+    df1['datetime'] = d
+    df = df.append(df1)  # append the new data to the dataframe
+
+    # write the csv to a buffer
+    csv_buffer = StringIO()
+    df.to_csv(csv_buffer, index=False)
+    s3_resource.Object(
+        bucket, f'parkingsensor/daily/{d.date()}.csv').put(Body=csv_buffer.getvalue())
+    return f"CSV now has {len(df)} rows"
+
+
+def collect_parkingsensor():
+    import datetime as dt
+    current_time = dt.datetime.now()
+
+    bucket = 'opendataplayground.deakin'
+    # get existing dataframe from csv on S3
+    s3_resource = boto3.resource('s3')
+
+    def get_daily_csv(filename): return get_csv(
+        bucket,
+        filename,
+        s3_resource,
+        fallback=pd.DataFrame(
+            {'bay_id': [], 'st_marker_id': [], 'status': [], 'datetime': []})
+    )
+
+    # collect data from previous 29 days
+    dataframe = None
+    for day in range(1, 29):
+        d = current_time - dt.timedelta(days=day)
+        daily_filename = f'parkingsensor/daily/{d.date()}.csv'
+        csv = get_daily_csv(
+            daily_filename)
+
+        if dataframe is None:
+            dataframe = csv
+        else:
+            dataframe = dataframe.append(csv)
+
+    csv_buffer = StringIO()
+    dataframe.to_csv(csv_buffer, index=False)
+    s3_resource.Object(
+        bucket, f'parkingsensor/parkingsensor.csv').put(Body=csv_buffer.getvalue())

.ARCHIVE/ETL/samconfig.toml

@@ -1,12 +1,12 @@
-version = 0.1
-[default]
-[default.deploy]
-[default.deploy.parameters]
-stack_name = "PlaygroundETL"
-s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-15sa8a759vnvg"
-s3_prefix = "PlaygroundETL"
-region = "ap-southeast-2"
-confirm_changeset = true
-capabilities = "CAPABILITY_IAM"
-disable_rollback = true
-image_repositories = ["PlaygroundETL=119291007423.dkr.ecr.ap-southeast-2.amazonaws.com/playgroundetl239ee983/playgroundetl239ee983repo"]
+version = 0.1
+[default]
+[default.deploy]
+[default.deploy.parameters]
+stack_name = "PlaygroundETL"
+s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-15sa8a759vnvg"
+s3_prefix = "PlaygroundETL"
+region = "ap-southeast-2"
+confirm_changeset = true
+capabilities = "CAPABILITY_IAM"
+disable_rollback = true
+image_repositories = ["PlaygroundETL=119291007423.dkr.ecr.ap-southeast-2.amazonaws.com/playgroundetl239ee983/playgroundetl239ee983repo"]

.ARCHIVE/google_cloud_test/.dockerignore

@@ -1,7 +1,7 @@
-Dockerfile
-README.md
-*.pyc
-*.pyo
-*.pyd
-__pycache__
+Dockerfile
+README.md
+*.pyc
+*.pyo
+*.pyd
+__pycache__
 .pytest_cache

.ARCHIVE/google_cloud_test/Dockerfile

@@ -1,22 +1,22 @@
-
-# Use the official lightweight Python image.
-# https://hub.docker.com/_/python
-FROM python:3.9-slim
-
-# Allow statements and log messages to immediately appear in the Knative logs
-ENV PYTHONUNBUFFERED True
-
-# Copy local code to the container image.
-ENV APP_HOME /app
-WORKDIR $APP_HOME
-COPY . ./
-
-# Install production dependencies.
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Run the web service on container startup. Here we use the gunicorn
-# webserver, with one worker process and 8 threads.
-# For environments with multiple CPU cores, increase the number of workers
-# to be equal to the cores available.
-# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling.
+
+# Use the official lightweight Python image.
+# https://hub.docker.com/_/python
+FROM python:3.9-slim
+
+# Allow statements and log messages to immediately appear in the Knative logs
+ENV PYTHONUNBUFFERED True
+
+# Copy local code to the container image.
+ENV APP_HOME /app
+WORKDIR $APP_HOME
+COPY . ./
+
+# Install production dependencies.
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Run the web service on container startup. Here we use the gunicorn
+# webserver, with one worker process and 8 threads.
+# For environments with multiple CPU cores, increase the number of workers
+# to be equal to the cores available.
+# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling.
 CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app

.ARCHIVE/google_cloud_test/ReadMe.md

@@ -1,30 +1,30 @@
-## Deployment & Architecture
-Below is outlined a pathway for deploying current webapp application (that is deployed in AWS) into Google Cloud.
-
-### Google Cloud Run
-Great tool for deploying docker applications as serverless web applications.
-[Learn why here.](https://cloud.google.com/blog/topics/developers-practitioners/cloud-run-story-serverless-containers)
-
-#### Steps to Develop
-1. Install the google cloud SDK
-1. Run the following login inside of the Google Console  
-    1. ```gcloud auth login``` - login with deakin
-    1. ```gcloud auth application-default login``` - this will setup your environment with user account principal so you can run and work with gcloud resources on your local machine (like cloud storage)  
-    
-    [View Auth Documentation here](https://googleapis.dev/python/google-api-core/latest/auth.html)
-1. At this point you should have everything setup on your machine to work with gcloud cli and gcloud resources.
-
-#### Steps to deploy
-1. Install Google Cloud SDK Cli
-1. Login using Deakin Credentials
-1. Copy Dockerfile to `../webapp` directory
-1. Run `gcloud run deploy test-cloud-run-again --source  . --allow-unauthenticated` from the webapp directory.
-1. Select 10 as region
-1. Application should be deployed
-    * In the Deakin Project, you will likely not have permission to perform the `--allow-unauthenticated` action.  
-    You can still test the instance as authenticated user using the steps outlined [here](https://cloud.google.com/run/docs/authenticating/developers).
-    * To deploy to production / public, check in with `Nghia Dang` to request `Cloud Run Invoker` role be added to anonymous users for your cloud run application ([Documentation here](https://cloud.google.com/run/docs/authenticating/public)).
-1. You can now test the application by calling 
-    ``` 
-        curl -H "Authorization: Bearer $(gcloud auth print-identity-token)" [url to deployed instance]
+## Deployment & Architecture
+Below is outlined a pathway for deploying current webapp application (that is deployed in AWS) into Google Cloud.
+
+### Google Cloud Run
+Great tool for deploying docker applications as serverless web applications.
+[Learn why here.](https://cloud.google.com/blog/topics/developers-practitioners/cloud-run-story-serverless-containers)
+
+#### Steps to Develop
+1. Install the google cloud SDK
+1. Run the following login inside of the Google Console  
+    1. ```gcloud auth login``` - login with deakin
+    1. ```gcloud auth application-default login``` - this will setup your environment with user account principal so you can run and work with gcloud resources on your local machine (like cloud storage)  
+
+    [View Auth Documentation here](https://googleapis.dev/python/google-api-core/latest/auth.html)
+1. At this point you should have everything setup on your machine to work with gcloud cli and gcloud resources.
+
+#### Steps to deploy
+1. Install Google Cloud SDK Cli
+1. Login using Deakin Credentials
+1. Copy Dockerfile to `../webapp` directory
+1. Run `gcloud run deploy test-cloud-run-again --source  . --allow-unauthenticated` from the webapp directory.
+1. Select 10 as region
+1. Application should be deployed
+    * In the Deakin Project, you will likely not have permission to perform the `--allow-unauthenticated` action.  
+    You can still test the instance as authenticated user using the steps outlined [here](https://cloud.google.com/run/docs/authenticating/developers).
+    * To deploy to production / public, check in with `Nghia Dang` to request `Cloud Run Invoker` role be added to anonymous users for your cloud run application ([Documentation here](https://cloud.google.com/run/docs/authenticating/public)).
+1. You can now test the application by calling 
+    ``` 
+        curl -H "Authorization: Bearer $(gcloud auth print-identity-token)" [url to deployed instance]
     ```