Feat: Add Dagger Pipelines to Harbor

.buildbaselog

@@ -8,6 +8,9 @@
 *   Add date here... Add signature here...
 -   Add your reason here...
 
+*   Aug 12 2025 <[email protected]>
+-   Refresh base image
+
 *   Oct 24 2024 <[email protected]>
 -   Refresh base image
 

.dagger/.gitattributes

@@ -0,0 +1,4 @@
+/dagger.gen.go linguist-generated
+/internal/dagger/** linguist-generated
+/internal/querybuilder/** linguist-generated
+/internal/telemetry/** linguist-generated

.dagger/.gitignore

@@ -0,0 +1,5 @@
+/dagger.gen.go
+/internal/dagger
+/internal/querybuilder
+/internal/telemetry
+/.env

.dagger/README.md

@@ -0,0 +1,115 @@
+# Dagger Harbor Setup and Usage Guide
+
+## Introduction
+
+This guide explains how to use Dagger to build, run, and test a Harbor setup locally.
+
+## Prerequisites
+
+- Dagger installed
+- Harbor source code available
+- Generate `consts.go`
+
+## Getting Started
+
+### Generate `consts.go` file
+```bash
+go run ./.dagger/scripts/parseMakefile.go
+```
+
+### Step 1: Build and Spin Up Harbor Components Locally
+
+Run the following command to build and start the necessary Harbor components inside Dagger:
+
+```bash
+dagger call run-dev up -v
+```
+
+This command will:
+
+- Build the Harbor components.
+- Spin up the Harbor backend inside Dagger, similar to running the entire Harbor server.
+- Bind Harbor to port `8080`.
+
+### Step 2: Start the Portal
+
+After running the Harbor backend, you can move to the portal directory to spin up the Harbor portal:
+
+```bash
+cd ./src/portal
+```
+
+> **Note:** The current portal image has some issues.
+So follow the instructions in the `./src/portal/README.md` for setup.
+
+Make sure to configure the portal to connect to the Harbor backend by setting the `target-harbor-server` in the portal's configuration to:
+
+```bash
+http://localhost:8080
+```
+
+This ensures the portal connects to the Harbor running inside Dagger.
+
+### Step 3: Running Harbor Locally
+
+Once both Harbor backend and the portal are set up, you will have a working Harbor setup. You can now use this setup for building, running, and testing Harbor locally.
+
+## Available Functions in Dagger
+
+You can list all available functions in Dagger by running:
+
+```bash
+dagger functions
+```
+
+This will display a list of functions you can use. Currently, we have the following functions:
+
+### 1. **publish-all-images**
+
+Publishes all images in the registry.
+
+Example usage:
+
+```bash
+dagger call publish-all-images --registry-username=harbor-cli --registry=demo.goharbor.io --registry-password=env:REGPASS --image-tags v3.0.0 --version v3.0.0 --debugbin=false --project-name=library/dagger-test -vvv
+```
+
+- `-vvv` flag is used for highly verbose output. You can remove this flag for a less verbose output.
+- Feel free to change the version and registry flags according to your needs.
+
+### 2. **publish-image**
+
+Publishes a specific image package.
+
+Example usage:
+
+```bash
+dagger call publish-image --registry-username=admin --registry=ttl.sh --registry-password=env:REGPASS --image-tags v3.2.2 --version v3.0 --pkg registryctl --debugbin=false --project-name=library/dagger-test -vvv -i
+```
+
+This will publish the `registryctl` package.
+
+### 3. **build-binary**
+
+Builds specific Harbor binaries for a given platform.
+
+Example usage:
+
+```bash
+dagger call build-binary --pkg core --platform "linux/amd64" --version v2.12.2 --debugbin=false export --path=bin/harbor_core
+```
+
+This command will build the `core` package for the `linux/amd64` platform and export the binary to `harbor_core`.
+
+#### Extras
+Use these to pull the `dagger.json` and `.dagger` folder
+In case if you want to use dagger in other branches this might be helpful. 
+Also while using this don't forget to Generate consts file based on the branch 
+```
+oras pull bupd/harbor-dagger-dir:latest
+oras pull bupd/harbor-dagger-json:latest
+```
+
+## Conclusion
+
+By following the above steps, you can have a fully functional Harbor setup running inside Dagger. You can use this setup for local development and testing. The available Dagger functions like `publish-all-images`, `publish-image`, and `build-binary` make it easy to manage Harbor images and binaries.

.dagger/config/core/app.conf

@@ -0,0 +1,7 @@
+appname = Harbor
+# runmode = prod
+runmode = dev
+enablegzip = true
+
+[prod]
+httpport = 8080

.dagger/config/core/env

@@ -0,0 +1,51 @@
+CONFIG_PATH=/etc/core/app.conf
+UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
+_REDIS_URL_CORE=redis://redis:6379?idle_timeout_seconds=30
+SYNC_QUOTA=true
+_REDIS_URL_REG=redis://redis:6379/1?idle_timeout_seconds=30
+
+LOG_LEVEL=debug
+EXT_ENDPOINT=http://localhost
+DATABASE_TYPE=postgresql
+POSTGRESQL_HOST=postgresql
+POSTGRESQL_PORT=5432
+POSTGRESQL_USERNAME=postgres
+POSTGRESQL_PASSWORD=root123
+POSTGRESQL_DATABASE=registry
+POSTGRESQL_SSLMODE=disable
+POSTGRESQL_MAX_IDLE_CONNS=100
+POSTGRESQL_MAX_OPEN_CONNS=900
+POSTGRESQL_CONN_MAX_LIFETIME=5m
+POSTGRESQL_CONN_MAX_IDLE_TIME=0
+REGISTRY_URL=http://registry:5000
+PORTAL_URL=http://portal:8080
+TOKEN_SERVICE_URL=http://core:8080/service/token
+HARBOR_ADMIN_PASSWORD=Harbor12345
+MAX_JOB_WORKERS=10
+CORE_SECRET=D1fWkYKg6OQgISeP
+JOBSERVICE_SECRET=gVQJX0AlVjstElN8
+WITH_TRIVY=False
+CORE_URL=http://core:8080
+CORE_LOCAL_URL=http://127.0.0.1:8080
+JOBSERVICE_URL=http://jobservice:8080
+TRIVY_ADAPTER_URL=http://trivy-adapter:8080
+REGISTRY_STORAGE_PROVIDER_NAME=filesystem
+READ_ONLY=false
+RELOAD_KEY=
+REGISTRY_CONTROLLER_URL=http://registryctl:8080
+REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user
+REGISTRY_CREDENTIAL_PASSWORD=TRJUhYbJgSjXZWAj3oLEet3ugJ3nAOk3
+CSRF_KEY=
+ROBOT_SCANNER_NAME_PREFIX=FFTP6U3S
+PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE=docker-hub,harbor,azure-acr,ali-acr,aws-ecr,google-gcr,quay,docker-registry,github-ghcr,jfrog-artifactory
+
+HTTP_PROXY=
+HTTPS_PROXY=
+NO_PROXY=db,localhost,redis,log,exporter,postgresql,trivy-adapter,portal,.internal,registry,127.0.0.1,jobservice,nginx,.local,core,registryctl
+
+PORT=8080
+
+
+
+
+QUOTA_UPDATE_PROVIDER=db

.dagger/config/debug_entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# Execute the command (passed as $1 arg)
+echo "Executing: dlv --headless=true --listen=0.0.0.0:4001 --accept-multiclient --log-output=debugger,debuglineerr,gdbwire,lldbout,rpc --log=true --continue --api-version=2 exec $1"
+
+# Start the dlv process in the background
+# /root/go/bin/dlv exec --headless --listen localhost:$2 $1
+dlv --headless=true --listen=0.0.0.0:4001 --accept-multiclient --log-output=debugger,debuglineerr,gdbwire,lldbout,rpc --log=true --continue --api-version=2 exec $1
+# dlv --headless=true --listen=0.0.0.0:4001 --accept-multiclient --log-output=debugger,debuglineerr,gdbwire,lldbout,rpc --log=true --api-version=2 attach $pid

.dagger/config/jobservice/config.yml

@@ -0,0 +1,47 @@
+# original config
+---
+#Protocol used to serve
+protocol: "http"
+
+#Server listening port
+port: 8080
+
+#Worker pool
+worker_pool:
+  #Worker concurrency
+  workers: 10
+  backend: "redis"
+  #Additional config if use 'redis' backend
+  redis_pool:
+    #redis://[arbitrary_username:password@]ipaddress:port/database_index
+    redis_url: redis://redis:6379/2?idle_timeout_seconds=30
+    namespace: "harbor_job_service_namespace"
+    idle_timeout_second: 3600
+#Loggers for the running job
+job_loggers:
+  # The jobLoggers backend name, only support "STD_OUTPUT", "FILE" and/or "DB"
+  - name: "STD_OUTPUT"
+    level: "DEBUG" # INFO/DEBUG/WARNING/ERROR/FATAL
+  - name: "FILE"
+    level: "DEBUG"
+    settings: # Customized settings of logger
+      base_dir: "/var/log/jobs"
+    sweeper:
+      duration: 1 #days
+      settings: # Customized settings of sweeper
+        work_dir: "/var/log/jobs"
+
+#Loggers for the job service
+loggers:
+  - name: "STD_OUTPUT" # Same with above
+    level: "DEBUG"
+
+
+reaper:
+  # the max time to wait for a task to finish, if unfinished after max_update_hours, the task will be mark as error, but the task will continue to run, default value is 24,
+  max_update_hours: 24
+  # the max time for execution in running state without new task created
+  max_dangling_hours: 168
+
+# the max size of job log returned by API, default is 10M
+max_retrieve_size_mb: 10

.dagger/config/jobservice/env

@@ -0,0 +1,17 @@
+CORE_SECRET=D1fWkYKg6OQgISeP
+REGISTRY_URL=http://registry:5000
+JOBSERVICE_SECRET=gVQJX0AlVjstElN8
+CORE_URL=http://core:8080
+REGISTRY_CONTROLLER_URL=http://registryctl:8080
+JOBSERVICE_WEBHOOK_JOB_MAX_RETRY=3
+JOBSERVICE_WEBHOOK_JOB_HTTP_CLIENT_TIMEOUT=3
+
+
+HTTP_PROXY=
+HTTPS_PROXY=
+NO_PROXY=db,localhost,redis,log,exporter,postgresql,trivy-adapter,portal,.internal,registry,127.0.0.1,jobservice,nginx,.local,core,registryctl
+REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user
+REGISTRY_CREDENTIAL_PASSWORD=TRJUhYbJgSjXZWAj3oLEet3ugJ3nAOk3
+
+
+

.dagger/config/portal/nginx.conf

@@ -0,0 +1,42 @@
+# original conf
+worker_processes auto;
+pid /tmp/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+
+    client_body_temp_path /tmp/client_body_temp;
+    proxy_temp_path /tmp/proxy_temp;
+    fastcgi_temp_path /tmp/fastcgi_temp;
+    uwsgi_temp_path /tmp/uwsgi_temp;
+    scgi_temp_path /tmp/scgi_temp;
+
+    server {
+        listen 8080;
+        server_name  localhost;
+
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+        include /etc/nginx/mime.types;
+
+        gzip on;
+        gzip_min_length 1000;
+        gzip_proxied expired no-cache no-store private auth;
+        gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+        location /devcenter-api-2.0 {
+            try_files $uri $uri/ /swagger-ui-index.html;
+        }
+
+        location / {
+            try_files $uri $uri/ /index.html;
+        }
+
+        location = /index.html {
+            add_header Cache-Control "no-store, no-cache, must-revalidate";
+        }
+    }
+}