Add GitHub Actions CI workflow for unit tests

.github/workflows/ci.yml

@@ -0,0 +1,120 @@
+name: CI
+
+on:
+  push:
+    branches: [ master, develop ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+    inputs:
+      coin_filter:
+        description: 'Coin filter regex (e.g. "bitcoin", "ethereum", "bitcoin|ethereum|bsc"). Leave empty for all coins.'
+        required: false
+        default: ''
+      run_unit_tests:
+        description: 'Run unit tests'
+        type: boolean
+        default: true
+      run_connectivity_tests:
+        description: 'Run connectivity tests'
+        type: boolean
+        default: true
+      run_integration_tests:
+        description: 'Run integration tests (rpc + sync)'
+        type: boolean
+        default: false
+
+jobs:
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_unit_tests }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Build Docker image
+      run: make build-images
+
+    - name: Run unit tests
+      run: make test
+
+  connectivity-tests:
+    name: Backend Connectivity Tests
+    runs-on: ubuntu-latest
+    needs: unit-tests
+    if: |
+      always() &&
+      (needs.unit-tests.result == 'success' || needs.unit-tests.result == 'skipped') &&
+      (github.event_name != 'workflow_dispatch' || inputs.run_connectivity_tests) &&
+      vars.BB_RPC_URL_HTTP_avalanche != ''
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Build Docker image
+      run: make build-images
+
+    - name: Run connectivity tests
+      env:
+        # EVM chains with publicly reachable endpoints only.
+        # Ethereum and BSC excluded until valid public RPC URLs are available.
+        # UTXO chains are tested on the dev server where internal backends are reachable.
+        BB_RPC_URL_HTTP_avalanche: ${{ vars.BB_RPC_URL_HTTP_avalanche }}
+        BB_RPC_URL_WS_avalanche: ${{ vars.BB_RPC_URL_WS_avalanche }}
+        BB_RPC_URL_HTTP_polygon_bor: ${{ vars.BB_RPC_URL_HTTP_polygon_bor }}
+        BB_RPC_URL_WS_polygon_bor: ${{ vars.BB_RPC_URL_WS_polygon_bor }}
+        BB_RPC_URL_HTTP_arbitrum: ${{ vars.BB_RPC_URL_HTTP_arbitrum }}
+        BB_RPC_URL_WS_arbitrum: ${{ vars.BB_RPC_URL_WS_arbitrum }}
+        BB_RPC_URL_HTTP_optimism: ${{ vars.BB_RPC_URL_HTTP_optimism }}
+        BB_RPC_URL_WS_optimism: ${{ vars.BB_RPC_URL_WS_optimism }}
+        BB_RPC_URL_HTTP_base: ${{ vars.BB_RPC_URL_HTTP_base }}
+        BB_RPC_URL_WS_base: ${{ vars.BB_RPC_URL_WS_base }}
+      run: |
+        # Default to EVM-only coins (publicly reachable from GitHub runners).
+        # Ethereum and BSC excluded until valid public RPC URLs are available.
+        # UTXO chains use internal backends and are tested on the dev server.
+        EVM_COINS="avalanche|arbitrum|base|optimism|polygon"
+        FILTER="${{ inputs.coin_filter }}"
+        FILTER="${FILTER:-$EVM_COINS}"
+        make test-connectivity ARGS="-run 'TestIntegration/($FILTER)=.*/connectivity'"
+
+  integration-tests:
+    name: Integration Tests (RPC + Sync)
+    runs-on: ubuntu-latest
+    needs: connectivity-tests
+    if: |
+      always() &&
+      (needs.connectivity-tests.result == 'success' || needs.connectivity-tests.result == 'skipped') &&
+      github.event_name == 'workflow_dispatch' && inputs.run_integration_tests
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Build Docker image
+      run: make build-images
+
+    - name: Run integration tests
+      env:
+        # EVM chains with publicly reachable endpoints only.
+        # Ethereum and BSC excluded until valid public RPC URLs are available.
+        BB_RPC_URL_HTTP_avalanche: ${{ vars.BB_RPC_URL_HTTP_avalanche }}
+        BB_RPC_URL_WS_avalanche: ${{ vars.BB_RPC_URL_WS_avalanche }}
+        BB_RPC_URL_HTTP_polygon_bor: ${{ vars.BB_RPC_URL_HTTP_polygon_bor }}
+        BB_RPC_URL_WS_polygon_bor: ${{ vars.BB_RPC_URL_WS_polygon_bor }}
+        BB_RPC_URL_HTTP_arbitrum: ${{ vars.BB_RPC_URL_HTTP_arbitrum }}
+        BB_RPC_URL_WS_arbitrum: ${{ vars.BB_RPC_URL_WS_arbitrum }}
+        BB_RPC_URL_HTTP_optimism: ${{ vars.BB_RPC_URL_HTTP_optimism }}
+        BB_RPC_URL_WS_optimism: ${{ vars.BB_RPC_URL_WS_optimism }}
+        BB_RPC_URL_HTTP_base: ${{ vars.BB_RPC_URL_HTTP_base }}
+        BB_RPC_URL_WS_base: ${{ vars.BB_RPC_URL_WS_base }}
+      run: |
+        # Default to EVM-only coins (publicly reachable from GitHub runners).
+        # Ethereum and BSC excluded until valid public RPC URLs are available.
+        EVM_COINS="avalanche|arbitrum|base|optimism|polygon"
+        FILTER="${{ inputs.coin_filter }}"
+        FILTER="${FILTER:-$EVM_COINS}"
+        make test-integration ARGS="-run 'TestIntegration/($FILTER)=.*/' -v"

.gitignore

@@ -15,4 +15,8 @@ build/*.deb
 .bin-image
 .deb-image
 \.idea/
-__debug*
+__debug*
+.gocache/
+
+# Environment files with credentials
+*.env

Makefile

@@ -7,34 +7,39 @@ NO_CACHE = false
 TCMALLOC =
 PORTABLE = 0
 ARGS ?=
+# Forward BB_RPC_* overrides into Docker so template generation sees desired endpoints/binds/allow lists.
+BB_RPC_ENV := $(shell env | awk -F= '/^BB_RPC_(URL_HTTP|URL_WS|BIND_HOST|ALLOW_IP)_/ {print "-e " $$1}')
 
 TARGETS=$(subst .json,, $(shell ls configs/coins))
 
 .PHONY: build build-debug test deb
 
 build: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build ARGS="$(ARGS)"
 
 build-debug: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build-debug ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build-debug ARGS="$(ARGS)"
 
 test: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test ARGS="$(ARGS)"
 
 test-integration: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-integration ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-integration ARGS="$(ARGS)"
+
+test-connectivity: .bin-image
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-connectivity ARGS="$(ARGS)"
 
 test-all: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-all ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-all ARGS="$(ARGS)"
 
 deb-backend-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v /var/run/docker.sock:/var/run/docker.sock -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v /var/run/docker.sock:/var/run/docker.sock -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
 
 deb-blockbook-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh blockbook $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh blockbook $* $(ARGS)
 
 deb-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v /var/run/docker.sock:/var/run/docker.sock -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh all $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) $(BB_RPC_ENV) -v /var/run/docker.sock:/var/run/docker.sock -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh all $* $(ARGS)
 
 deb-blockbook-all: clean-deb $(addprefix deb-blockbook-, $(TARGETS))
 

api/worker.go

@@ -965,7 +965,7 @@ func computePaging(count, page, itemsOnPage int) (Paging, int, int, int) {
 	}, from, to, page
 }
 
-func (w *Worker) getEthereumContractBalance(addrDesc bchain.AddressDescriptor, index int, c *db.AddrContract, details AccountDetails, ticker *common.CurrencyRatesTicker, secondaryCoin string) (*Token, error) {
+func (w *Worker) getEthereumContractBalance(addrDesc bchain.AddressDescriptor, index int, c *db.AddrContract, details AccountDetails, ticker *common.CurrencyRatesTicker, secondaryCoin string, erc20Balance *big.Int) (*Token, error) {
 	standard := bchain.EthereumTokenStandardMap[c.Standard]
 	ci, validContract, err := w.getContractDescriptorInfo(c.Contract, standard)
 	if err != nil {
@@ -985,11 +985,16 @@ func (w *Worker) getEthereumContractBalance(addrDesc bchain.AddressDescriptor, i
 	if details >= AccountDetailsTokenBalances && validContract {
 		if c.Standard == bchain.FungibleToken {
 			// get Erc20 Contract Balance from blockchain, balance obtained from adding and subtracting transfers is not correct
-			b, err := w.chain.EthereumTypeGetErc20ContractBalance(addrDesc, c.Contract)
-			if err != nil {
-				// return nil, nil, nil, errors.Annotatef(err, "EthereumTypeGetErc20ContractBalance %v %v", addrDesc, c.Contract)
-				glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err)
-			} else {
+			// Prefer pre-fetched batch balance when available to avoid redundant RPC calls.
+			b := erc20Balance
+			if b == nil {
+				b, err = w.chain.EthereumTypeGetErc20ContractBalance(addrDesc, c.Contract)
+				if err != nil {
+					// return nil, nil, nil, errors.Annotatef(err, "EthereumTypeGetErc20ContractBalance %v %v", addrDesc, c.Contract)
+					glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err)
+				}
+			}
+			if b != nil {
 				t.BalanceSat = (*Amount)(b)
 				if secondaryCoin != "" {
 					baseRate, found := w.GetContractBaseRate(ticker, t.Contract, 0)
@@ -1102,22 +1107,26 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto
 	var n uint64
 	// unknown number of results for paging initially
 	d := ethereumTypeAddressData{totalResults: -1}
+	// Load cached contract list and totals from the index; this drives token lookups.
 	ca, err := w.db.GetAddrDescContracts(addrDesc)
 	if err != nil {
 		return nil, nil, NewAPIError(fmt.Sprintf("Address not found, %v", err), true)
 	}
+	// Always fetch the native balance from the backend.
 	b, err := w.chain.EthereumTypeGetBalance(addrDesc)
 	if err != nil {
 		return nil, nil, errors.Annotatef(err, "EthereumTypeGetBalance %v", addrDesc)
 	}
 	var filterDesc bchain.AddressDescriptor
 	if filter.Contract != "" {
+		// Optional contract filter narrows token balances and tx paging to a single contract.
 		filterDesc, err = w.chainParser.GetAddrDescFromAddress(filter.Contract)
 		if err != nil {
 			return nil, nil, NewAPIError(fmt.Sprintf("Invalid contract filter, %v", err), true)
 		}
 	}
 	if ca != nil {
+		// Address has indexed contract/tx data; include totals and nonce.
 		ba = &db.AddrBalance{
 			Txs: uint32(ca.TotalTxs),
 		}
@@ -1129,6 +1138,36 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto
 			return nil, nil, errors.Annotatef(err, "EthereumTypeGetNonce %v", addrDesc)
 		}
 		ticker := w.fiatRates.GetCurrentTicker("", "")
+		var erc20Balances map[string]*big.Int
+		if details >= AccountDetailsTokenBalances && len(ca.Contracts) > 1 {
+			// Batch ERC20 balanceOf calls to cut per-contract RPC; fallback is single-call per contract.
+			erc20Contracts := make([]bchain.AddressDescriptor, 0, len(ca.Contracts))
+			for i := range ca.Contracts {
+				c := &ca.Contracts[i]
+				// Only fungible tokens are eligible; respect a contract filter if present.
+				if c.Standard != bchain.FungibleToken {
+					continue
+				}
+				if len(filterDesc) > 0 && !bytes.Equal(filterDesc, c.Contract) {
+					continue
+				}
+				erc20Contracts = append(erc20Contracts, c.Contract)
+			}
+			if len(erc20Contracts) > 1 {
+				balances, err := w.chain.EthereumTypeGetErc20ContractBalances(addrDesc, erc20Contracts)
+				if err != nil {
+					glog.Warningf("EthereumTypeGetErc20ContractBalances addr %v: %v", addrDesc, err)
+				} else if len(balances) == len(erc20Contracts) {
+					// Keep only successful batch results; missing entries will trigger per-contract calls.
+					erc20Balances = make(map[string]*big.Int, len(erc20Contracts))
+					for i, bal := range balances {
+						if bal != nil {
+							erc20Balances[string(erc20Contracts[i])] = bal
+						}
+					}
+				}
+			}
+		}
 		if details > AccountDetailsBasic {
 			d.tokens = make([]Token, len(ca.Contracts))
 			var j int
@@ -1141,7 +1180,12 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto
 					// filter only transactions of this contract
 					filter.Vout = i + db.ContractIndexOffset
 				}
-				t, err := w.getEthereumContractBalance(addrDesc, i+db.ContractIndexOffset, c, details, ticker, secondaryCoin)
+				// Use prefetched batch balances when available; nil triggers per-contract RPC in helper.
+				var erc20Balance *big.Int
+				if erc20Balances != nil {
+					erc20Balance = erc20Balances[string(c.Contract)]
+				}
+				t, err := w.getEthereumContractBalance(addrDesc, i+db.ContractIndexOffset, c, details, ticker, secondaryCoin, erc20Balance)
 				if err != nil {
 					return nil, nil, err
 				}
@@ -1191,6 +1235,7 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto
 	d.nonce = strconv.Itoa(int(n))
 	// special handling if filtering for a contract, return the contract details even though the address had no transactions with it
 	if len(d.tokens) == 0 && len(filterDesc) > 0 && details >= AccountDetailsTokens {
+		// Query the backend directly to return contract metadata/balance for filtered views.
 		t, err := w.getEthereumContractBalanceFromBlockchain(addrDesc, filterDesc, details)
 		if err != nil {
 			return nil, nil, err
@@ -1203,6 +1248,7 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto
 	// if staking pool enabled, fetch the staking pool details
 	if details >= AccountDetailsBasic {
 		if len(w.chain.EthereumTypeGetSupportedStakingPools()) > 0 {
+			// Staking pools are fetched separately and do not participate in ERC20 batching.
 			d.stakingPools, err = w.getStakingPoolsData(addrDesc)
 			if err != nil {
 				return nil, nil, err

bchain/basechain.go

@@ -74,6 +74,11 @@ func (b *BaseChain) EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc A
 	return nil, errors.New("not supported")
 }
 
+// EthereumTypeGetErc20ContractBalances is not supported
+func (b *BaseChain) EthereumTypeGetErc20ContractBalances(addrDesc AddressDescriptor, contractDescs []AddressDescriptor) ([]*big.Int, error) {
+	return nil, errors.New("not supported")
+}
+
 // GetTokenURI returns URI of non fungible or multi token defined by token id
 func (p *BaseChain) GetTokenURI(contractDesc AddressDescriptor, tokenID *big.Int) (string, error) {
 	return "", errors.New("not supported")

bchain/coins/arbitrum/arbitrumrpc.go

@@ -38,7 +38,7 @@ func NewArbitrumRPC(config json.RawMessage, pushHandler func(bchain.Notification
 func (b *ArbitrumRPC) Initialize() error {
 	b.OpenRPC = eth.OpenRPC
 
-	rc, ec, err := b.OpenRPC(b.ChainConfig.RPCURL)
+	rc, ec, err := b.OpenRPC(b.ChainConfig.RPCURL, b.ChainConfig.RPCURLWS)
 	if err != nil {
 		return err
 	}

bchain/coins/avalanche/avalancherpc.go

@@ -21,6 +21,44 @@ const (
 	MainNet eth.Network = 43114
 )
 
+func dialRPC(rawURL string) (*rpc.Client, error) {
+	if rawURL == "" {
+		return nil, errors.New("empty rpc url")
+	}
+	opts := []rpc.ClientOption{}
+	if parsed, err := url.Parse(rawURL); err == nil {
+		if parsed.Scheme == "ws" || parsed.Scheme == "wss" {
+			opts = append(opts, rpc.WithWebsocketMessageSizeLimit(0))
+		}
+	}
+	return rpc.DialOptions(context.Background(), rawURL, opts...)
+}
+
+// OpenRPC opens RPC connections for Avalanche to separate HTTP and WS endpoints.
+var OpenRPC = func(httpURL, wsURL string) (bchain.EVMRPCClient, bchain.EVMClient, error) {
+	callURL, subURL, err := eth.NormalizeRPCURLs(httpURL, wsURL)
+	if err != nil {
+		return nil, nil, err
+	}
+	callClient, err := dialRPC(callURL)
+	if err != nil {
+		return nil, nil, err
+	}
+	callRPC := &AvalancheRPCClient{Client: callClient}
+	subRPC := callRPC
+	if subURL != callURL {
+		subClient, err := dialRPC(subURL)
+		if err != nil {
+			callClient.Close()
+			return nil, nil, err
+		}
+		subRPC = &AvalancheRPCClient{Client: subClient}
+	}
+	rc := &AvalancheDualRPCClient{CallClient: callRPC, SubClient: subRPC}
+	c := &AvalancheClient{Client: ethclient.NewClient(callClient), AvalancheRPCClient: callRPC}
+	return rc, c, nil
+}
+
 // AvalancheRPC is an interface to JSON-RPC avalanche service.
 type AvalancheRPC struct {
 	*eth.EthereumRPC
@@ -43,17 +81,9 @@ func NewAvalancheRPC(config json.RawMessage, pushHandler func(bchain.Notificatio
 
 // Initialize avalanche rpc interface
 func (b *AvalancheRPC) Initialize() error {
-	b.OpenRPC = func(url string) (bchain.EVMRPCClient, bchain.EVMClient, error) {
-		r, err := rpc.Dial(url)
-		if err != nil {
-			return nil, nil, err
-		}
-		rc := &AvalancheRPCClient{Client: r}
-		c := &AvalancheClient{Client: ethclient.NewClient(r), AvalancheRPCClient: rc}
-		return rc, c, nil
-	}
+	b.OpenRPC = OpenRPC
 
-	rpcClient, client, err := b.OpenRPC(b.ChainConfig.RPCURL)
+	rpcClient, client, err := b.OpenRPC(b.ChainConfig.RPCURL, b.ChainConfig.RPCURLWS)
 	if err != nil {
 		return err
 	}