Merge pull request #141 from OneBusAway/feat/gtfs-tidy

aaronbrethorst · web-flow · commit 3c643b9f7909 · 2025-12-02T19:27:33.000-08:00
Feat/gtfs-tidy
diff --git a/Makefile b/Makefile
@@ -3,9 +3,12 @@
 run: build
 	bin/maglev -f config.json
 
-build:
+build: gtfstidy
 	go build -gcflags "all=-N -l" -o bin/maglev ./cmd/api
 
+gtfstidy:
+	go build -o bin/gtfstidy github.com/patrickbr/gtfstidy
+
 clean:
 	go clean
 	rm -f maglev
diff --git a/README.markdown b/README.markdown
@@ -50,7 +50,8 @@ Example `config.json`:
   "gtfs-static-feed": {
     "url": "https://example.com/gtfs.zip",
     "auth-header-name": "Authorization",
-    "auth-header-value": "Bearer token456"
+    "auth-header-value": "Bearer token456",
+    "enable-gtfs-tidy": true
   },
   "gtfs-rt-feeds": [
     {
diff --git a/cmd/api/main.go b/cmd/api/main.go
@@ -3,9 +3,10 @@ package main
 import (
 	"flag"
 	"log/slog"
+	"os"
+
 	"maglev.onebusaway.org/internal/appconf"
 	"maglev.onebusaway.org/internal/gtfs"
-	"os"
 )
 
 func main() {
@@ -72,6 +73,7 @@ func main() {
 			GTFSDataPath:            gtfsCfgData.GTFSDataPath,
 			Env:                     gtfsCfgData.Env,
 			Verbose:                 gtfsCfgData.Verbose,
+			EnableGTFSTidy:          gtfsCfgData.EnableGTFSTidy,
 		}
 	} else {
 		// Use command-line flags for configuration
diff --git a/config.example.json b/config.example.json
@@ -5,7 +5,8 @@
   "api-keys": ["test"],
   "rate-limit": 100,
   "gtfs-static-feed": {
-    "url": "https://www.soundtransit.org/GTFS-rail/40_gtfs.zip"
+    "url": "https://www.soundtransit.org/GTFS-rail/40_gtfs.zip",
+    "enable-gtfs-tidy": false
   },
   "gtfs-rt-feeds": [
     {
diff --git a/config.schema.json b/config.schema.json
@@ -50,6 +50,11 @@
         "auth-header-value": {
           "type": "string",
           "description": "Optional header value for static GTFS feed authentication"
+        },
+        "enable-gtfs-tidy": {
+          "type": "boolean",
+          "description": "Enable GTFS tidying with gtfstidy tool (requires gtfstidy to be installed)",
+          "default": false
         }
       },
       "required": ["url"],
diff --git a/go.mod b/go.mod
@@ -31,6 +31,10 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1 // indirect
+	github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868 // indirect
+	github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8 // indirect
+	github.com/paulmach/go.geojson v1.5.0 // indirect
 	github.com/pganalyze/pg_query_go/v6 v6.1.0 // indirect
 	github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect
 	github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect
@@ -45,6 +49,7 @@ require (
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/tetratelabs/wazero v1.9.0 // indirect
 	github.com/tidwall/geoindex v1.7.0 // indirect
+	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 // indirect
 	github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
diff --git a/go.sum b/go.sum
@@ -64,6 +64,14 @@ github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBW
 github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1 h1:ei2LAhpj7frAPBzbjqTA9ICXi7H2KhBXlYzO0WwP8hI=
+github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1/go.mod h1:WsjXLsxSQc+KfBJ7APQhk3yz+4DztzMYARQ+EfxKYSQ=
+github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868 h1:5RbZdJNYJbywbCwaQOzHUckzJ+F0w54imKZhkL0eUB0=
+github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868/go.mod h1:qNytqg0F2xv/NCYHECvfjr/QPioK94q2rhPTC7NI4KA=
+github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8 h1:xEQhft28izewbL3fnF10Yj8zSNu2r+lW157GQq+eaaY=
+github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8/go.mod h1:zYAfZRXtDhc4Tq+LYrvyksefggNiViL2pKMRUrGvUbE=
+github.com/paulmach/go.geojson v1.5.0 h1:7mhpMK89SQdHFcEGomT7/LuJhwhEgfmpWYVlVmLEdQw=
+github.com/paulmach/go.geojson v1.5.0/go.mod h1:DgdUy2rRVDDVgKqrjMe2vZAHMfhDTrjVKt3LmHIXGbU=
 github.com/pganalyze/pg_query_go/v6 v6.1.0 h1:jG5ZLhcVgL1FAw4C/0VNQaVmX1SUJx71wBGdtTtBvls=
 github.com/pganalyze/pg_query_go/v6 v6.1.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50=
 github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
@@ -113,6 +121,8 @@ github.com/tidwall/rtree v1.10.0 h1:+EcI8fboEaW1L3/9oW/6AMoQ8HiEIHyR7bQOGnmz4Mg=
 github.com/tidwall/rtree v1.10.0/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ=
 github.com/twpayne/go-polyline v1.1.1 h1:/tSF1BR7rN4HWj4XKqvRUNrCiYVMCvywxTFVofvDV0w=
 github.com/twpayne/go-polyline v1.1.1/go.mod h1:ybd9IWWivW/rlXPXuuckeKUyF3yrIim+iqA7kSl4NFY=
+github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
+github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 h1:mJdDDPblDfPe7z7go8Dvv1AJQDI3eQ/5xith3q2mFlo=
 github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07/go.mod h1:Ak17IJ037caFp4jpCw/iQQ7/W74Sqpb1YuKJU6HTKfM=
 github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
diff --git a/internal/appconf/json_config.go b/internal/appconf/json_config.go
@@ -14,6 +14,7 @@ type GtfsStaticFeed struct {
 	URL             string `json:"url"`
 	AuthHeaderName  string `json:"auth-header-name"`
 	AuthHeaderValue string `json:"auth-header-value"`
+	EnableGTFSTidy  bool   `json:"enable-gtfs-tidy"`
 }
 
 // GtfsRtFeed represents a single GTFS-RT feed configuration
@@ -187,6 +188,7 @@ type GtfsConfigData struct {
 	GTFSDataPath            string
 	Env                     Environment
 	Verbose                 bool
+	EnableGTFSTidy          bool
 }
 
 // ToGtfsConfigData converts JSONConfig to GtfsConfigData
@@ -199,6 +201,7 @@ func (j *JSONConfig) ToGtfsConfigData() GtfsConfigData {
 		GTFSDataPath:          j.DataPath,
 		Env:                   EnvFlagToEnvironment(j.Env),
 		Verbose:               true, // Always set to true like in main.go
+		EnableGTFSTidy:        j.GtfsStaticFeed.EnableGTFSTidy,
 	}
 
 	// Use first GTFS-RT feed if available
diff --git a/internal/gtfs/config.go b/internal/gtfs/config.go
@@ -16,6 +16,7 @@ type Config struct {
 	GTFSDataPath            string
 	Env                     appconf.Environment
 	Verbose                 bool
+	EnableGTFSTidy          bool
 }
 
 func (config Config) realTimeDataEnabled() bool {
diff --git a/internal/gtfs/gtfs_manager.go b/internal/gtfs/gtfs_manager.go
@@ -46,7 +46,7 @@ type Manager struct {
 func InitGTFSManager(config Config) (*Manager, error) {
 	isLocalFile := !strings.HasPrefix(config.GtfsURL, "http://") && !strings.HasPrefix(config.GtfsURL, "https://")
 
-	staticData, err := loadGTFSData(config.GtfsURL, isLocalFile, config.StaticAuthHeaderKey, config.StaticAuthHeaderValue)
+	staticData, err := loadGTFSData(config.GtfsURL, isLocalFile, config)
 	if err != nil {
 		return nil, err
 	}
diff --git a/internal/gtfs/static.go b/internal/gtfs/static.go
@@ -14,10 +14,12 @@ import (
 	"maglev.onebusaway.org/internal/logging"
 )
 
-func rawGtfsData(source string, isLocalFile bool, authHeaderKey, authHeaderValue string) ([]byte, error) {
+func rawGtfsData(source string, isLocalFile bool, config Config) ([]byte, error) {
 	var b []byte
 	var err error
 
+	logger := slog.Default().With(slog.String("component", "gtfs_loader"))
+
 	if isLocalFile {
 		b, err = os.ReadFile(source)
 		if err != nil {
@@ -30,8 +32,8 @@ func rawGtfsData(source string, isLocalFile bool, authHeaderKey, authHeaderValue
 		}
 
 		// Add auth header if provided
-		if authHeaderKey != "" && authHeaderValue != "" {
-			req.Header.Set(authHeaderKey, authHeaderValue)
+		if config.StaticAuthHeaderKey != "" && config.StaticAuthHeaderValue != "" {
+			req.Header.Set(config.StaticAuthHeaderKey, config.StaticAuthHeaderValue)
 		}
 
 		client := &http.Client{}
@@ -48,6 +50,18 @@ func rawGtfsData(source string, isLocalFile bool, authHeaderKey, authHeaderValue
 			return nil, fmt.Errorf("error reading GTFS data: %w", err)
 		}
 	}
+
+	// Process through gtfstidy if enabled
+	if config.EnableGTFSTidy {
+		logging.LogOperation(logger, "gtfstidy_enabled_processing_gtfs_data")
+		tidiedData, err := tidyGTFSData(b, logger)
+		if err != nil {
+			logging.LogError(logger, "Failed to tidy GTFS data, using original data", err)
+		} else {
+			b = tidiedData
+		}
+	}
+
 	return b, nil
 }
 
@@ -82,8 +96,8 @@ func buildGtfsDB(config Config, isLocalFile bool) (*gtfsdb.Client, error) {
 }
 
 // loadGTFSData loads and parses GTFS data from either a URL or a local file
-func loadGTFSData(source string, isLocalFile bool, authHeaderKey, authHeaderValue string) (*gtfs.Static, error) {
-	b, err := rawGtfsData(source, isLocalFile, authHeaderKey, authHeaderValue)
+func loadGTFSData(source string, isLocalFile bool, config Config) (*gtfs.Static, error) {
+	b, err := rawGtfsData(source, isLocalFile, config)
 	if err != nil {
 		return nil, fmt.Errorf("error reading GTFS data: %w", err)
 	}
@@ -122,7 +136,7 @@ func (manager *Manager) updateStaticGTFS() { // nolint
 			_, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 
 			// Download and parse the GTFS feed
-			staticData, err := loadGTFSData(manager.gtfsSource, false, manager.config.StaticAuthHeaderKey, manager.config.StaticAuthHeaderValue)
+			staticData, err := loadGTFSData(manager.gtfsSource, false, manager.config)
 			cancel() // Always cancel the context when done
 
 			if err != nil {
diff --git a/internal/gtfs/tidy.go b/internal/gtfs/tidy.go
@@ -0,0 +1,112 @@
+package gtfs
+
+import (
+	"bytes"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"maglev.onebusaway.org/internal/logging"
+)
+
+// checkGTFSTidyAvailable checks if gtfstidy is available and returns its path
+func checkGTFSTidyAvailable() (string, error) {
+	exePath, err := os.Executable()
+	if err == nil {
+		exeDir := filepath.Dir(exePath)
+
+		localPath := filepath.Join(exeDir, "gtfstidy")
+		if _, err := os.Stat(localPath); err == nil {
+			return localPath, nil
+		}
+		if _, err := os.Stat(localPath + ".exe"); err == nil {
+			return localPath + ".exe", nil
+		}
+
+		binPath := filepath.Join(exeDir, "bin", "gtfstidy")
+		if _, err := os.Stat(binPath); err == nil {
+			return binPath, nil
+		}
+		if _, err := os.Stat(binPath + ".exe"); err == nil {
+			return binPath + ".exe", nil
+		}
+
+		devBinPath := filepath.Join(exeDir, "..", "bin", "gtfstidy")
+		if _, err := os.Stat(devBinPath); err == nil {
+			return devBinPath, nil
+		}
+		if _, err := os.Stat(devBinPath + ".exe"); err == nil {
+			return devBinPath + ".exe", nil
+		}
+	}
+
+	path, err := exec.LookPath("gtfstidy")
+	if err != nil {
+		return "", fmt.Errorf("gtfstidy not found in PATH or local directories: %w", err)
+	}
+	return path, nil
+}
+
+// tidyGTFSData processes GTFS data through gtfstidy with flags -OscRCSmeD
+// Returns the tidied GTFS zip data or an error
+func tidyGTFSData(inputZip []byte, logger *slog.Logger) ([]byte, error) {
+	gtfstidyPath, err := checkGTFSTidyAvailable()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create temp directory for processing
+	// Note: We should clean up the temp dir on error || after processing
+	tempDir, err := os.MkdirTemp("", "gtfs-tidy-*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp directory: %w", err)
+	}
+	defer func() {
+		if err := os.RemoveAll(tempDir); err != nil {
+			logging.LogError(logger, "Failed to clean up temp directory", err,
+				slog.String("temp_dir", tempDir))
+		}
+	}()
+
+	pristineZipPath := filepath.Join(tempDir, "gtfs_pristine.zip")
+	if err := os.WriteFile(pristineZipPath, inputZip, 0600); err != nil {
+		return nil, fmt.Errorf("failed to write pristine GTFS zip: %w", err)
+	}
+
+	outputZipPath := filepath.Join(tempDir, "gtfs_tidied.zip")
+
+	logging.LogOperation(logger, "running_gtfstidy",
+		slog.String("input_file", pristineZipPath),
+		slog.Int("input_size_bytes", len(inputZip)))
+
+	// -o: output file
+	cmd := exec.Command(gtfstidyPath, "-OscRCSmeD", "-o", outputZipPath, pristineZipPath)
+	cmd.Dir = tempDir
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	if err := cmd.Run(); err != nil {
+		logging.LogError(logger, "gtfstidy command failed", err,
+			slog.String("stdout", stdout.String()),
+			slog.String("stderr", stderr.String()))
+		return nil, fmt.Errorf("gtfstidy failed: %w", err)
+	}
+
+	logging.LogOperation(logger, "gtfstidy_completed",
+		slog.String("stdout", stdout.String()))
+
+	tidiedZip, err := os.ReadFile(outputZipPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read tidied GTFS zip: %w", err)
+	}
+
+	logging.LogOperation(logger, "gtfs_tidied_successfully",
+		slog.Int("output_size_bytes", len(tidiedZip)),
+		slog.Int("size_reduction_bytes", len(inputZip)-len(tidiedZip)))
+
+	return tidiedZip, nil
+}
diff --git a/internal/gtfs/tidy_test.go b/internal/gtfs/tidy_test.go

-Original file line number
+Diff line change
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 +	github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1 // indirect
 +	github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868 // indirect
 +	github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8 // indirect
 +	github.com/paulmach/go.geojson v1.5.0 // indirect
 	github.com/pganalyze/pg_query_go/v6 v6.1.0 // indirect
 	github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect
 	github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/tetratelabs/wazero v1.9.0 // indirect
 	github.com/tidwall/geoindex v1.7.0 // indirect
 +	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 // indirect
 	github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@ type GtfsStaticFeed struct {`
`14`	`14`	URL string `json:"url"`
`15`	`15`	AuthHeaderName string `json:"auth-header-name"`
`16`	`16`	AuthHeaderValue string `json:"auth-header-value"`
	`17`	+ EnableGTFSTidy bool `json:"enable-gtfs-tidy"`
`17`	`18`	`}`
`18`	`19`
`19`	`20`	`// GtfsRtFeed represents a single GTFS-RT feed configuration`
`@@ -187,6 +188,7 @@ type GtfsConfigData struct {`
`187`	`188`	`GTFSDataPath string`
`188`	`189`	`Env Environment`
`189`	`190`	`Verbose bool`
	`191`	`+ EnableGTFSTidy bool`
`190`	`192`	`}`
`191`	`193`
`192`	`194`	`// ToGtfsConfigData converts JSONConfig to GtfsConfigData`
`@@ -199,6 +201,7 @@ func (j *JSONConfig) ToGtfsConfigData() GtfsConfigData {`
`199`	`201`	`GTFSDataPath: j.DataPath,`
`200`	`202`	`Env: EnvFlagToEnvironment(j.Env),`
`201`	`203`	`Verbose: true, // Always set to true like in main.go`
	`204`	`+ EnableGTFSTidy: j.GtfsStaticFeed.EnableGTFSTidy,`
`202`	`205`	`}`
`203`	`206`
`204`	`207`	`// Use first GTFS-RT feed if available`
Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,7 @@ type Config struct {`
`16`	`16`	`GTFSDataPath string`
`17`	`17`	`Env appconf.Environment`
`18`	`18`	`Verbose bool`
	`19`	`+ EnableGTFSTidy bool`
`19`	`20`	`}`
`20`	`21`
`21`	`22`	`func (config Config) realTimeDataEnabled() bool {`
Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@ type Manager struct {`
`46`	`46`	`func InitGTFSManager(config Config) (*Manager, error) {`
`47`	`47`	`isLocalFile := !strings.HasPrefix(config.GtfsURL, "http://") && !strings.HasPrefix(config.GtfsURL, "https://")`
`48`	`48`
`49`		`- staticData, err := loadGTFSData(config.GtfsURL, isLocalFile, config.StaticAuthHeaderKey, config.StaticAuthHeaderValue)`
	`49`	`+ staticData, err := loadGTFSData(config.GtfsURL, isLocalFile, config)`
`50`	`50`	`if err != nil {`
`51`	`51`	`return nil, err`
`52`	`52`	`}`