diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 56b2a38..158574d 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -25,6 +25,7 @@ jobs: service: - api-service - replicator-service + - database-service env: IS_BETA_VERSION: ${{ inputs.is_beta_version }} SERVICE: ${{ matrix.service }} diff --git a/Dockerfile b/Dockerfile index 1d420ed..4856bea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,6 @@ RUN go run github.com/swaggo/swag/cmd/swag@v1.8.1 init -g cmd/api-service/main.g FROM gcr.io/distroless/static-debian13:nonroot COPY --from=build /out/service /service +COPY --from=build /src/migrations /migrations +ENV MIGRATIONS_URL=file:///migrations ENTRYPOINT ["/service"] diff --git a/Makefile b/Makefile index f43adc7..a5f0809 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,10 @@ migrate-down: migrate-force: migrate -path migrations -database $(DB_URL) force $(version) +.PHONY: database-service +database-service: + go run ./cmd/database-service $(ARGS) + .PHONY: backfill-package-slugs backfill-package-slugs: migrate-up podman exec -i "$(POSTGRES_CONTAINER)" psql -v batch_size="$(BATCH_SIZE)" -v pause_seconds="$(PAUSE_SECONDS)" -U "$${POSTGRES_USER:-user}" -d "$${POSTGRES_DB:-postgres}" -f - < scripts/backfill_packages_slug.sql @@ -105,6 +109,9 @@ help: @echo "make migrate-up - Run database migrations up" @echo "make migrate-down - Run database migrations down" @echo "make migrate-force - Force database migration to a specific version (usage: make migrate-force version=)" + @echo "make database-service - Apply pending migrations and run first init when needed" + @echo "make database-service ARGS='migrate up|down|version|force ' - Run a migration command" + @echo "make database-service ARGS='first-init' - Apply migrations and initialize npm OSV data" @echo "make backfill-package-slugs - Backfill and uniquely index package slugs" @echo " (options: BATCH_SIZE=2000 PAUSE_SECONDS=0.20)" @echo "make reindex-package-slugs - Populate slugs in the Typesense packages collection with parameters ARGS='-batch-size 128 -pause 250ms'" diff --git a/README.md b/README.md index 8668a35..037dae1 100644 --- a/README.md +++ b/README.md @@ -146,12 +146,53 @@ leaving a partial history record. Generating a report reads PostgreSQL only and OSV records with a `withdrawn` timestamp are retained and marked with `vulnerabilities.withdrawn_at`. If an ID from `modified_id.csv` returns HTTP 404, an existing local record is marked as withdrawn as well. -Apply migrations before starting the service: +To apply migrations manually: ```bash -make migrate-up +make database-service ARGS='migrate up' ``` +### Database service + +The one-shot `database-service` container manages migrations and initializes the npm OSV +dataset without exposing HTTP endpoints. Run a local operation with: + +```bash +podman exec database_service help +make database-service +make database-service ARGS='migrate version' +make database-service ARGS='migrate up' +make database-service ARGS='migrate down' +make database-service ARGS='migrate force 13' +make database-service ARGS='first-init' +``` + +With no arguments, the service runs its startup handler: it checks for pending migrations, applies +them, then checks whether the initial npm OSV import is needed and runs it when necessary. Explicit +CLI commands perform only the requested operation, so `migrate down` and `migrate force` are not +preceded by automatic migrations. `migrate version` reports the current migration and whether the +database is dirty. `migrate down` reverts all applied migrations, while `migrate force ` +sets the migration version without running SQL. `first-init` applies pending migrations and then +runs the existing npm OSV replication flow; a database without OSV replication state receives the +full initial import. The first-init action does not require Redis. +The `help` command prints usage without connecting to PostgreSQL. +Its descriptions explain the effects and risks of each operation; in particular, `migrate down` +runs every down migration and may delete schema and data, while `migrate force` only changes the +recorded version and does not run SQL. + +Build and run it as a container with: + +```bash +docker build --build-arg SERVICE=database-service -t trustpkg-database-service . +docker run --rm --env-file .env trustpkg-database-service +docker run --rm --env-file .env trustpkg-database-service migrate up +``` + +The runtime image includes the repository migrations at `/migrations`. +The image build workflow also builds and publishes the `database-service` image to +`ghcr.io/trustpkg/trustpkg-api/database-service` alongside the other service images, using the +same PR and release tag rules. + ## Package search API The package search endpoint is available at `GET /api/packages/search`. diff --git a/cmd/api-service/main.go b/cmd/api-service/main.go index d3504aa..a4fd6b1 100644 --- a/cmd/api-service/main.go +++ b/cmd/api-service/main.go @@ -15,7 +15,7 @@ import ( ) // @title TrustPkg API -// @version 1.7.2 +// @version 2.0.0 // @description API for searching npm packages and their vulnerability data. // @BasePath /api // @schemes http https diff --git a/cmd/database-service/main.go b/cmd/database-service/main.go new file mode 100644 index 0000000..fbe3995 --- /dev/null +++ b/cmd/database-service/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + + "github.com/trustpkg/trustpkg-api/db" + databaseService "github.com/trustpkg/trustpkg-api/internal/database-service" +) + +// @title TrustPkg Database Service +// @version 1.7.2 +// @description Database service for managing migrations and initializations. +func main() { + command, err := databaseService.ParseCommand(os.Args[1:]) + if err != nil { + log.Printf("database service failed: %v", err) + os.Exit(1) + } + if command.Action == databaseService.ActionHelp { + if err := databaseService.Execute(context.Background(), command, nil, nil, os.Stdout); err != nil { + log.Printf("database service failed: %v", err) + os.Exit(1) + } + return + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + db.ConnectDb() + defer db.Pool.Close() + + migrationsURL := os.Getenv("MIGRATIONS_URL") + if migrationsURL == "" { + migrationsURL = "file://migrations" + } + + migrations := databaseService.NewMigrationManager(db.Pool, migrationsURL) + initializer := databaseService.NewOSVInitializer("npm") + handler := databaseService.NewHandler(migrations, initializer, log.Default()) + if err := databaseService.Execute(ctx, command, migrations, handler, os.Stdout); err != nil { + log.Printf("database service failed: %v", err) + os.Exit(1) + } +} diff --git a/go.mod b/go.mod index 178614f..5d5493f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.5 require ( github.com/go-ozzo/ozzo-validation/v4 v4.4.1 + github.com/golang-migrate/migrate/v4 v4.19.1 github.com/jackc/pgx/v5 v5.10.0 github.com/joho/godotenv v1.5.1 github.com/robfig/cron/v3 v3.0.0 @@ -28,6 +29,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.20.2 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.15 // indirect diff --git a/go.sum b/go.sum index edf865a..069e63d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/agiledragon/gomonkey/v2 v2.3.1 h1:k+UnUY0EMNYUFUAQVETGY9uUTxjMdnUkP0ARyJS1zzs= github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= @@ -16,18 +20,39 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/getkin/kin-openapi v0.149.0 h1:ZbhmVJ4yq5RZDUsyP8lcBcGMsjsaTqXEFt6isdtMDfA= github.com/getkin/kin-openapi v0.149.0/go.mod h1:1+BHDzstro+P5CKtPy1X4PfofnFgmRe6uvMy9+r9fKY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -54,6 +79,10 @@ github.com/gofiber/schema v1.8.3 h1:06ZedxIYjngzc0095PYy7uWnFnbRflWFpikvZH61fDc= github.com/gofiber/schema v1.8.3/go.mod h1:jWnnZdhcW1mHyV+VnfRxKJDPNcepJsTZ9RIWxrr32Ng= github.com/gofiber/utils/v2 v2.4.1 h1:E2X9G8O5Mn7b2GDb0JU3IUk42Rw2npuhhepIbuJQ2po= github.com/gofiber/utils/v2 v2.4.1/go.mod h1:I+RTsgMUdzFuifVc3LOEkfh32wQW9BfRl7l5RYjamW4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= +github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -85,6 +114,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -96,6 +127,12 @@ github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= @@ -103,12 +140,19 @@ github.com/oasdiff/yaml v0.1.1 h1:6nHx+pn9gBRM6YpBlFZFQGCCd1nuvqOBtTD3KKTgGxY= github.com/oasdiff/yaml v0.1.1/go.mod h1:EYJNoyktvWMJ0Hmhx+6qTaqMOsalUaRGT8Sj1hNcegU= github.com/oasdiff/yaml3 v0.0.14 h1:aLJee3hxBK2H5wdXd9iPcIXb93Nty1Ge0pT171eHtkw= github.com/oasdiff/yaml3 v0.0.14/go.mod h1:csto2xfDjYccdUn/yw/bPjj/cYTdp6HtFA0J4TWG+gg= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE= github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= @@ -159,6 +203,16 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= diff --git a/internal/database-service/command-execution_test.go b/internal/database-service/command-execution_test.go new file mode 100644 index 0000000..c78be79 --- /dev/null +++ b/internal/database-service/command-execution_test.go @@ -0,0 +1,54 @@ +package databaseService + +import ( + "bytes" + "context" + "testing" +) + +func TestExecuteStartupRunsHandler(t *testing.T) { + migrations := &fakeMigrationManager{ + status: MigrationStatus{CurrentVersion: 1, LatestVersion: 1, HasCurrent: true, HasMigrations: true}, + } + initializer := &fakeInitializer{} + handler := NewHandler(migrations, initializer, &fakeLogger{}) + + err := Execute(context.Background(), Command{Action: actionStartup}, migrations, handler, &bytes.Buffer{}) + if err != nil { + t.Fatalf("Execute() error = %v", err) + } + if migrations.upCalls != 0 || initializer.calls != 0 { + t.Fatalf("startup executed unexpected work: migrations=%d init=%d", migrations.upCalls, initializer.calls) + } +} + +func TestExecuteMigrationVersionReportsCurrentAndLatest(t *testing.T) { + migrations := &fakeMigrationManager{ + status: MigrationStatus{CurrentVersion: 2, LatestVersion: 3, HasCurrent: true, HasMigrations: true, Dirty: true}, + } + var output bytes.Buffer + + err := Execute(context.Background(), Command{Action: actionVersion}, migrations, nil, &output) + if err != nil { + t.Fatalf("Execute() error = %v", err) + } + want := "Current migration version: 2 (latest: 3, dirty: true)\n" + if output.String() != want { + t.Fatalf("Execute() output = %q, want %q", output.String(), want) + } +} + +func TestExecuteHelpDoesNotRequireDatabase(t *testing.T) { + var output bytes.Buffer + + err := Execute(context.Background(), Command{Action: ActionHelp}, nil, nil, &output) + if err != nil { + t.Fatalf("Execute() error = %v", err) + } + if !bytes.Contains(output.Bytes(), []byte("migrate version")) || + !bytes.Contains(output.Bytes(), []byte("first-init")) || + !bytes.Contains(output.Bytes(), []byte("may remove database tables and data")) || + !bytes.Contains(output.Bytes(), []byte("without executing migration SQL")) { + t.Fatalf("Execute() help output is incomplete: %q", output.String()) + } +} diff --git a/internal/database-service/command.go b/internal/database-service/command.go new file mode 100644 index 0000000..c562322 --- /dev/null +++ b/internal/database-service/command.go @@ -0,0 +1,114 @@ +package databaseService + +import ( + "context" + "errors" + "fmt" + "io" + "strconv" +) + +const ( + ActionHelp = "help" + actionStartup = "startup" + actionUp = "up" + actionDown = "down" + actionForce = "force" + actionVersion = "version" + actionInit = "first-init" +) + +func ParseCommand(args []string) (Command, error) { + if len(args) == 0 { + return Command{Action: actionStartup}, nil + } + if len(args) == 1 && args[0] == actionInit { + return Command{Action: actionInit}, nil + } + if len(args) == 1 && args[0] == ActionHelp { + return Command{Action: ActionHelp}, nil + } + if len(args) < 2 || args[0] != "migrate" { + return Command{}, usageError() + } + + switch args[1] { + case actionUp, actionDown, actionVersion: + if len(args) != 2 { + return Command{}, usageError() + } + return Command{Action: args[1]}, nil + case actionForce: + if len(args) != 3 { + return Command{}, usageError() + } + version, err := strconv.Atoi(args[2]) + if err != nil || version < -1 { + return Command{}, fmt.Errorf("invalid migration version %q: expected an integer greater than or equal to -1", args[2]) + } + return Command{Action: actionForce, Version: version}, nil + default: + return Command{}, usageError() + } +} + +func Execute(ctx context.Context, command Command, migrations MigrationManager, handler *Handler, output io.Writer) error { + switch command.Action { + case ActionHelp: + return WriteHelp(output) + case actionStartup, actionInit: + if handler == nil { + return errors.New("database startup handler is nil") + } + return handler.Run(ctx) + case actionUp: + if migrations == nil { + return errors.New("migration manager is nil") + } + return migrations.Up() + case actionDown: + if migrations == nil { + return errors.New("migration manager is nil") + } + return migrations.Down() + case actionForce: + if migrations == nil { + return errors.New("migration manager is nil") + } + return migrations.Force(command.Version) + case actionVersion: + if migrations == nil { + return errors.New("migration manager is nil") + } + return WriteMigrationStatus(output, migrations) + default: + return usageError() + } +} + +func WriteMigrationStatus(output io.Writer, migrations MigrationManager) error { + if output == nil { + return errors.New("migration status output is nil") + } + if migrations == nil { + return errors.New("migration manager is nil") + } + status, err := migrations.Status() + if err != nil { + return fmt.Errorf("read migration status: %w", err) + } + if !status.HasMigrations { + _, err = fmt.Fprintln(output, "No migrations are available.") + return err + } + if !status.HasCurrent { + _, err = fmt.Fprintf(output, "No migrations have been applied. Latest available version: %d (dirty: false)\n", status.LatestVersion) + return err + } + _, err = fmt.Fprintf(output, "Current migration version: %d (latest: %d, dirty: %t)\n", status.CurrentVersion, status.LatestVersion, status.Dirty) + return err +} + +func usageError() error { + return errors.New("usage: database-service [help | migrate > | first-init]") +} diff --git a/internal/database-service/command_test.go b/internal/database-service/command_test.go new file mode 100644 index 0000000..68c8f96 --- /dev/null +++ b/internal/database-service/command_test.go @@ -0,0 +1,38 @@ +package databaseService + +import "testing" + +func TestParseCommand(t *testing.T) { + tests := []struct { + name string + args []string + want Command + wantErr bool + }{ + {name: "default startup", want: Command{Action: actionStartup}}, + {name: "help", args: []string{"help"}, want: Command{Action: ActionHelp}}, + {name: "first init", args: []string{"first-init"}, want: Command{Action: actionInit}}, + {name: "migrate up", args: []string{"migrate", "up"}, want: Command{Action: actionUp}}, + {name: "migrate down", args: []string{"migrate", "down"}, want: Command{Action: actionDown}}, + {name: "migration version", args: []string{"migrate", "version"}, want: Command{Action: actionVersion}}, + {name: "force version", args: []string{"migrate", "force", "13"}, want: Command{Action: actionForce, Version: 13}}, + {name: "force unset version", args: []string{"migrate", "force", "-1"}, want: Command{Action: actionForce, Version: -1}}, + {name: "unknown action", args: []string{"migrate", "status"}, wantErr: true}, + {name: "unexpected argument", args: []string{"migrate", "up", "extra"}, wantErr: true}, + {name: "missing force version", args: []string{"migrate", "force"}, wantErr: true}, + {name: "invalid force version", args: []string{"migrate", "force", "latest"}, wantErr: true}, + {name: "force version below unset", args: []string{"migrate", "force", "-2"}, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseCommand(tt.args) + if (err != nil) != tt.wantErr { + t.Fatalf("ParseCommand() error = %v, wantErr %t", err, tt.wantErr) + } + if err == nil && got != tt.want { + t.Fatalf("ParseCommand() = %#v, want %#v", got, tt.want) + } + }) + } +} diff --git a/internal/database-service/handler.go b/internal/database-service/handler.go new file mode 100644 index 0000000..feb0342 --- /dev/null +++ b/internal/database-service/handler.go @@ -0,0 +1,66 @@ +package databaseService + +import ( + "context" + "fmt" + "log" +) + +type Handler struct { + migrations MigrationManager + initializer Initializer + logger Logger +} + +func NewHandler(migrations MigrationManager, initializer Initializer, logger Logger) *Handler { + if logger == nil { + logger = log.Default() + } + return &Handler{ + migrations: migrations, + initializer: initializer, + logger: logger, + } +} + +func (handler *Handler) Run(ctx context.Context) error { + if handler == nil || handler.migrations == nil || handler.initializer == nil { + return fmt.Errorf("database startup handler is not fully configured") + } + + status, err := handler.migrations.Status() + if err != nil { + return fmt.Errorf("check migration status: %w", err) + } + if status.Dirty { + return fmt.Errorf("database migration state is dirty at version %d; repair it before startup", status.CurrentVersion) + } + if status.HasCurrent && status.HasMigrations && status.CurrentVersion > status.LatestVersion { + return fmt.Errorf("database migration version %d is newer than the latest available migration %d", status.CurrentVersion, status.LatestVersion) + } + if status.Pending() { + handler.logger.Printf("pending migrations detected: current=%d latest=%d", status.CurrentVersion, status.LatestVersion) + if err := handler.migrations.Up(); err != nil { + return fmt.Errorf("apply pending migrations: %w", err) + } + handler.logger.Printf("pending migrations applied") + } else { + handler.logger.Printf("database migrations are up to date at version %d", status.CurrentVersion) + } + + needed, err := handler.initializer.Needed(ctx) + if err != nil { + return fmt.Errorf("check whether first init is needed: %w", err) + } + if !needed { + handler.logger.Printf("first init is already complete") + return nil + } + + handler.logger.Printf("first init is needed; starting initialization") + if err := handler.initializer.Initialize(ctx); err != nil { + return fmt.Errorf("run first init: %w", err) + } + handler.logger.Printf("first init completed") + return nil +} diff --git a/internal/database-service/handler_test.go b/internal/database-service/handler_test.go new file mode 100644 index 0000000..14f6275 --- /dev/null +++ b/internal/database-service/handler_test.go @@ -0,0 +1,152 @@ +package databaseService + +import ( + "context" + "errors" + "fmt" + "testing" +) + +type fakeMigrationManager struct { + status MigrationStatus + statusErr error + upErr error + upCalls int + downCalls int + forceCalls int +} + +func (manager *fakeMigrationManager) Status() (MigrationStatus, error) { + return manager.status, manager.statusErr +} + +func (manager *fakeMigrationManager) Up() error { + manager.upCalls++ + return manager.upErr +} + +func (manager *fakeMigrationManager) Down() error { + manager.downCalls++ + return nil +} + +func (manager *fakeMigrationManager) Force(int) error { + manager.forceCalls++ + return nil +} + +type fakeInitializer struct { + needed bool + neededErr error + initializeErr error + calls int +} + +func (initializer *fakeInitializer) Needed(context.Context) (bool, error) { + return initializer.needed, initializer.neededErr +} + +func (initializer *fakeInitializer) Initialize(context.Context) error { + initializer.calls++ + return initializer.initializeErr +} + +type fakeLogger struct { + messages []string +} + +func (logger *fakeLogger) Printf(format string, args ...any) { + logger.messages = append(logger.messages, fmt.Sprintf(format, args...)) +} + +func TestHandlerRunAppliesPendingMigrationsThenInitializes(t *testing.T) { + migrations := &fakeMigrationManager{ + status: MigrationStatus{CurrentVersion: 2, LatestVersion: 4, HasCurrent: true, HasMigrations: true}, + } + initializer := &fakeInitializer{needed: true} + logger := &fakeLogger{} + handler := NewHandler(migrations, initializer, logger) + + if err := handler.Run(context.Background()); err != nil { + t.Fatalf("Run() error = %v", err) + } + if migrations.upCalls != 1 { + t.Fatalf("migration Up() calls = %d, want 1", migrations.upCalls) + } + if initializer.calls != 1 { + t.Fatalf("initializer calls = %d, want 1", initializer.calls) + } +} + +func TestHandlerRunSkipsCompletedWork(t *testing.T) { + migrations := &fakeMigrationManager{ + status: MigrationStatus{CurrentVersion: 4, LatestVersion: 4, HasCurrent: true, HasMigrations: true}, + } + initializer := &fakeInitializer{} + handler := NewHandler(migrations, initializer, &fakeLogger{}) + + if err := handler.Run(context.Background()); err != nil { + t.Fatalf("Run() error = %v", err) + } + if migrations.upCalls != 0 || initializer.calls != 0 { + t.Fatalf("Run() performed unnecessary work: migration calls=%d init calls=%d", migrations.upCalls, initializer.calls) + } +} + +func TestHandlerRunStopsOnDirtyMigration(t *testing.T) { + migrations := &fakeMigrationManager{status: MigrationStatus{CurrentVersion: 3, Dirty: true}} + initializer := &fakeInitializer{needed: true} + handler := NewHandler(migrations, initializer, &fakeLogger{}) + + if err := handler.Run(context.Background()); err == nil { + t.Fatal("Run() error = nil, want dirty migration error") + } + if migrations.upCalls != 0 || initializer.calls != 0 { + t.Fatalf("Run() continued after dirty migration: migration calls=%d init calls=%d", migrations.upCalls, initializer.calls) + } +} + +func TestHandlerRunStopsAfterMigrationFailure(t *testing.T) { + migrationErr := errors.New("migration failed") + migrations := &fakeMigrationManager{ + status: MigrationStatus{LatestVersion: 1, HasMigrations: true}, + upErr: migrationErr, + } + initializer := &fakeInitializer{needed: true} + handler := NewHandler(migrations, initializer, &fakeLogger{}) + + if err := handler.Run(context.Background()); !errors.Is(err, migrationErr) { + t.Fatalf("Run() error = %v, want %v", err, migrationErr) + } + if initializer.calls != 0 { + t.Fatalf("initializer calls = %d, want 0", initializer.calls) + } +} + +func TestHandlerRunReturnsInitializationFailure(t *testing.T) { + initializationErr := errors.New("initialization failed") + migrations := &fakeMigrationManager{ + status: MigrationStatus{CurrentVersion: 1, LatestVersion: 1, HasCurrent: true, HasMigrations: true}, + } + initializer := &fakeInitializer{needed: true, initializeErr: initializationErr} + handler := NewHandler(migrations, initializer, &fakeLogger{}) + + if err := handler.Run(context.Background()); !errors.Is(err, initializationErr) { + t.Fatalf("Run() error = %v, want %v", err, initializationErr) + } +} + +func TestHandlerRunRejectsDatabaseAheadOfMigrations(t *testing.T) { + migrations := &fakeMigrationManager{ + status: MigrationStatus{CurrentVersion: 3, LatestVersion: 2, HasCurrent: true, HasMigrations: true}, + } + initializer := &fakeInitializer{needed: true} + handler := NewHandler(migrations, initializer, &fakeLogger{}) + + if err := handler.Run(context.Background()); err == nil { + t.Fatal("Run() error = nil, want migration source mismatch") + } + if initializer.calls != 0 { + t.Fatalf("initializer calls = %d, want 0", initializer.calls) + } +} diff --git a/internal/database-service/help.go b/internal/database-service/help.go new file mode 100644 index 0000000..00fbc5d --- /dev/null +++ b/internal/database-service/help.go @@ -0,0 +1,53 @@ +package databaseService + +import ( + "errors" + "fmt" + "io" +) + +func WriteHelp(output io.Writer) error { + if output == nil { + return errors.New("help output is nil") + } + + _, err := fmt.Fprintln(output, `Usage: database-service [command] + +With no command, the service checks the database migration state and applies all +pending migrations. It then checks whether the npm OSV dataset has been initialized +and imports the full dataset if it has not. This is the recommended startup mode. + +Commands: + help + Show this help without connecting to the database. + + migrate version + Show the database's current migration version, the latest migration available + in this service, and whether the current migration state is dirty. + + migrate up + Apply all migrations newer than the database's current migration version. + Does not run the OSV first initialization. + + migrate down + Revert all applied migrations. This runs every available down migration and + may remove database tables and data. Does not run the OSV first initialization. + + migrate force + Set the recorded migration version without executing migration SQL. Use this + only to repair a migration state after verifying the actual database schema. + Use -1 to mark the database as having no applied migrations. + + first-init + Apply pending migrations, then check the npm OSV replication state. If the + initial OSV import is not complete, download and import the full npm dataset. + If initialization is already complete, no import is run. This command does + not run Redis-dependent replication jobs. + +Examples: + podman exec database_service help + podman exec database_service migrate version + podman exec database_service migrate up + podman exec database_service first-init`) + return err +} diff --git a/internal/database-service/interfaces.go b/internal/database-service/interfaces.go new file mode 100644 index 0000000..46d2db5 --- /dev/null +++ b/internal/database-service/interfaces.go @@ -0,0 +1,19 @@ +package databaseService + +import "context" + +type MigrationManager interface { + Status() (MigrationStatus, error) + Up() error + Down() error + Force(version int) error +} + +type Initializer interface { + Needed(ctx context.Context) (bool, error) + Initialize(ctx context.Context) error +} + +type Logger interface { + Printf(format string, args ...any) +} diff --git a/internal/database-service/migration-manager.go b/internal/database-service/migration-manager.go new file mode 100644 index 0000000..c62ead6 --- /dev/null +++ b/internal/database-service/migration-manager.go @@ -0,0 +1,131 @@ +package databaseService + +import ( + "errors" + "fmt" + "os" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/postgres" + "github.com/golang-migrate/migrate/v4/source" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" +) + +type migrationManager struct { + pool *pgxpool.Pool + migrationsURL string +} + +func NewMigrationManager(pool *pgxpool.Pool, migrationsURL string) MigrationManager { + return &migrationManager{pool: pool, migrationsURL: migrationsURL} +} + +func (manager *migrationManager) Status() (MigrationStatus, error) { + migrationSource, err := source.Open(manager.migrationsURL) + if err != nil { + return MigrationStatus{}, fmt.Errorf("open migration source: %w", err) + } + latestVersion, hasMigrations, sourceErr := latestMigrationVersion(migrationSource) + closeErr := migrationSource.Close() + if sourceErr != nil || closeErr != nil { + return MigrationStatus{}, errors.Join(sourceErr, closeErr) + } + + status := MigrationStatus{LatestVersion: latestVersion, HasMigrations: hasMigrations} + err = manager.withMigrator(func(migrator *migrate.Migrate) error { + version, dirty, err := migrator.Version() + if errors.Is(err, migrate.ErrNilVersion) { + return nil + } + if err != nil { + return fmt.Errorf("read current migration version: %w", err) + } + status.CurrentVersion = version + status.HasCurrent = true + status.Dirty = dirty + return nil + }) + if err != nil { + return MigrationStatus{}, err + } + return status, nil +} + +func (manager *migrationManager) Up() error { + return manager.withMigrator(func(migrator *migrate.Migrate) error { + err := migrator.Up() + if errors.Is(err, migrate.ErrNoChange) { + return nil + } + return err + }) +} + +func (manager *migrationManager) Down() error { + return manager.withMigrator(func(migrator *migrate.Migrate) error { + err := migrator.Down() + if errors.Is(err, migrate.ErrNoChange) { + return nil + } + return err + }) +} + +func (manager *migrationManager) Force(version int) error { + return manager.withMigrator(func(migrator *migrate.Migrate) error { + return migrator.Force(version) + }) +} + +func (manager *migrationManager) withMigrator(operation func(*migrate.Migrate) error) (resultErr error) { + if manager.pool == nil { + return fmt.Errorf("PostgreSQL connection pool is nil") + } + + sqlDB := stdlib.OpenDBFromPool(manager.pool) + defer func() { + if err := sqlDB.Close(); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("close PostgreSQL connection: %w", err)) + } + }() + + databaseDriver, err := postgres.WithInstance(sqlDB, &postgres.Config{}) + if err != nil { + return fmt.Errorf("initialize PostgreSQL migration driver: %w", err) + } + migrator, err := migrate.NewWithDatabaseInstance(manager.migrationsURL, "postgres", databaseDriver) + if err != nil { + return fmt.Errorf("initialize migration engine: %w", errors.Join(err, databaseDriver.Close())) + } + defer func() { + sourceErr, databaseErr := migrator.Close() + if sourceErr != nil || databaseErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("close migration resources: %w", errors.Join(sourceErr, databaseErr))) + } + }() + + return operation(migrator) +} + +func latestMigrationVersion(migrationSource source.Driver) (uint, bool, error) { + version, err := migrationSource.First() + if errors.Is(err, os.ErrNotExist) { + return 0, false, nil + } + if err != nil { + return 0, false, fmt.Errorf("find first migration: %w", err) + } + + for { + next, err := migrationSource.Next(version) + if errors.Is(err, os.ErrNotExist) { + return version, true, nil + } + if err != nil { + return 0, false, fmt.Errorf("find next migration after %d: %w", version, err) + } + version = next + } +} diff --git a/internal/database-service/models.go b/internal/database-service/models.go new file mode 100644 index 0000000..8705fd1 --- /dev/null +++ b/internal/database-service/models.go @@ -0,0 +1,18 @@ +package databaseService + +type MigrationStatus struct { + CurrentVersion uint + LatestVersion uint + HasCurrent bool + HasMigrations bool + Dirty bool +} + +func (status MigrationStatus) Pending() bool { + return status.HasMigrations && (!status.HasCurrent || status.CurrentVersion < status.LatestVersion) +} + +type Command struct { + Action string + Version int +} diff --git a/internal/database-service/models_test.go b/internal/database-service/models_test.go new file mode 100644 index 0000000..7f8bf25 --- /dev/null +++ b/internal/database-service/models_test.go @@ -0,0 +1,25 @@ +package databaseService + +import "testing" + +func TestMigrationStatusPending(t *testing.T) { + tests := []struct { + name string + status MigrationStatus + want bool + }{ + {name: "no migrations", status: MigrationStatus{}}, + {name: "no migration applied", status: MigrationStatus{LatestVersion: 1, HasMigrations: true}, want: true}, + {name: "older migration applied", status: MigrationStatus{CurrentVersion: 1, LatestVersion: 2, HasCurrent: true, HasMigrations: true}, want: true}, + {name: "at latest migration", status: MigrationStatus{CurrentVersion: 2, LatestVersion: 2, HasCurrent: true, HasMigrations: true}}, + {name: "current exceeds latest", status: MigrationStatus{CurrentVersion: 3, LatestVersion: 2, HasCurrent: true, HasMigrations: true}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.status.Pending(); got != tt.want { + t.Fatalf("Pending() = %t, want %t", got, tt.want) + } + }) + } +} diff --git a/internal/database-service/osv-initializer.go b/internal/database-service/osv-initializer.go new file mode 100644 index 0000000..5589361 --- /dev/null +++ b/internal/database-service/osv-initializer.go @@ -0,0 +1,27 @@ +package databaseService + +import ( + "context" + + "github.com/trustpkg/trustpkg-api/internal/osv" +) + +type osvInitializer struct { + ecosystem string +} + +func NewOSVInitializer(ecosystem string) Initializer { + return &osvInitializer{ecosystem: ecosystem} +} + +func (initializer *osvInitializer) Needed(ctx context.Context) (bool, error) { + initialized, err := osv.IsEcosystemInitialized(ctx, initializer.ecosystem) + if err != nil { + return false, err + } + return !initialized, nil +} + +func (initializer *osvInitializer) Initialize(ctx context.Context) error { + return osv.InitializeEcosystem(ctx, osv.FetchOSVEcosystemDumpPayload{Ecosystem: initializer.ecosystem}) +} diff --git a/internal/osv/initialization.go b/internal/osv/initialization.go new file mode 100644 index 0000000..57e2e04 --- /dev/null +++ b/internal/osv/initialization.go @@ -0,0 +1,23 @@ +package osv + +import ( + "context" + "fmt" + "strings" +) + +func IsEcosystemInitialized(ctx context.Context, ecosystem string) (bool, error) { + if strings.TrimSpace(ecosystem) == "" { + return false, fmt.Errorf("OSV ecosystem cannot be empty") + } + + state, err := dbGetOsvReplicationState(ctx, ecosystem) + if err != nil { + return false, fmt.Errorf("get OSV replication state: %w", err) + } + return state.IsFirstSynchronised, nil +} + +func InitializeEcosystem(ctx context.Context, payload FetchOSVEcosystemDumpPayload) error { + return newReplicator(ctx, payload).RunInitial() +} diff --git a/internal/osv/replicator.go b/internal/osv/replicator.go index add3e1e..92fd113 100644 --- a/internal/osv/replicator.go +++ b/internal/osv/replicator.go @@ -37,8 +37,8 @@ func (replicator *Replicator) Run() error { ctx := replicator.ctx payload := replicator.payload osvLog.Printf("OSV replication started: ecosystem=%s", payload.Ecosystem) - if strings.TrimSpace(payload.Ecosystem) == "" { - return fmt.Errorf("OSV ecosystem cannot be empty") + if err := replicator.validatePayload(); err != nil { + return err } state, err := dbGetOsvReplicationState(ctx, payload.Ecosystem) @@ -76,6 +76,39 @@ func (replicator *Replicator) Run() error { return nil } +func (replicator *Replicator) RunInitial() error { + if err := replicator.validatePayload(); err != nil { + return err + } + + state, err := dbGetOsvReplicationState(replicator.ctx, replicator.payload.Ecosystem) + if err != nil { + return fmt.Errorf("get OSV replication state: %w", err) + } + if state.IsFirstSynchronised { + osvLog.Printf("OSV first initialization skipped: ecosystem=%s, already synchronized", replicator.payload.Ecosystem) + return nil + } + + osvLog.Printf("OSV first initialization started: ecosystem=%s", replicator.payload.Ecosystem) + lastModifiedAt, err := replicator.runFull() + if err != nil { + return err + } + if err := dbCompleteOsvFullReplication(replicator.ctx, replicator.payload.Ecosystem, lastModifiedAt); err != nil { + return fmt.Errorf("complete OSV initial synchronization: %w", err) + } + osvLog.Printf("OSV first initialization completed: ecosystem=%s, last_modified_at=%v", replicator.payload.Ecosystem, lastModifiedAt) + return nil +} + +func (replicator *Replicator) validatePayload() error { + if strings.TrimSpace(replicator.payload.Ecosystem) == "" { + return fmt.Errorf("OSV ecosystem cannot be empty") + } + return nil +} + func (replicator *Replicator) runFull() (*time.Time, error) { osvLog.Printf("OSV full dump download started: ecosystem=%s", replicator.payload.Ecosystem) dump, err := fetchOSVEcosystemDump(replicator.ctx, replicator.payload)