From 44f715a4e1461e8e69b60f0ea0006892c3753ccb Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 14 Aug 2024 08:24:45 +0300 Subject: [PATCH] Add central entities table with properties (#4123) * Add central entities table with properties Signed-off-by: Juan Antonio Osorio * Add entity population to migration command This way it runs every time minder updates Signed-off-by: Juan Antonio Osorio * Create repositories on entities table as well Signed-off-by: Juan Antonio Osorio --------- Signed-off-by: Juan Antonio Osorio --- cmd/server/app/migrate_up.go | 16 ++ .../000090_entity_properties.down.sql | 20 ++ .../000090_entity_properties.up.sql | 37 +++ database/mock/store.go | 116 +++++++++ database/query/entities.sql | 66 ++++++ internal/db/entities.sql.go | 224 ++++++++++++++++++ internal/db/models.go | 18 ++ internal/db/querier.go | 14 ++ internal/repositories/github/service.go | 97 +++++--- internal/repositories/github/service_test.go | 10 + 10 files changed, 579 insertions(+), 39 deletions(-) create mode 100644 database/migrations/000090_entity_properties.down.sql create mode 100644 database/migrations/000090_entity_properties.up.sql create mode 100644 database/query/entities.sql create mode 100644 internal/db/entities.sql.go diff --git a/cmd/server/app/migrate_up.go b/cmd/server/app/migrate_up.go index ae9a81cbfd..4b5eb0f87e 100644 --- a/cmd/server/app/migrate_up.go +++ b/cmd/server/app/migrate_up.go @@ -32,6 +32,7 @@ import ( "github.com/stacklok/minder/internal/authz" "github.com/stacklok/minder/internal/config" serverconfig "github.com/stacklok/minder/internal/config/server" + "github.com/stacklok/minder/internal/db" "github.com/stacklok/minder/internal/logger" ) @@ -111,6 +112,21 @@ var upCmd = &cobra.Command{ return fmt.Errorf("error preparing authz client: %w", err) } + cmd.Println("Performing entity migrations...") + store := db.NewStore(dbConn) + + if err := store.TemporaryPopulateRepositories(ctx); err != nil { + cmd.Printf("Error while populating entities table with repos: %v\n", err) + } + + if err := store.TemporaryPopulateArtifacts(ctx); err != nil { + cmd.Printf("Error while populating entities table with artifacts: %v\n", err) + } + + if err := store.TemporaryPopulatePullRequests(ctx); err != nil { + cmd.Printf("Error while populating entities table with pull requests: %v\n", err) + } + return nil }, } diff --git a/database/migrations/000090_entity_properties.down.sql b/database/migrations/000090_entity_properties.down.sql new file mode 100644 index 0000000000..23ab3e3680 --- /dev/null +++ b/database/migrations/000090_entity_properties.down.sql @@ -0,0 +1,20 @@ +-- Copyright 2024 Stacklok, Inc +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +BEGIN; + +DROP TABLE IF EXISTS properties; +DROP TABLE IF EXISTS entities; + +COMMIT; \ No newline at end of file diff --git a/database/migrations/000090_entity_properties.up.sql b/database/migrations/000090_entity_properties.up.sql new file mode 100644 index 0000000000..d0655da760 --- /dev/null +++ b/database/migrations/000090_entity_properties.up.sql @@ -0,0 +1,37 @@ +-- Copyright 2024 Stacklok, Inc +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +BEGIN; + +CREATE TABLE IF NOT EXISTS entity_instances ( + id UUID NOT NULL DEFAULT gen_random_uuid() PRIMARY KEY, + entity_type entities NOT NULL, + name TEXT NOT NULL, + project_id UUID NOT NULL REFERENCES projects(id), + provider_id UUID NOT NULL REFERENCES providers(id), + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + originated_from UUID REFERENCES entity_instances(id) ON DELETE CASCADE, -- this is for entities that originate from other entities + UNIQUE(project_id, provider_id, entity_type, name) +); + +CREATE TABLE IF NOT EXISTS properties( + id UUID PRIMARY KEY, -- surrogate ID + entity_id UUID NOT NULL REFERENCES entity_instances(id) ON DELETE CASCADE, + key TEXT NOT NULL, -- we need to validate and ensure there are no dots + value JSONB NOT NULL, + updated_at TIMESTAMP NOT NULL DEFAULT NOW(), + UNIQUE (entity_id, key) +); + +COMMIT; \ No newline at end of file diff --git a/database/mock/store.go b/database/mock/store.go index 8b5ba14c87..aa3ce96b68 100644 --- a/database/mock/store.go +++ b/database/mock/store.go @@ -161,6 +161,36 @@ func (mr *MockStoreMockRecorder) CountUsers(arg0 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountUsers", reflect.TypeOf((*MockStore)(nil).CountUsers), arg0) } +// CreateEntity mocks base method. +func (m *MockStore) CreateEntity(arg0 context.Context, arg1 db.CreateEntityParams) (db.EntityInstance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateEntity", arg0, arg1) + ret0, _ := ret[0].(db.EntityInstance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateEntity indicates an expected call of CreateEntity. +func (mr *MockStoreMockRecorder) CreateEntity(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEntity", reflect.TypeOf((*MockStore)(nil).CreateEntity), arg0, arg1) +} + +// CreateEntityWithID mocks base method. +func (m *MockStore) CreateEntityWithID(arg0 context.Context, arg1 db.CreateEntityWithIDParams) (db.EntityInstance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateEntityWithID", arg0, arg1) + ret0, _ := ret[0].(db.EntityInstance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateEntityWithID indicates an expected call of CreateEntityWithID. +func (mr *MockStoreMockRecorder) CreateEntityWithID(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEntityWithID", reflect.TypeOf((*MockStore)(nil).CreateEntityWithID), arg0, arg1) +} + // CreateInvitation mocks base method. func (m *MockStore) CreateInvitation(arg0 context.Context, arg1 db.CreateInvitationParams) (db.UserInvite, error) { m.ctrl.T.Helper() @@ -370,6 +400,20 @@ func (mr *MockStoreMockRecorder) DeleteArtifact(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteArtifact", reflect.TypeOf((*MockStore)(nil).DeleteArtifact), arg0, arg1) } +// DeleteEntity mocks base method. +func (m *MockStore) DeleteEntity(arg0 context.Context, arg1 db.DeleteEntityParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteEntity", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteEntity indicates an expected call of DeleteEntity. +func (mr *MockStoreMockRecorder) DeleteEntity(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteEntity", reflect.TypeOf((*MockStore)(nil).DeleteEntity), arg0, arg1) +} + // DeleteEvaluationHistoryByIDs mocks base method. func (m *MockStore) DeleteEvaluationHistoryByIDs(arg0 context.Context, arg1 []uuid.UUID) (int64, error) { m.ctrl.T.Helper() @@ -763,6 +807,36 @@ func (mr *MockStoreMockRecorder) GetChildrenProjects(arg0, arg1 any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChildrenProjects", reflect.TypeOf((*MockStore)(nil).GetChildrenProjects), arg0, arg1) } +// GetEntitiesByType mocks base method. +func (m *MockStore) GetEntitiesByType(arg0 context.Context, arg1 db.GetEntitiesByTypeParams) ([]db.EntityInstance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEntitiesByType", arg0, arg1) + ret0, _ := ret[0].([]db.EntityInstance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetEntitiesByType indicates an expected call of GetEntitiesByType. +func (mr *MockStoreMockRecorder) GetEntitiesByType(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEntitiesByType", reflect.TypeOf((*MockStore)(nil).GetEntitiesByType), arg0, arg1) +} + +// GetEntityByID mocks base method. +func (m *MockStore) GetEntityByID(arg0 context.Context, arg1 db.GetEntityByIDParams) (db.EntityInstance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEntityByID", arg0, arg1) + ret0, _ := ret[0].(db.EntityInstance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetEntityByID indicates an expected call of GetEntityByID. +func (mr *MockStoreMockRecorder) GetEntityByID(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEntityByID", reflect.TypeOf((*MockStore)(nil).GetEntityByID), arg0, arg1) +} + // GetEvaluationHistory mocks base method. func (m *MockStore) GetEvaluationHistory(arg0 context.Context, arg1 db.GetEvaluationHistoryParams) (db.GetEvaluationHistoryRow, error) { m.ctrl.T.Helper() @@ -1942,6 +2016,48 @@ func (mr *MockStoreMockRecorder) SetCurrentVersion(arg0, arg1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCurrentVersion", reflect.TypeOf((*MockStore)(nil).SetCurrentVersion), arg0, arg1) } +// TemporaryPopulateArtifacts mocks base method. +func (m *MockStore) TemporaryPopulateArtifacts(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TemporaryPopulateArtifacts", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// TemporaryPopulateArtifacts indicates an expected call of TemporaryPopulateArtifacts. +func (mr *MockStoreMockRecorder) TemporaryPopulateArtifacts(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TemporaryPopulateArtifacts", reflect.TypeOf((*MockStore)(nil).TemporaryPopulateArtifacts), arg0) +} + +// TemporaryPopulatePullRequests mocks base method. +func (m *MockStore) TemporaryPopulatePullRequests(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TemporaryPopulatePullRequests", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// TemporaryPopulatePullRequests indicates an expected call of TemporaryPopulatePullRequests. +func (mr *MockStoreMockRecorder) TemporaryPopulatePullRequests(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TemporaryPopulatePullRequests", reflect.TypeOf((*MockStore)(nil).TemporaryPopulatePullRequests), arg0) +} + +// TemporaryPopulateRepositories mocks base method. +func (m *MockStore) TemporaryPopulateRepositories(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TemporaryPopulateRepositories", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// TemporaryPopulateRepositories indicates an expected call of TemporaryPopulateRepositories. +func (mr *MockStoreMockRecorder) TemporaryPopulateRepositories(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TemporaryPopulateRepositories", reflect.TypeOf((*MockStore)(nil).TemporaryPopulateRepositories), arg0) +} + // UpdateEncryptedSecret mocks base method. func (m *MockStore) UpdateEncryptedSecret(arg0 context.Context, arg1 db.UpdateEncryptedSecretParams) error { m.ctrl.T.Helper() diff --git a/database/query/entities.sql b/database/query/entities.sql new file mode 100644 index 0000000000..8fb793ea69 --- /dev/null +++ b/database/query/entities.sql @@ -0,0 +1,66 @@ +-- CreateEntity adds an entry to the entity_instances table so it can be tracked by Minder. + +-- name: CreateEntity :one +INSERT INTO entity_instances ( + entity_type, + name, + project_id, + provider_id, + originated_from +) VALUES ($1, $2, sqlc.arg(project_id), sqlc.arg(provider_id), sqlc.narg(originated_from)) +RETURNING *; + +-- CreateEntityWithID adds an entry to the entities table with a specific ID so it can be tracked by Minder. + +-- name: CreateEntityWithID :one +INSERT INTO entity_instances ( + id, + entity_type, + name, + project_id, + provider_id, + originated_from +) VALUES ($1, $2, $3, sqlc.arg(project_id), sqlc.arg(provider_id), sqlc.narg(originated_from)) +RETURNING *; + +-- DeleteEntity removes an entity from the entity_instances table for a project. + +-- name: DeleteEntity :exec +DELETE FROM entity_instances +WHERE id = $1 AND project_id = $2; + +-- GetEntityByID retrieves an entity by its ID for a project or hierarchy of projects. + +-- name: GetEntityByID :one +SELECT * FROM entity_instances +WHERE entity_instances.id = $1 AND entity_instances.project_id = ANY(sqlc.arg(projects)::uuid[]) +LIMIT 1; + +-- GetEntityByName retrieves an entity by its name for a project or hierarchy of projects. +SELECT * FROM entity_instances +WHERE lower(entity_instances.name) = lower(sqlc.arg(name)) AND entity_instances.project_id = $1 +LIMIT 1; + +-- GetEntitiesByType retrieves all entities of a given type for a project or hierarchy of projects. +-- this is how one would get all repositories, artifacts, etc. + +-- name: GetEntitiesByType :many +SELECT * FROM entity_instances +WHERE entity_instances.entity_type = $1 AND entity_instances.project_id = ANY(sqlc.arg(projects)::uuid[]); + +-- name: TemporaryPopulateRepositories :exec +INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at) +SELECT id, 'repository', repo_owner || '/' || repo_name, project_id, provider_id, created_at FROM repositories +WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = repositories.id AND entity_instances.entity_type = 'repository'); + +-- name: TemporaryPopulateArtifacts :exec +INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at, originated_from) +SELECT artifacts.id, 'artifact', LOWER(repositories.repo_owner) || '/' || artifacts.artifact_name, repositories.project_id, repositories.provider_id, artifacts.created_at, artifacts.repository_id FROM artifacts +JOIN repositories ON repositories.id = artifacts.repository_id +WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = artifacts.id AND entity_instances.entity_type = 'artifact'); + +-- name: TemporaryPopulatePullRequests :exec +INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at, originated_from) +SELECT pull_requests.id, 'pull_request', repositories.repo_owner || '/' || repositories.repo_name || '/' || pull_requests.pr_number::TEXT, repositories.project_id, repositories.provider_id, pull_requests.created_at, pull_requests.repository_id FROM pull_requests +JOIN repositories ON repositories.id = pull_requests.repository_id +WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = pull_requests.id AND entity_instances.entity_type = 'pull_request'); diff --git a/internal/db/entities.sql.go b/internal/db/entities.sql.go new file mode 100644 index 0000000000..132c684cda --- /dev/null +++ b/internal/db/entities.sql.go @@ -0,0 +1,224 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 +// source: entities.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/lib/pq" +) + +const createEntity = `-- name: CreateEntity :one + +INSERT INTO entity_instances ( + entity_type, + name, + project_id, + provider_id, + originated_from +) VALUES ($1, $2, $3, $4, $5) +RETURNING id, entity_type, name, project_id, provider_id, created_at, originated_from +` + +type CreateEntityParams struct { + EntityType Entities `json:"entity_type"` + Name string `json:"name"` + ProjectID uuid.UUID `json:"project_id"` + ProviderID uuid.UUID `json:"provider_id"` + OriginatedFrom uuid.NullUUID `json:"originated_from"` +} + +// CreateEntity adds an entry to the entity_instances table so it can be tracked by Minder. +func (q *Queries) CreateEntity(ctx context.Context, arg CreateEntityParams) (EntityInstance, error) { + row := q.db.QueryRowContext(ctx, createEntity, + arg.EntityType, + arg.Name, + arg.ProjectID, + arg.ProviderID, + arg.OriginatedFrom, + ) + var i EntityInstance + err := row.Scan( + &i.ID, + &i.EntityType, + &i.Name, + &i.ProjectID, + &i.ProviderID, + &i.CreatedAt, + &i.OriginatedFrom, + ) + return i, err +} + +const createEntityWithID = `-- name: CreateEntityWithID :one + +INSERT INTO entity_instances ( + id, + entity_type, + name, + project_id, + provider_id, + originated_from +) VALUES ($1, $2, $3, $4, $5, $6) +RETURNING id, entity_type, name, project_id, provider_id, created_at, originated_from +` + +type CreateEntityWithIDParams struct { + ID uuid.UUID `json:"id"` + EntityType Entities `json:"entity_type"` + Name string `json:"name"` + ProjectID uuid.UUID `json:"project_id"` + ProviderID uuid.UUID `json:"provider_id"` + OriginatedFrom uuid.NullUUID `json:"originated_from"` +} + +// CreateEntityWithID adds an entry to the entities table with a specific ID so it can be tracked by Minder. +func (q *Queries) CreateEntityWithID(ctx context.Context, arg CreateEntityWithIDParams) (EntityInstance, error) { + row := q.db.QueryRowContext(ctx, createEntityWithID, + arg.ID, + arg.EntityType, + arg.Name, + arg.ProjectID, + arg.ProviderID, + arg.OriginatedFrom, + ) + var i EntityInstance + err := row.Scan( + &i.ID, + &i.EntityType, + &i.Name, + &i.ProjectID, + &i.ProviderID, + &i.CreatedAt, + &i.OriginatedFrom, + ) + return i, err +} + +const deleteEntity = `-- name: DeleteEntity :exec + +DELETE FROM entity_instances +WHERE id = $1 AND project_id = $2 +` + +type DeleteEntityParams struct { + ID uuid.UUID `json:"id"` + ProjectID uuid.UUID `json:"project_id"` +} + +// DeleteEntity removes an entity from the entity_instances table for a project. +func (q *Queries) DeleteEntity(ctx context.Context, arg DeleteEntityParams) error { + _, err := q.db.ExecContext(ctx, deleteEntity, arg.ID, arg.ProjectID) + return err +} + +const getEntitiesByType = `-- name: GetEntitiesByType :many + +SELECT id, entity_type, name, project_id, provider_id, created_at, originated_from FROM entity_instances +WHERE entity_instances.entity_type = $1 AND entity_instances.project_id = ANY($2::uuid[]) +` + +type GetEntitiesByTypeParams struct { + EntityType Entities `json:"entity_type"` + Projects []uuid.UUID `json:"projects"` +} + +// GetEntitiesByType retrieves all entities of a given type for a project or hierarchy of projects. +// this is how one would get all repositories, artifacts, etc. +func (q *Queries) GetEntitiesByType(ctx context.Context, arg GetEntitiesByTypeParams) ([]EntityInstance, error) { + rows, err := q.db.QueryContext(ctx, getEntitiesByType, arg.EntityType, pq.Array(arg.Projects)) + if err != nil { + return nil, err + } + defer rows.Close() + items := []EntityInstance{} + for rows.Next() { + var i EntityInstance + if err := rows.Scan( + &i.ID, + &i.EntityType, + &i.Name, + &i.ProjectID, + &i.ProviderID, + &i.CreatedAt, + &i.OriginatedFrom, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getEntityByID = `-- name: GetEntityByID :one + +SELECT id, entity_type, name, project_id, provider_id, created_at, originated_from FROM entity_instances +WHERE entity_instances.id = $1 AND entity_instances.project_id = ANY($2::uuid[]) +LIMIT 1 +` + +type GetEntityByIDParams struct { + ID uuid.UUID `json:"id"` + Projects []uuid.UUID `json:"projects"` +} + +// GetEntityByID retrieves an entity by its ID for a project or hierarchy of projects. +func (q *Queries) GetEntityByID(ctx context.Context, arg GetEntityByIDParams) (EntityInstance, error) { + row := q.db.QueryRowContext(ctx, getEntityByID, arg.ID, pq.Array(arg.Projects)) + var i EntityInstance + err := row.Scan( + &i.ID, + &i.EntityType, + &i.Name, + &i.ProjectID, + &i.ProviderID, + &i.CreatedAt, + &i.OriginatedFrom, + ) + return i, err +} + +const temporaryPopulateArtifacts = `-- name: TemporaryPopulateArtifacts :exec +INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at, originated_from) +SELECT artifacts.id, 'artifact', LOWER(repositories.repo_owner) || '/' || artifacts.artifact_name, repositories.project_id, repositories.provider_id, artifacts.created_at, artifacts.repository_id FROM artifacts +JOIN repositories ON repositories.id = artifacts.repository_id +WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = artifacts.id AND entity_instances.entity_type = 'artifact') +` + +func (q *Queries) TemporaryPopulateArtifacts(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, temporaryPopulateArtifacts) + return err +} + +const temporaryPopulatePullRequests = `-- name: TemporaryPopulatePullRequests :exec +INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at, originated_from) +SELECT pull_requests.id, 'pull_request', repositories.repo_owner || '/' || repositories.repo_name || '/' || pull_requests.pr_number::TEXT, repositories.project_id, repositories.provider_id, pull_requests.created_at, pull_requests.repository_id FROM pull_requests +JOIN repositories ON repositories.id = pull_requests.repository_id +WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = pull_requests.id AND entity_instances.entity_type = 'pull_request') +` + +func (q *Queries) TemporaryPopulatePullRequests(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, temporaryPopulatePullRequests) + return err +} + +const temporaryPopulateRepositories = `-- name: TemporaryPopulateRepositories :exec +INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at) +SELECT id, 'repository', repo_owner || '/' || repo_name, project_id, provider_id, created_at FROM repositories +WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = repositories.id AND entity_instances.entity_type = 'repository') +` + +func (q *Queries) TemporaryPopulateRepositories(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, temporaryPopulateRepositories) + return err +} diff --git a/internal/db/models.go b/internal/db/models.go index b5e6c75b55..18d65e45a5 100644 --- a/internal/db/models.go +++ b/internal/db/models.go @@ -469,6 +469,16 @@ type EntityExecutionLock struct { ProjectID uuid.NullUUID `json:"project_id"` } +type EntityInstance struct { + ID uuid.UUID `json:"id"` + EntityType Entities `json:"entity_type"` + Name string `json:"name"` + ProjectID uuid.UUID `json:"project_id"` + ProviderID uuid.UUID `json:"provider_id"` + CreatedAt time.Time `json:"created_at"` + OriginatedFrom uuid.NullUUID `json:"originated_from"` +} + type EntityProfile struct { ID uuid.UUID `json:"id"` Entity Entities `json:"entity"` @@ -570,6 +580,14 @@ type Project struct { UpdatedAt time.Time `json:"updated_at"` } +type Property struct { + ID uuid.UUID `json:"id"` + EntityID uuid.UUID `json:"entity_id"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + UpdatedAt time.Time `json:"updated_at"` +} + type Provider struct { ID uuid.UUID `json:"id"` Name string `json:"name"` diff --git a/internal/db/querier.go b/internal/db/querier.go index af29a278b9..2cc3fec66a 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -18,6 +18,10 @@ type Querier interface { CountProfilesByName(ctx context.Context, name string) (int64, error) CountRepositories(ctx context.Context) (int64, error) CountUsers(ctx context.Context) (int64, error) + // CreateEntity adds an entry to the entity_instances table so it can be tracked by Minder. + CreateEntity(ctx context.Context, arg CreateEntityParams) (EntityInstance, error) + // CreateEntityWithID adds an entry to the entities table with a specific ID so it can be tracked by Minder. + CreateEntityWithID(ctx context.Context, arg CreateEntityWithIDParams) (EntityInstance, error) // CreateInvitation creates a new invitation. The code is a secret that is sent // to the invitee, and the email is the address to which the invitation will be // sent. The role is the role that the invitee will have when they accept the @@ -38,6 +42,8 @@ type Querier interface { CreateSubscription(ctx context.Context, arg CreateSubscriptionParams) (Subscription, error) CreateUser(ctx context.Context, identitySubject string) (User, error) DeleteArtifact(ctx context.Context, id uuid.UUID) error + // DeleteEntity removes an entity from the entity_instances table for a project. + DeleteEntity(ctx context.Context, arg DeleteEntityParams) error DeleteEvaluationHistoryByIDs(ctx context.Context, evaluationids []uuid.UUID) (int64, error) DeleteExpiredSessionStates(ctx context.Context) (int64, error) DeleteInstallationIDByAppID(ctx context.Context, appInstallationID int64) error @@ -71,6 +77,11 @@ type Querier interface { GetArtifactByName(ctx context.Context, arg GetArtifactByNameParams) (Artifact, error) GetBundle(ctx context.Context, arg GetBundleParams) (Bundle, error) GetChildrenProjects(ctx context.Context, id uuid.UUID) ([]GetChildrenProjectsRow, error) + // GetEntitiesByType retrieves all entities of a given type for a project or hierarchy of projects. + // this is how one would get all repositories, artifacts, etc. + GetEntitiesByType(ctx context.Context, arg GetEntitiesByTypeParams) ([]EntityInstance, error) + // GetEntityByID retrieves an entity by its ID for a project or hierarchy of projects. + GetEntityByID(ctx context.Context, arg GetEntityByIDParams) (EntityInstance, error) GetEvaluationHistory(ctx context.Context, arg GetEvaluationHistoryParams) (GetEvaluationHistoryRow, error) // GetFeatureInProject verifies if a feature is available for a specific project. // It returns the settings for the feature if it is available. @@ -219,6 +230,9 @@ type Querier interface { ReleaseLock(ctx context.Context, arg ReleaseLockParams) error RepositoryExistsAfterID(ctx context.Context, id uuid.UUID) (bool, error) SetCurrentVersion(ctx context.Context, arg SetCurrentVersionParams) error + TemporaryPopulateArtifacts(ctx context.Context) error + TemporaryPopulatePullRequests(ctx context.Context) error + TemporaryPopulateRepositories(ctx context.Context) error UpdateEncryptedSecret(ctx context.Context, arg UpdateEncryptedSecretParams) error // UpdateInvitationRole updates an invitation by its code. This is intended to be // called by a user who has issued an invitation and then decided to change the diff --git a/internal/repositories/github/service.go b/internal/repositories/github/service.go index 4cc92525e3..e56d68a223 100644 --- a/internal/repositories/github/service.go +++ b/internal/repositories/github/service.go @@ -360,49 +360,68 @@ func (r *repositoryService) persistRepository( projectID uuid.UUID, provider *db.Provider, ) (uuid.UUID, *pb.Repository, error) { - // instantiate the response object - pbRepo := &pb.Repository{ - Name: githubRepo.GetName(), - Owner: githubRepo.GetOwner().GetLogin(), - RepoId: githubRepo.GetID(), - HookId: githubHook.GetID(), - HookUrl: githubHook.GetURL(), - DeployUrl: githubRepo.GetDeploymentsURL(), - CloneUrl: githubRepo.GetCloneURL(), - HookType: githubHook.GetType(), - HookName: githubHook.GetName(), - HookUuid: hookUUID, - IsPrivate: githubRepo.GetPrivate(), - IsFork: githubRepo.GetFork(), - DefaultBranch: githubRepo.GetDefaultBranch(), - } + var outid uuid.UUID + pbr, err := db.WithTransaction(r.store, func(t db.ExtendQuerier) (*pb.Repository, error) { + // instantiate the response object + pbRepo := &pb.Repository{ + Name: githubRepo.GetName(), + Owner: githubRepo.GetOwner().GetLogin(), + RepoId: githubRepo.GetID(), + HookId: githubHook.GetID(), + HookUrl: githubHook.GetURL(), + DeployUrl: githubRepo.GetDeploymentsURL(), + CloneUrl: githubRepo.GetCloneURL(), + HookType: githubHook.GetType(), + HookName: githubHook.GetName(), + HookUuid: hookUUID, + IsPrivate: githubRepo.GetPrivate(), + IsFork: githubRepo.GetFork(), + DefaultBranch: githubRepo.GetDefaultBranch(), + } - // update the database - dbRepo, err := r.store.CreateRepository(ctx, db.CreateRepositoryParams{ - Provider: provider.Name, - ProviderID: provider.ID, - ProjectID: projectID, - RepoOwner: pbRepo.Owner, - RepoName: pbRepo.Name, - RepoID: pbRepo.RepoId, - IsPrivate: pbRepo.IsPrivate, - IsFork: pbRepo.IsFork, - WebhookID: sql.NullInt64{ - Int64: pbRepo.HookId, - Valid: true, - }, - CloneUrl: pbRepo.CloneUrl, - WebhookUrl: pbRepo.HookUrl, - DeployUrl: pbRepo.DeployUrl, - DefaultBranch: sql.NullString{ - String: pbRepo.DefaultBranch, - Valid: true, - }, + // update the database + dbRepo, err := t.CreateRepository(ctx, db.CreateRepositoryParams{ + Provider: provider.Name, + ProviderID: provider.ID, + ProjectID: projectID, + RepoOwner: pbRepo.Owner, + RepoName: pbRepo.Name, + RepoID: pbRepo.RepoId, + IsPrivate: pbRepo.IsPrivate, + IsFork: pbRepo.IsFork, + WebhookID: sql.NullInt64{ + Int64: pbRepo.HookId, + Valid: true, + }, + CloneUrl: pbRepo.CloneUrl, + WebhookUrl: pbRepo.HookUrl, + DeployUrl: pbRepo.DeployUrl, + DefaultBranch: sql.NullString{ + String: pbRepo.DefaultBranch, + Valid: true, + }, + }) + if err != nil { + return pbRepo, err + } + + outid = dbRepo.ID + pbRepo.Id = ptr.Ptr(dbRepo.ID.String()) + + // TODO: Replace with CreateEntity call + _, err = t.CreateEntityWithID(ctx, db.CreateEntityWithIDParams{ + ID: dbRepo.ID, + EntityType: db.EntitiesRepository, + Name: fmt.Sprintf("%s/%s", pbRepo.Owner, pbRepo.Name), + ProjectID: projectID, + ProviderID: provider.ID, + }) + + return pbRepo, err }) if err != nil { return uuid.Nil, nil, err } - pbRepo.Id = ptr.Ptr(dbRepo.ID.String()) - return dbRepo.ID, pbRepo, nil + return outid, pbr, nil } diff --git a/internal/repositories/github/service_test.go b/internal/repositories/github/service_test.go index 99e6ec4bd2..1d37a64c2a 100644 --- a/internal/repositories/github/service_test.go +++ b/internal/repositories/github/service_test.go @@ -501,15 +501,25 @@ func withSuccessfulGetByName(mock dbMock) { } func withFailedCreate(mock dbMock) { + mock.EXPECT().GetQuerierWithTransaction(gomock.Any()).Return(mock) + mock.EXPECT().BeginTransaction().Return(nil, nil) mock.EXPECT(). CreateRepository(gomock.Any(), gomock.Any()). Return(db.Repository{}, errDefault) + mock.EXPECT().Rollback(gomock.Any()).Return(nil) } func withSuccessfulCreate(mock dbMock) { + mock.EXPECT().GetQuerierWithTransaction(gomock.Any()).Return(mock) + mock.EXPECT().BeginTransaction().Return(nil, nil) mock.EXPECT(). CreateRepository(gomock.Any(), gomock.Any()). Return(dbRepo, nil) + mock.EXPECT(). + CreateEntityWithID(gomock.Any(), gomock.Any()). + Return(db.EntityInstance{}, nil) + mock.EXPECT().Commit(gomock.Any()).Return(nil) + mock.EXPECT().Rollback(gomock.Any()).Return(nil) } func withPrivateReposEnabled(mock dbMock) {