Skip to content
Snippets Groups Projects
Commit 899e36f7 authored by David Fernandez's avatar David Fernandez
Browse files

Add NPM dependencies support


Add `Packages::Dependency` and `Packages::DependencyLink`
  models
Add or update related services
Update `NpmPackagePresenter` to properly include dependencies

Co-Authored-By: default avatarSara Ahbabou <sahbabou@gitlab.com>
parent bf2e083d
No related branches found
No related tags found
No related merge requests found
Showing
with 509 additions and 17 deletions
# frozen_string_literal: true
class CreatePackagesDependencies < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
create_table :packages_dependencies do |t|
t.string :name, null: false, limit: 255
t.string :version_pattern, null: false, limit: 255
end
add_index :packages_dependencies, [:name, :version_pattern], unique: true
end
end
# frozen_string_literal: true
class CreatePackagesDependencyLinks < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
create_table :packages_dependency_links do |t|
t.references :package, index: false, null: false, foreign_key: { to_table: :packages_packages, on_delete: :cascade }, type: :bigint
t.references :dependency, null: false, foreign_key: { to_table: :packages_dependencies, on_delete: :cascade }, type: :bigint
t.integer :dependency_type, limit: 2, null: false
end
add_index :packages_dependency_links, [:package_id, :dependency_id, :dependency_type], unique: true, name: 'idx_pkgs_dep_links_on_pkg_id_dependency_id_dependency_type'
end
end
# frozen_string_literal: true
class AddProjectIdNameVersionPackageTypeIndexToPackagesPackages < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'idx_packages_packages_on_project_id_name_version_package_type'.freeze
disable_ddl_transaction!
def up
add_concurrent_index :packages_packages,
[:project_id, :name, :version, :package_type],
name: INDEX_NAME
end
def down
remove_concurrent_index :packages_packages,
[:project_id, :name, :version, :package_type],
name: INDEX_NAME
end
end
# frozen_string_literal: true
class DropPackagesPackageMetadataTable < ActiveRecord::Migration[5.2]
DOWNTIME = false
def up
drop_table :packages_package_metadata
end
def down
create_table :packages_package_metadata do |t|
t.references :package, index: { unique: true }, null: false, foreign_key: { to_table: :packages_packages, on_delete: :cascade }, type: :integer
t.binary :metadata, null: false
end
end
end
Loading
Loading
@@ -2822,6 +2822,20 @@ ActiveRecord::Schema.define(version: 2019_12_04_093410) do
t.index ["package_id"], name: "index_packages_conan_metadata_on_package_id", unique: true
end
 
create_table "packages_dependencies", force: :cascade do |t|
t.string "name", limit: 255, null: false
t.string "version_pattern", limit: 255, null: false
t.index ["name", "version_pattern"], name: "index_packages_dependencies_on_name_and_version_pattern", unique: true
end
create_table "packages_dependency_links", force: :cascade do |t|
t.bigint "package_id", null: false
t.bigint "dependency_id", null: false
t.integer "dependency_type", limit: 2, null: false
t.index ["dependency_id"], name: "index_packages_dependency_links_on_dependency_id"
t.index ["package_id", "dependency_id", "dependency_type"], name: "idx_pkgs_dep_links_on_pkg_id_dependency_id_dependency_type", unique: true
end
create_table "packages_maven_metadata", force: :cascade do |t|
t.bigint "package_id", null: false
t.datetime_with_timezone "created_at", null: false
Loading
Loading
@@ -2847,12 +2861,6 @@ ActiveRecord::Schema.define(version: 2019_12_04_093410) do
t.index ["package_id", "file_name"], name: "index_packages_package_files_on_package_id_and_file_name"
end
 
create_table "packages_package_metadata", force: :cascade do |t|
t.integer "package_id", null: false
t.binary "metadata", null: false
t.index ["package_id"], name: "index_packages_package_metadata_on_package_id", unique: true
end
create_table "packages_package_tags", force: :cascade do |t|
t.integer "package_id", null: false
t.string "name", limit: 255, null: false
Loading
Loading
@@ -2867,6 +2875,7 @@ ActiveRecord::Schema.define(version: 2019_12_04_093410) do
t.string "version"
t.integer "package_type", limit: 2, null: false
t.index ["name"], name: "index_packages_packages_on_name_trigram", opclass: :gin_trgm_ops, using: :gin
t.index ["project_id", "name", "version", "package_type"], name: "idx_packages_packages_on_project_id_name_version_package_type"
t.index ["project_id"], name: "index_packages_packages_on_project_id"
end
 
Loading
Loading
@@ -4565,9 +4574,10 @@ ActiveRecord::Schema.define(version: 2019_12_04_093410) do
add_foreign_key "operations_feature_flags_clients", "projects", on_delete: :cascade
add_foreign_key "packages_conan_file_metadata", "packages_package_files", column: "package_file_id", on_delete: :cascade
add_foreign_key "packages_conan_metadata", "packages_packages", column: "package_id", on_delete: :cascade
add_foreign_key "packages_dependency_links", "packages_dependencies", column: "dependency_id", on_delete: :cascade
add_foreign_key "packages_dependency_links", "packages_packages", column: "package_id", on_delete: :cascade
add_foreign_key "packages_maven_metadata", "packages_packages", column: "package_id", name: "fk_be88aed360", on_delete: :cascade
add_foreign_key "packages_package_files", "packages_packages", column: "package_id", name: "fk_86f0f182f8", on_delete: :cascade
add_foreign_key "packages_package_metadata", "packages_packages", column: "package_id", on_delete: :cascade
add_foreign_key "packages_package_tags", "packages_packages", column: "package_id", on_delete: :cascade
add_foreign_key "packages_packages", "projects", on_delete: :cascade
add_foreign_key "pages_domain_acme_orders", "pages_domains", on_delete: :cascade
Loading
Loading
Loading
Loading
@@ -122,7 +122,7 @@ Then, you could run `npm publish` either locally or via GitLab CI/CD:
 
- **GitLab CI/CD:** Set an `NPM_TOKEN` [variable](../../../ci/variables/README.md)
under your project's **Settings > CI/CD > Variables**.
### Authenticating with a CI job token
 
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9104) in GitLab Premium 12.5.
Loading
Loading
@@ -130,7 +130,7 @@ Then, you could run `npm publish` either locally or via GitLab CI/CD:
If you’re using NPM with GitLab CI/CD, a CI job token can be used instead of a personal access token.
The token will inherit the permissions of the user that generates the pipeline.
 
Add a corresponding section to your `.npmrc` file:
Add a corresponding section to your `.npmrc` file:
 
```ini
@foo:registry=https://gitlab.com/api/v4/packages/npm/
Loading
Loading
@@ -226,3 +226,19 @@ And the `.npmrc` file should look like:
//gitlab.com/api/v4/packages/npm/:_authToken=<your_oauth_token>
@foo:registry=https://gitlab.com/api/v4/packages/npm/
```
## NPM dependencies metadata
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11867) in GitLab Premium 12.6.
Starting from GitLab 12.6, new packages published to the GitLab NPM Registry expose the following attributes to the NPM client:
- name
- version
- dist-tags
- dependencies
- dependencies
- devDependencies
- bundleDependencies
- peerDependencies
- deprecated
# frozen_string_literal: true
class Packages::Dependency < ApplicationRecord
has_many :dependency_links, class_name: 'Packages::DependencyLink'
validates :name, :version_pattern, presence: true
validates :name, uniqueness: { scope: :version_pattern }
NAME_VERSION_PATTERN_TUPLE_MATCHING = '(name, version_pattern) = (?, ?)'.freeze
MAX_STRING_LENGTH = 255.freeze
MAX_CHUNKED_QUERIES_COUNT = 10.freeze
def self.for_package_names_and_version_patterns(names_and_version_patterns = {}, chunk_size = 50, max_rows_limit = 200)
names_and_version_patterns.reject! { |key, value| key.size > MAX_STRING_LENGTH || value.size > MAX_STRING_LENGTH }
raise ArgumentError, 'Too many names_and_version_patterns' if names_and_version_patterns.size > MAX_CHUNKED_QUERIES_COUNT * chunk_size
matched_ids = []
names_and_version_patterns.each_slice(chunk_size) do |tuples|
where_statement = Array.new(tuples.size, NAME_VERSION_PATTERN_TUPLE_MATCHING)
.join(' OR ')
ids = where(where_statement, *tuples.flatten)
.limit(max_rows_limit + 1)
.pluck(:id)
matched_ids.concat(ids)
raise ArgumentError, 'Too many Dependencies selected' if matched_ids.size > max_rows_limit
end
return none if matched_ids.empty?
where(id: matched_ids)
end
def self.pluck_ids_and_names
pluck(:id, :name)
end
def orphaned?
self.dependency_links.empty?
end
end
# frozen_string_literal: true
class Packages::DependencyLink < ApplicationRecord
belongs_to :package, inverse_of: :dependency_links
belongs_to :dependency, inverse_of: :dependency_links, class_name: 'Packages::Dependency'
validates :package, :dependency, presence: true
validates :dependency_type,
uniqueness: { scope: %i[package_id dependency_id] }
enum dependency_type: { dependencies: 1, devDependencies: 2, bundleDependencies: 3, peerDependencies: 4, deprecated: 5 }
scope :with_dependency_type, ->(dependency_type) { where(dependency_type: dependency_type) }
scope :includes_dependency, -> { includes(:dependency) }
end
Loading
Loading
@@ -5,6 +5,7 @@ class Packages::Package < ApplicationRecord
belongs_to :project
# package_files must be destroyed by ruby code in order to properly remove carrierwave uploads and update project statistics
has_many :package_files, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :dependency_links, inverse_of: :package, class_name: 'Packages::DependencyLink'
has_one :conan_metadatum, inverse_of: :package
has_one :maven_metadatum, inverse_of: :package
 
Loading
Loading
@@ -19,6 +20,9 @@ class Packages::Package < ApplicationRecord
presence: true,
format: { with: Gitlab::Regex.package_name_regex }
 
validates :name,
uniqueness: { scope: %i[project_id version package_type] }
validate :valid_npm_package_name, if: :npm?
validate :package_already_taken, if: :npm?
 
Loading
Loading
Loading
Loading
@@ -3,10 +3,11 @@
class NpmPackagePresenter
include API::Helpers::RelatedResourcesHelpers
 
attr_reader :project, :name, :packages
attr_reader :name, :packages
 
def initialize(project, name, packages)
@project = project
NPM_VALID_DEPENDENCY_TYPES = %i[dependencies devDependencies bundleDependencies peerDependencies deprecated].freeze
def initialize(name, packages)
@name = name
@packages = packages
end
Loading
Loading
@@ -41,7 +42,9 @@ class NpmPackagePresenter
shasum: package_file.file_sha1,
tarball: tarball_url(package, package_file)
}
}
}.tap do |package_version|
package_version.merge!(build_package_dependencies(package))
end
end
 
def tarball_url(package, package_file)
Loading
Loading
@@ -50,6 +53,22 @@ class NpmPackagePresenter
"/-/#{package_file.file_name}"
end
 
def build_package_dependencies(package)
return {} if package.dependency_links.empty?
dependencies = Hash.new { |h, key| h[key] = {} }
dependency_links = package.dependency_links
.with_dependency_type(NPM_VALID_DEPENDENCY_TYPES)
.includes_dependency
dependency_links.find_each do |dependency_link|
dependency = dependency_link.dependency
dependencies[dependency_link.dependency_type][dependency.name] = dependency.version_pattern
end
dependencies
end
def sorted_versions
versions = packages.map(&:version).compact
VersionSorter.sort(versions)
Loading
Loading
# frozen_string_literal: true
module Packages
class CreateDependencyService < BaseService
attr_reader :package, :dependencies
def initialize(package, dependencies)
@package = package
@dependencies = dependencies
end
def execute
Packages::DependencyLink.dependency_types.each_key do |type|
create_dependency(type)
end
end
private
def create_dependency(type)
return unless dependencies.key?(type)
names_and_version_patterns = dependencies[type].to_a
existing_ids, existing_names = find_existing_ids_and_names(names_and_version_patterns)
dependencies_to_insert = names_and_version_patterns
if existing_names.any?
dependencies_to_insert = names_and_version_patterns.reject { |e| e.first.in?(existing_names) }
end
ActiveRecord::Base.transaction do
inserted_ids = bulk_insert_package_dependencies(dependencies_to_insert)
bulk_insert_package_dependency_links(type, (existing_ids + inserted_ids))
end
end
def find_existing_ids_and_names(names_and_version_patterns)
ids_and_names = Packages::Dependency.for_package_names_and_version_patterns(names_and_version_patterns)
.pluck_ids_and_names
ids = ids_and_names.map(&:first) || []
names = ids_and_names.map(&:second) || []
[ids, names]
end
def bulk_insert_package_dependencies(names_and_version_patterns)
return [] if names_and_version_patterns.empty?
rows = names_and_version_patterns.map do |name, version_pattern|
{
name: name,
version_pattern: version_pattern
}
end
database.bulk_insert(Packages::Dependency.table_name, rows, return_ids: true)
end
def bulk_insert_package_dependency_links(type, dependency_ids)
rows = dependency_ids.map do |dependency_id|
{
package_id: package.id,
dependency_id: dependency_id,
dependency_type: Packages::DependencyLink.dependency_types[type.to_s]
}
end
database.bulk_insert(Packages::DependencyLink.table_name, rows)
end
def database
::Gitlab::Database
end
end
end
Loading
Loading
@@ -26,9 +26,17 @@ module Packages
file_name: package_file_name
}
 
::Packages::CreatePackageFileService.new(package, file_params).execute
package.transaction do
::Packages::CreatePackageFileService.new(package, file_params).execute
::Packages::CreateDependencyService.new(package, package_dependencies).execute
end
 
package
end
def package_dependencies
_version, version_data = params[:versions].first
version_data
end
end
end
---
title: Fix dependency metadata on the NPM registry responses
merge_request: 20549
author:
type: fixed
Loading
Loading
@@ -40,7 +40,7 @@ module API
packages = ::Packages::NpmPackagesFinder
.new(project, package_name).execute
 
present NpmPackagePresenter.new(project, package_name, packages),
present NpmPackagePresenter.new(package_name, packages),
with: EE::API::Entities::NpmPackage
end
 
Loading
Loading
Loading
Loading
@@ -3,7 +3,7 @@ FactoryBot.define do
factory :package, class: Packages::Package do
project
name { 'my/company/app/my-app' }
version { '1.0-SNAPSHOT' }
sequence(:version) { |n| "1.#{n}-SNAPSHOT" }
package_type { 'maven' }
 
factory :maven_package do
Loading
Loading
@@ -188,4 +188,15 @@ FactoryBot.define do
conan_package_reference { '123456789' }
end
end
factory :packages_dependency, class: Packages::Dependency do
sequence(:name) { |n| "@test/package-#{n}"}
sequence(:version_pattern) { |n| "~6.2.#{n}" }
end
factory :packages_dependency_link, class: Packages::DependencyLink do
package
dependency { create(:packages_dependency) }
dependency_type { :dependencies }
end
end
Loading
Loading
@@ -4,13 +4,43 @@
"properties" : {
"name": { "type": "string" },
"version": { "type": "string" },
"dist": {
"dist": {
"type": "object",
"required": ["shasum", "tarball"],
"properties" : {
"shasum": { "type": "string" },
"tarball": { "type": "string" }
}
},
"dependencies": {
"type": "object",
"patternProperties": {
".{1,}": { "type": "string" }
}
},
"devDependencies": {
"type": "object",
"patternProperties": {
".{1,}": { "type": "string" }
}
},
"bundleDependencies": {
"type": "object",
"patternProperties": {
".{1,}": { "type": "string" }
}
},
"peerDependencies": {
"type": "object",
"patternProperties": {
".{1,}": { "type": "string" }
}
},
"deprecated": {
"type": "object",
"patternProperties": {
".{1,}": { "type": "string" }
}
}
}
}
{
"_id":"@root/npm-test",
"name":"@root/npm-test",
"description":"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
"dist-tags":{
"latest":"1.0.1"
},
"versions":{
"1.0.1":{
"name":"@root/npm-test",
"version":"1.0.1",
"main":"app.js",
"dependencies":{
"express":"^4.16.4",
"dagre-d3": "~0.3.2"
},
"devDependencies": {
"dagre-d3": "~0.3.2",
"d3": "~3.4.13"
},
"bundleDependencies": {
"d3": "~3.4.13"
},
"peerDependencies": {
"d3": "~3.3.0"
},
"deprecated": {
"express":"^4.16.4"
},
"dist":{
"shasum":"f572d396fae9206628714fb2ce00f72e94f2258f"
}
}
},
"_attachments":{
"@root/npm-test-1.0.1.tgz":{
"content_type":"application/octet-stream",
"data":"aGVsbG8K",
"length":8
}
},
"id":"10",
"package_name":"@root/npm-test"
}
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::DependencyLink, type: :model do
describe 'relationships' do
it { is_expected.to belong_to(:package).inverse_of(:dependency_links) }
it { is_expected.to belong_to(:dependency).inverse_of(:dependency_links) }
end
describe 'validations' do
subject { create(:packages_dependency_link) }
it { is_expected.to validate_presence_of(:package) }
it { is_expected.to validate_presence_of(:dependency) }
context 'package_id and package_dependency_id uniqueness for dependency_type' do
it 'is not valid' do
exisiting_link = subject
link = build(
:packages_dependency_link,
package: exisiting_link.package,
dependency: exisiting_link.dependency,
dependency_type: exisiting_link.dependency_type
)
expect(link).not_to be_valid
expect(link.errors.to_a).to include("Dependency type has already been taken")
end
end
end
describe '.with_dependency_type' do
let_it_be(:link1) { create(:packages_dependency_link) }
let_it_be(:link2) { create(:packages_dependency_link, dependency: link1.dependency, dependency_type: :devDependencies) }
let_it_be(:link3) { create(:packages_dependency_link, dependency: link1.dependency, dependency_type: :bundleDependencies) }
subject { described_class }
it 'returns links of the given type' do
expect(subject.with_dependency_type(:bundleDependencies)).to eq([link3])
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Dependency, type: :model do
describe 'relationships' do
it { is_expected.to have_many(:dependency_links) }
end
describe 'validations' do
subject { create(:packages_dependency) }
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_presence_of(:version_pattern) }
it { is_expected.to validate_uniqueness_of(:name).scoped_to(:version_pattern) }
end
describe '.for_package_names_and_version_patterns' do
let_it_be(:package_dependency1) { create(:packages_dependency, name: 'foo', version_pattern: '~1.0.0') }
let_it_be(:package_dependency2) { create(:packages_dependency, name: 'bar', version_pattern: '~2.5.0') }
let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2) }
let(:chunk_size) { 50 }
let(:rows_limit) { 50 }
subject { Packages::Dependency.for_package_names_and_version_patterns(names_and_version_patterns, chunk_size, rows_limit) }
it { is_expected.to match_array([package_dependency1, package_dependency2]) }
context 'with unknown names' do
let(:names_and_version_patterns) { { unknown: '~1.0.0' } }
it { is_expected.to be_empty }
end
context 'with unknown version patterns' do
let(:names_and_version_patterns) { { 'foo' => '~1.0.0beta' } }
it { is_expected.to be_empty }
end
context 'with a name bigger than column size' do
let_it_be(:big_name) { 'a' * (Packages::Dependency::MAX_STRING_LENGTH + 1) }
let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2).merge(big_name => '~1.0.0') }
it { is_expected.to match_array([package_dependency1, package_dependency2]) }
end
context 'with a version pattern bigger than column size' do
let_it_be(:big_version_pattern) { 'a' * (Packages::Dependency::MAX_STRING_LENGTH + 1) }
let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2).merge('test' => big_version_pattern) }
it { is_expected.to match_array([package_dependency1, package_dependency2]) }
end
context 'with too big parameter' do
let(:size) { (Packages::Dependency::MAX_CHUNKED_QUERIES_COUNT * chunk_size) + 1 }
let(:names_and_version_patterns) { Hash[(1..size).map { |v| [v, v] }] }
it { expect { subject }.to raise_error(ArgumentError, 'Too many names_and_version_patterns') }
end
context 'with parameters size' do
let_it_be(:package_dependency3) { create(:packages_dependency, name: 'foo3', version_pattern: '~1.5.3') }
let_it_be(:package_dependency4) { create(:packages_dependency, name: 'foo4', version_pattern: '~1.5.4') }
let_it_be(:package_dependency5) { create(:packages_dependency, name: 'foo5', version_pattern: '~1.5.5') }
let_it_be(:package_dependency6) { create(:packages_dependency, name: 'foo6', version_pattern: '~1.5.6') }
let_it_be(:package_dependency7) { create(:packages_dependency, name: 'foo7', version_pattern: '~1.5.7') }
let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2, package_dependency3, package_dependency4, package_dependency5, package_dependency6, package_dependency7) }
context 'above the chunk size' do
let(:chunk_size) { 2 }
it { is_expected.to match_array([package_dependency1, package_dependency2, package_dependency3, package_dependency4, package_dependency5, package_dependency6, package_dependency7]) }
end
context 'selecting too many rows' do
let(:rows_limit) { 2 }
it { expect { subject }.to raise_error(ArgumentError, 'Too many Dependencies selected') }
end
end
def build_names_and_version_patterns(*package_dependencies)
result = Hash.new { |h, dependency| h[dependency.name] = dependency.version_pattern }
package_dependencies.each { |dependency| result[dependency] }
result
end
end
end
Loading
Loading
@@ -7,10 +7,16 @@ RSpec.describe Packages::Package, type: :model do
describe 'relationships' do
it { is_expected.to belong_to(:project) }
it { is_expected.to have_many(:package_files).dependent(:destroy) }
it { is_expected.to have_many(:dependency_links).inverse_of(:package) }
it { is_expected.to have_one(:conan_metadatum).inverse_of(:package) }
it { is_expected.to have_one(:maven_metadatum).inverse_of(:package) }
end
 
describe 'validations' do
subject { create(:package) }
it { is_expected.to validate_presence_of(:project) }
it { is_expected.to validate_uniqueness_of(:name).scoped_to(:project_id, :version, :package_type) }
 
describe '#name' do
it { is_expected.to allow_value("my/domain/com/my-app").for(:name) }
Loading
Loading
@@ -39,6 +45,18 @@ RSpec.describe Packages::Package, type: :model do
end
end
end
Packages::Package.package_types.keys.each do |pt|
context "project id, name, version and package type uniqueness for package type #{pt}" do
let(:package) { create("#{pt}_package") }
it "will not allow a #{pt} package with same project, name, version and package_type" do
new_package = build("#{pt}_package", project: package.project, name: package.name, version: package.version)
expect(new_package).not_to be_valid
expect(new_package.errors.to_a).to include("Name has already been taken")
end
end
end
end
 
describe '#destroy' do
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment