From 088f8c96272999e3889a713c25f01ff4d2fa70e9 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Fri, 14 Jun 2024 17:08:11 -0400 Subject: [PATCH 1/5] use GraphQL to check snapshot --- openneuro/app/models/open_neuro.rb | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 17974959..d3b32472 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -40,6 +40,8 @@ class OpenNeuro DATALAD_REPO_URL_PREFIX = 'https://github.com/OpenNeuroDatasets' GITHUB_VALIDATION_URL = 'https://api.github.com/repos/OpenNeuroDatasets/:name/git/ref/tags/:version' + OPENNEURO_API_URL = 'https://openneuro.org/crn/graphql' + # Creates an OpenNeuro object that represents # the dataset internally as a pair, a WorkGroup @@ -332,6 +334,48 @@ def self.data_provider_name_builder(name, version) def self.valid_name_and_version?(name, version) return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ + query = '{snapshot(datasetId:"%s", tag:"%s"){id}}' % [name, version] + + response = Typhoeus.post(OPENNEURO_API_URL, + {:body => JSON.pretty_generate({"query" => query}) + }, + :headers => { :Accept => 'application/json' } + ) + + + # Parse the response + body = response.response_body + json = JSON.parse(body) + return ! json["errors"] + rescue => ex + Rails.logger.error "OpenNeuro API request failed: #{ex.class} #{ex.message}" + return nil + end + + + # Validation of a pair [ dataset, tag ] on github performed with: + # + # curl -H "Accept: application/vnd.github+json" + # -H "X-GitHub-Api-Version: 2022-11-28" + # "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/ref/tags/2.4.0" + # + # The typical response is like this: + # + # { + # "ref": "refs/tags/2.4.0", + # "node_id": "REF_kwDOK8fpRq9yZWZzL3RhZ3MvMi40LjA", + # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/refs/tags/2.4.0", + # "object": { + # "sha": "1aa6d3a098d16009d39adde6a7abe1c34d4b07d6", + # "type": "commit", + # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/commits/1aa6d3a098d16009d39adde6a7abe1c34d4b07d6" + # } + # } + # + # The method just returns true or false. + def self.valid_name_and_github_tag?(name, version) + return false unless name =~ /\Ads\d+\z/ + return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ validation_url = GITHUB_VALIDATION_URL .sub(':name', name ) From a97172f393aadbe0907a6530202ce422bc8a0f37 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Fri, 14 Jun 2024 18:04:44 -0400 Subject: [PATCH 2/5] use GraphQL to check OpenNeuro snapshot - add headers --- openneuro/app/models/open_neuro.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index d3b32472..04a1b442 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -332,27 +332,26 @@ def self.data_provider_name_builder(name, version) # # The method just returns true or false. def self.valid_name_and_version?(name, version) - return false unless name =~ /\Ads\d+\z/ + return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ query = '{snapshot(datasetId:"%s", tag:"%s"){id}}' % [name, version] response = Typhoeus.post(OPENNEURO_API_URL, - {:body => JSON.pretty_generate({"query" => query}) - }, - :headers => { :Accept => 'application/json' } + :body => JSON.pretty_generate({ "query" => query }), + :headers => { 'Content-Type' => 'application/json', + 'Accept' => 'application/json' + } ) - # Parse the response - body = response.response_body - json = JSON.parse(body) - return ! json["errors"] + body = response.response_body + json = JSON.parse(body) + return !json["errors"] rescue => ex Rails.logger.error "OpenNeuro API request failed: #{ex.class} #{ex.message}" return nil end - # Validation of a pair [ dataset, tag ] on github performed with: # # curl -H "Accept: application/vnd.github+json" From b5f94e2a56c2fd5e9c1a5fd567b1d545ac7efcfb Mon Sep 17 00:00:00 2001 From: Serge Date: Sun, 16 Jun 2024 08:59:23 -0400 Subject: [PATCH 3/5] Update open_neuro.rb copyright year --- openneuro/app/models/open_neuro.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 04a1b442..47cb475b 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -2,7 +2,7 @@ # # CBRAIN Project # -# Copyright (C) 2008-2023 +# Copyright (C) 2008-2024 # The Royal Institution for the Advancement of Learning # McGill University # @@ -42,7 +42,6 @@ class OpenNeuro GITHUB_VALIDATION_URL = 'https://api.github.com/repos/OpenNeuroDatasets/:name/git/ref/tags/:version' OPENNEURO_API_URL = 'https://openneuro.org/crn/graphql' - # Creates an OpenNeuro object that represents # the dataset internally as a pair, a WorkGroup # and a DataladDataProvider. The naming of these From 8f3afa090d7f9d670dc375c184b38d0aad641c22 Mon Sep 17 00:00:00 2001 From: Serge Date: Mon, 17 Jun 2024 16:46:08 -0400 Subject: [PATCH 4/5] change comments for name and version validation method --- openneuro/app/models/open_neuro.rb | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 47cb475b..342866ae 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -310,26 +310,15 @@ def self.data_provider_name_builder(name, version) "OpenNeuro-#{name}-#{version.gsub('.','_')}" end - # Validation of a pair [ dataset, version ] performed with: + # Validation of a pair [ dataset, version ] with OpenNeuro GraphQL API + # + # The result is sometimes different + # "https://api.github.com/repos/OpenNeuroDatasets/ # - # curl -H "Accept: application/vnd.github+json" - # -H "X-GitHub-Api-Version: 2022-11-28" - # "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/ref/tags/2.4.0" - # - # The typical response is like this: - # - # { - # "ref": "refs/tags/2.4.0", - # "node_id": "REF_kwDOK8fpRq9yZWZzL3RhZ3MvMi40LjA", - # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/refs/tags/2.4.0", - # "object": { - # "sha": "1aa6d3a098d16009d39adde6a7abe1c34d4b07d6", - # "type": "commit", - # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/commits/1aa6d3a098d16009d39adde6a7abe1c34d4b07d6" - # } - # } - # - # The method just returns true or false. + # Some datasets or dataset versions, availabe on OpenNeuro portal or with OpenNeuro client are not always + # on GitHub + # + # At the moment the method just returns true or false. def self.valid_name_and_version?(name, version) return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ From 5443756b701cae521a1f35cb7c56e447615e1ff4 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Mon, 17 Jun 2024 16:35:39 -0400 Subject: [PATCH 5/5] Allow alternative GitHub repos for OpenNeuro --- openneuro/app/models/open_neuro.rb | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 342866ae..d63fc14f 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -38,9 +38,10 @@ class OpenNeuro DATA_PROVIDER_OWNER = User.admin USERFILES_OWNER = User.admin - DATALAD_REPO_URL_PREFIX = 'https://github.com/OpenNeuroDatasets' - GITHUB_VALIDATION_URL = 'https://api.github.com/repos/OpenNeuroDatasets/:name/git/ref/tags/:version' - OPENNEURO_API_URL = 'https://openneuro.org/crn/graphql' + DATALAD_REPO_URL_PREFIX = 'https://github.com/OpenNeuroDatasets' + DATALAD_ALT_URL_PREFIXES = ['https://github.com/MontrealSergiy', 'https://github.com/aces'] + GITHUB_VALIDATION_URL = 'https://api.github.com/repos/OpenNeuroDatasets/:name/git/ref/tags/:version' + OPENNEURO_API_URL = 'https://openneuro.org/crn/graphql' # Creates an OpenNeuro object that represents # the dataset internally as a pair, a WorkGroup @@ -58,7 +59,7 @@ def self.find(name, version) groupquery = work_group_builder(name, version) group = groupquery.first dpquery = data_provider_builder(name, version, group&.id) # group.id can be nil - dp = dpquery.first + dp = data_provider_builder(name, version, group&.id, alt_urls = true).first self.new.tap do |open_neuro| open_neuro.name = name @@ -283,12 +284,14 @@ def self.work_group_builder(name, version) # Returns a DataladDataProvider fetcher or constructor # representing an OpenNeuro dataset. (ActiveRecord query) - def self.data_provider_builder(name, version, group_id) + def self.data_provider_builder(name, version, group_id, alt_urls=false) + urls = "#{DATALAD_REPO_URL_PREFIX}/#{name}" + urls = DATALAD_ALT_URL_PREFIXES.map {|p| "#{p}/#{name}"}.unshift urls if alt_urls DataladDataProvider.where( :name => data_provider_name_builder(name, version), :user_id => DATA_PROVIDER_OWNER.id, :group_id => group_id, - :datalad_repository_url => "#{DATALAD_REPO_URL_PREFIX}/#{name}", + :datalad_repository_url => urls, :containerized_path => version, :online => true, )