Skip to content

refactor: replace insecure custom downloader with Down::NetHttp #260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alexskr wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

refactor: replace insecure custom downloader with Down::NetHttp #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f88c4e
refactor: replace insecure custom downloader with Down::NetHttp
alexskr Oct 25, 2025
c55a8f6
Merge branch 'develop' into refactor/file-download-and-copy
alexskr Nov 14, 2025
10c4e51
Remove FTP support, refactor remote_file_exists? to FileHelpers, repl…
alexskr Nov 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions Gemfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ gem 'rake', '~> 10.0'
gem 'request_store'
gem 'rest-client'
gem 'rsolr'
gem 'thin', '~> 1.0' # compatibility version pin. thin should be replaced with webmoc
gem "down", "~> 5.0"

# Testing
group :test do
Expand All @@ -31,6 +29,7 @@ group :test do
gem 'rack-test', '~> 0.6'
gem 'simplecov'
gem 'simplecov-cobertura' # for codecov.io
gem 'webrick'
end

group :development do
Expand Down
83 changes: 36 additions & 47 deletions Gemfile.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
GIT
remote: https://github.com/ncbo/goo.git
revision: ca5f9d858eef89923903236fe6f76c78271e538d
revision: 1d6b8ace1b06f7cb80fd990baf1067b88caaa5a0
branch: develop
specs:
goo (0.0.2)
Expand Down Expand Up @@ -30,16 +30,15 @@ PATH
ontologies_linked_data (0.0.1)
activesupport
bcrypt
down (~> 5.0)
goo
json
libxml-ruby
multi_json
net-ftp
oj
omni_logger
pony
rack
rack-test
rsolr
rubyzip (~> 3.0)

Expand All @@ -57,16 +56,15 @@ GEM
ansi (1.5.0)
ast (2.4.3)
bcrypt (3.1.20)
bigdecimal (3.2.2)
bigdecimal (3.3.1)
builder (3.3.0)
childprocess (5.1.0)
logger (~> 1.5)
coderay (1.1.3)
concurrent-ruby (1.3.5)
connection_pool (2.5.3)
connection_pool (2.5.4)
cube-ruby (0.0.3)
daemons (1.4.1)
date (3.4.1)
date (3.5.0)
docile (1.4.1)
domain_name (0.6.20240107)
down (5.4.2)
Expand All @@ -75,24 +73,23 @@ GEM
htmlentities (~> 4.3.3)
launchy (>= 2.1, < 4.0)
mail (~> 2.7)
eventmachine (1.2.7)
faraday (2.13.4)
faraday (2.14.0)
faraday-net_http (>= 2.0, < 3.5)
json
logger
faraday-net_http (3.4.1)
net-http (>= 0.5.0)
faraday-net_http (3.4.2)
net-http (~> 0.5)
ffi (1.17.2-aarch64-linux-gnu)
ffi (1.17.2-arm64-darwin)
ffi (1.17.2-x86_64-linux-gnu)
hashie (5.0.0)
htmlentities (4.3.4)
http-accept (1.7.0)
http-cookie (1.0.8)
http-cookie (1.1.0)
domain_name (~> 0.5)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
json (2.13.2)
json (2.16.0)
json_pure (2.8.1)
language_server-protocol (3.17.0.5)
launchy (3.1.1)
Expand All @@ -104,7 +101,8 @@ GEM
logger (1.7.0)
macaddr (1.7.2)
systemu (~> 2.6.5)
mail (2.8.1)
mail (2.9.0)
logger
mini_mime (>= 0.1.1)
net-imap
net-pop
Expand All @@ -113,24 +111,21 @@ GEM
mime-types (3.7.0)
logger
mime-types-data (~> 3.2025, >= 3.2025.0507)
mime-types-data (3.2025.0826)
mime-types-data (3.2025.0924)
mini_mime (1.1.5)
minitest (4.7.5)
minitest-reporters (0.14.24)
ansi
builder
minitest (>= 2.12, < 5.0)
powerbar
mocha (2.7.1)
mocha (2.8.0)
ruby2_keywords (>= 0.0.5)
mock_redis (0.51.0)
mock_redis (0.53.0)
redis (~> 5)
multi_json (1.17.0)
net-ftp (0.3.8)
net-protocol
time
net-http (0.6.0)
uri
net-http (0.8.0)
uri (>= 0.11.1)
net-http-persistent (2.9.4)
net-imap (0.4.22)
date
Expand All @@ -142,27 +137,27 @@ GEM
net-smtp (0.5.1)
net-protocol
netrc (0.11.0)
oj (3.16.11)
oj (3.16.12)
bigdecimal (>= 3.0)
ostruct (>= 0.2)
omni_logger (0.1.4)
logger
ostruct (0.6.3)
parallel (1.27.0)
parser (3.3.9.0)
parser (3.3.10.0)
ast (~> 2.4.1)
racc
pony (1.13.1)
mail (>= 2.0)
powerbar (2.0.1)
hashie (>= 1.1.0)
prism (1.4.0)
prism (1.6.0)
pry (0.15.2)
coderay (~> 1.1)
method_source (~> 1.0)
public_suffix (5.1.1)
racc (1.8.1)
rack (2.2.17)
rack (2.2.21)
rack-test (0.8.3)
rack (>= 1.0, < 3)
rainbow (3.1.1)
Expand All @@ -171,62 +166,57 @@ GEM
addressable (>= 2.2)
redis (5.4.1)
redis-client (>= 0.22.0)
redis-client (0.25.2)
redis-client (0.26.1)
connection_pool
regexp_parser (2.11.2)
regexp_parser (2.11.3)
request_store (1.7.0)
rack (>= 1.4)
rest-client (2.1.0)
http-accept (>= 1.7.0, < 2.0)
http-cookie (>= 1.0.2, < 2.0)
mime-types (>= 1.16, < 4.0)
netrc (~> 0.8)
rexml (3.4.1)
rexml (3.4.4)
rsolr (2.6.0)
builder (>= 2.1.2)
faraday (>= 0.9, < 3, != 2.0.0)
rubocop (1.79.2)
rubocop (1.81.7)
json (~> 2.3)
language_server-protocol (~> 3.17.0.2)
lint_roller (~> 1.1.0)
parallel (~> 1.10)
parser (>= 3.3.0.2)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 2.9.3, < 3.0)
rubocop-ast (>= 1.46.0, < 2.0)
rubocop-ast (>= 1.47.1, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 4.0)
rubocop-ast (1.46.0)
rubocop-ast (1.48.0)
parser (>= 3.3.7.2)
prism (~> 1.4)
ruby-progressbar (1.13.0)
ruby2_keywords (0.0.5)
rubyzip (3.0.2)
rubyzip (3.2.2)
simplecov (0.22.0)
docile (~> 1.1)
simplecov-html (~> 0.11)
simplecov_json_formatter (~> 0.1)
simplecov-cobertura (3.0.0)
simplecov-cobertura (3.1.0)
rexml
simplecov (~> 0.19)
simplecov-html (0.13.2)
simplecov_json_formatter (0.1.4)
systemu (2.6.5)
thin (1.8.2)
daemons (~> 1.0, >= 1.0.9)
eventmachine (~> 1.0, >= 1.0.4)
rack (>= 1, < 3)
thread_safe (0.3.6)
time (0.4.1)
date
timeout (0.4.3)
timeout (0.4.4)
tzinfo (0.3.62)
unicode-display_width (3.1.5)
unicode-emoji (~> 4.0, >= 4.0.4)
unicode-emoji (4.0.4)
uri (1.0.3)
unicode-display_width (3.2.0)
unicode-emoji (~> 4.1)
unicode-emoji (4.1.0)
uri (1.1.1)
uuid (2.3.9)
macaddr (~> 1.0)
webrick (1.9.1)

PLATFORMS
aarch64-linux
Expand All @@ -240,7 +230,6 @@ DEPENDENCIES
addressable (~> 2.8)
bcrypt (~> 3.0)
cube-ruby
down (~> 5.0)
email_spec
ffi
goo!
Expand All @@ -267,7 +256,7 @@ DEPENDENCIES
simplecov
simplecov-cobertura
sparql-client!
thin (~> 1.0)
webrick

BUNDLED WITH
2.6.3
81 changes: 18 additions & 63 deletions lib/ontologies_linked_data/models/ontology_submission.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
require 'net/ftp'
require 'net/http'
require 'uri'
require 'open-uri'
Expand Down Expand Up @@ -282,37 +281,42 @@ def self.submission_id_generator(ss)
end

def self.copy_file_repository(acronym, submission_id, src, filename = nil)
path_to_repo = File.join(
LinkedData.settings.repository_folder,
acronym.to_s,
submission_id.to_s
)
repo_root = LinkedData.settings.repository_folder
dst_dir = File.join(repo_root.to_s, acronym.to_s, submission_id.to_s)

src_path = src.respond_to?(:path) ? src.path.to_s : src.to_s
raise ArgumentError, "Source file does not exist: #{src_path}" unless ::File.exist?(src_path)

name = filename || File.basename(src)
dst = File.join(path_to_repo, name)
name = LinkedData::Utils::FileHelpers.sanitize_filename(name)

dst_final = File.join(dst_dir, name)
dst_tmp = "#{dst_final}.tmp-#{Process.pid}-#{rand(1_000_000)}"

begin
FileUtils.mkdir_p(path_to_repo)
FileUtils.chmod(REPOSITORY_DIR_MODE, path_to_repo)
FileUtils.mkdir_p(dst_dir)
FileUtils.chmod(REPOSITORY_DIR_MODE, dst_dir)

FileUtils.copy(src, dst)
FileUtils.copy(src, dst_tmp)
# Uploaded files are initially written to a Tempfile in tmpdir with
# permissions 0600 (owner read/write only) for security. To ensure
# repository files are also accessible by the service group as intended,
# we explicitly chmod the destination file to REPOSITORY_FILE_MODE.
FileUtils.chmod(REPOSITORY_FILE_MODE, dst)
FileUtils.chmod(REPOSITORY_FILE_MODE, dst_tmp)
FileUtils.mv(dst_tmp, dst_final)
rescue StandardError => e
FileUtils.rm_f(dst_tmp)
raise e.class, "Failed to copy #{src} to #{dst}: #{e.message}", e.backtrace
end

# Sanity check: ensure the file actually exists after copy and chmod
# This guards against rare cases like silent file storage failures or
# race conditions
unless File.exist?(dst)
unless File.exist?(dst_final)
raise IOError, "Copy operation completed without error, but file '#{dst}' does not exist"
end

dst
dst_final
end

def valid?
Expand Down Expand Up @@ -808,18 +812,6 @@ def ontology_uri
RDF::URI.new(self.uri)
end













def roots_sorted(extra_include=nil)
classes = roots(extra_include)
LinkedData::Models::Class.sort_classes(classes)
Expand All @@ -833,17 +825,7 @@ def download_and_store_ontology_file
end

def remote_file_exists?(url)
begin
url = URI.parse(url)
if url.kind_of?(URI::FTP)
check = check_ftp_file(url)
else
check = check_http_file(url)
end
rescue Exception
check = false
end
check
LinkedData::Utils::FileHelpers.remote_file_exists?(url)
end

def download_ontology_file
Expand Down Expand Up @@ -907,33 +889,6 @@ def owlapi_parser_input
File.expand_path(path)
end

def check_http_file(url)
session = Net::HTTP.new(url.host, url.port)
session.use_ssl = true if url.port == 443
session.start do |http|
response_valid = http.head(url.request_uri).code.to_i < 400
return response_valid
end
end

def check_ftp_file(uri)
ftp = Net::FTP.new(uri.host, uri.user, uri.password)
ftp.login
begin
file_exists = ftp.size(uri.path) > 0
rescue Exception => e
# Check using another method
path = uri.path.split("/")
filename = path.pop
path = path.join("/")
ftp.chdir(path)
files = ftp.dir
# Dumb check, just see if the filename is somewhere in the list
files.each { |file| return true if file.include?(filename) }
end
file_exists
end

def self.loom_transform_literal(lit)
res = []
lit.each_char do |c|
Expand Down
Loading