From 9768ad4c879c80ca5b945214e282e25e7353d1e8 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 23 Feb 2012 16:14:09 -0500 Subject: [PATCH 01/20] created a Node object that will represent a FilePaths node --- lib/MogileFS/Plugin/FilePaths.pm | 22 ++++++++++++ lib/MogileFS/Plugin/FilePaths/Node.pm | 48 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 lib/MogileFS/Plugin/FilePaths/Node.pm diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 8a570bd..6de09cc 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -17,6 +17,7 @@ $VERSION = eval $VERSION; use MogileFS::FID; use MogileFS::Worker::Query; use MogileFS::Plugin::MetaData; +use MogileFS::Plugin::FilePaths::Node; sub _parse_path { my $fullpath = shift; @@ -533,6 +534,27 @@ sub plugin_filepaths_delete_node { return 1; } +#returns a hash ref containing +sub plugin_filepaths_node_row_from_nodeid { + my ($self, $nodeid) = @_; + return $self->dbh->selectrow_hashref("SELECT nodeid, dmid, parentnodeid, nodename, fid ". + "FROM plugin_filepaths_paths WHERE nodeid=?", + undef, $nodeid); +} + +# return the node for the specified node +sub plugin_filepaths_get_node_by_parent { + my $self = shift; + my ($dmid, $parentnodeid, $nodename) = @_; + my $dbh = $self->dbh; + my $row = $dbh->selectrow_hashref('SELECT nodeid, dmid, parentnodeid, nodename, fid ' . + 'FROM plugin_filepaths_paths ' . + 'WHERE dmid = ? AND parentnodeid = ? AND nodename = ?', + undef, $dmid, $parentnodeid, $nodename); + return undef if !$row || $dbh->err; + return MogileFS::Plugin::FilePaths::Node->new_from_db_row($row); +} + # return the nodeid for the specified node sub plugin_filepaths_get_nodeid { my $self = shift; diff --git a/lib/MogileFS/Plugin/FilePaths/Node.pm b/lib/MogileFS/Plugin/FilePaths/Node.pm new file mode 100644 index 0000000..0fe6f55 --- /dev/null +++ b/lib/MogileFS/Plugin/FilePaths/Node.pm @@ -0,0 +1,48 @@ +package MogileFS::Plugin::FilePaths::Node; + +use strict; +use warnings; +use Carp qw(croak); + +sub new { + my ($class, $nodeid) = @_; + croak("Invalid nodeid") unless $nodeid; + return bless { + nodeid => $nodeid, + nodename => undef, + dmid => undef, + parentnodeid => undef, + fid => undef, + _loaded => 0, + }, $class; +} + +# mutates/blesses given row. +sub new_from_db_row { + my ($class, $row) = @_; + die "Missing 'nodeid' column" if(!$row->{nodeid}); + $row->{_loaded} = 1; + return bless $row, $class; +} + +sub id { $_[0]{nodeid} } + +# force loading, or die. +sub _load { + return 1 if $_[0]{_loaded}; + my $self = shift; + croak('NODE#'.$_[0]->id.' doesn\'t exist') unless $self->_tryload; +} + +# return 1 if loaded, or 0 if not exist +sub _tryload { + return 1 if $_[0]{_loaded}; + my $self = shift; + my $row = Mgd::get_store()->plugin_filepaths_node_row_from_nodeid($self->{nodeid}) + or return 0; + $self->{$_} = $row->{$_} foreach qw(nodename dmid parentnodeid fid); + $self->{_loaded} = 1; + return 1; +} + +1; From ca2eed4a6bd5707894865186511b44819fed0304 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 23 Feb 2012 16:34:57 -0500 Subject: [PATCH 02/20] updated _traverse_path to return a Node object --- lib/MogileFS/Plugin/FilePaths.pm | 44 +++++++++++++-------------- lib/MogileFS/Plugin/FilePaths/Node.pm | 9 +++++- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 6de09cc..a5c3dc8 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -294,7 +294,8 @@ sub unload { sub vivify_path { my ($dmid, $path) = @_; return undef unless $dmid && $path; - return _traverse_path($dmid, $path, 1); + my $node = _traverse_path($dmid, $path, 1); + return $node ? $node->id : undef; } # called to load the nodeid of the final element in a path, which is useful for finding @@ -302,7 +303,8 @@ sub vivify_path { sub load_path { my ($dmid, $path) = @_; return undef unless $dmid && $path; - return _traverse_path($dmid, $path, 0); + my $node = _traverse_path($dmid, $path, 0); + return $node ? $node->id : undef; } # does the internal work of traversing a path @@ -310,42 +312,40 @@ sub _traverse_path { my ($dmid, $path, $vivify) = @_; return undef unless $dmid && $path; - my @paths = grep { $_ } split /\//, $path; - return 0 unless @paths; #toplevel + # start with the root path node + my $node = MogileFS::Plugin::FilePaths::Node->new(0); - my $parentnodeid = 0; - foreach my $node (@paths) { - # try to get the id for this node - my $nodeid = _find_node($dmid, $parentnodeid, $node, $vivify); - return undef unless $nodeid; - - # this becomes the new parent - $parentnodeid = $nodeid; + # recurse the specified path + foreach my $part (grep { $_ } split /\//, $path) { + # look for the current path part + $node = _find_node($dmid, $node->id, $part, $vivify); + return undef unless $node; } - # we're done, so the parentnodeid is what we return - return $parentnodeid; + # we're done, so return the most recent node + return $node; } # checks to see if a node exists, and if not, creates it if $vivify is set sub _find_node { - my ($dmid, $parentnodeid, $node, $vivify) = @_; - return undef unless $dmid && defined $parentnodeid && $node; + my ($dmid, $parentnodeid, $name, $vivify) = @_; + return undef unless $dmid && defined $parentnodeid && $name; my $sto = Mgd::get_store(); - my $nodeid = $sto->plugin_filepaths_get_nodeid($dmid, $parentnodeid, $node); - return $nodeid if $nodeid; + my $node = $sto->plugin_filepaths_get_node_by_parent($dmid, $parentnodeid, $name); + return $node if $node; if ($vivify) { - $nodeid = $sto->plugin_filepaths_add_node( + my $nodeid = $sto->plugin_filepaths_add_node( 'dmid' => $dmid, 'parentnodeid' => $parentnodeid, - 'nodename' => $node, + 'nodename' => $name, ); + + return MogileFS::Plugin::FilePaths::Node->new($nodeid) if $nodeid && $nodeid > 0; } - return undef unless $nodeid && $nodeid > 0; - return $nodeid; + return undef; } # sets the mapping of a file from a name to a fid diff --git a/lib/MogileFS/Plugin/FilePaths/Node.pm b/lib/MogileFS/Plugin/FilePaths/Node.pm index 0fe6f55..09bc4f8 100644 --- a/lib/MogileFS/Plugin/FilePaths/Node.pm +++ b/lib/MogileFS/Plugin/FilePaths/Node.pm @@ -6,7 +6,7 @@ use Carp qw(croak); sub new { my ($class, $nodeid) = @_; - croak("Invalid nodeid") unless $nodeid; + croak("Invalid nodeid") unless defined($nodeid); return bless { nodeid => $nodeid, nodename => undef, @@ -38,6 +38,13 @@ sub _load { sub _tryload { return 1 if $_[0]{_loaded}; my $self = shift; + + # special case for root directory node + if($self->{nodeid} == 0) { + $self->{_loaded} = 1; + return 1; + } + my $row = Mgd::get_store()->plugin_filepaths_node_row_from_nodeid($self->{nodeid}) or return 0; $self->{$_} = $row->{$_} foreach qw(nodename dmid parentnodeid fid); From 36387d2e67487d083c2a605b329da99d2e375b8a Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 23 Feb 2012 17:32:23 -0500 Subject: [PATCH 03/20] updated cmd_delete hook to use the node object --- lib/MogileFS/Plugin/FilePaths.pm | 10 ++++++---- lib/MogileFS/Plugin/FilePaths/Node.pm | 12 ++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index a5c3dc8..13910dd 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -122,15 +122,17 @@ sub load { # get the fid of the file, bail out if it doesn't have one (directory nodes) my $sto = Mgd::get_store(); - my $fid = $sto->plugin_filepaths_get_fid_by_mapping($args->{dmid}, $parentnodeid, $filename); - return 0 unless $fid; + my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnodeid, $filename); + return 0 unless $node; + my $fidid = $node->fidid; + return 0 unless $fidid; # great, delete this file - delete_file_mapping( $args->{dmid}, $parentnodeid, $filename ); + $sto->plugin_filepaths_delete_node($node->id); # FIXME What should happen if this delete fails? # now pretend they asked for it and continue - $args->{key} = 'fid:' . $fid->id; + $args->{key} = 'fid:' . $fidid; }); MogileFS::register_worker_command( 'filepaths_enable', sub { diff --git a/lib/MogileFS/Plugin/FilePaths/Node.pm b/lib/MogileFS/Plugin/FilePaths/Node.pm index 09bc4f8..cdba0ba 100644 --- a/lib/MogileFS/Plugin/FilePaths/Node.pm +++ b/lib/MogileFS/Plugin/FilePaths/Node.pm @@ -27,6 +27,12 @@ sub new_from_db_row { sub id { $_[0]{nodeid} } +sub fidid { + my $self = shift; + $self->_load; + return $self->{fid}; +} + # force loading, or die. sub _load { return 1 if $_[0]{_loaded}; @@ -52,4 +58,10 @@ sub _tryload { return 1; } +sub fid { + my $fidid = $_[0]->fidid; + return MogileFS::FID->new($fidid) if($fidid); + return undef; +} + 1; From d420188908d35c0f1432d1470598abdbed0ed173 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 23 Feb 2012 17:33:25 -0500 Subject: [PATCH 04/20] removed unused delete_file_mapping function --- lib/MogileFS/Plugin/FilePaths.pm | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 13910dd..1a6c2ee 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -371,16 +371,6 @@ sub set_file_mapping { return $nodeid; } -sub delete_file_mapping { - my ($dmid, $parentnodeid, $filename) = @_; - return undef unless $dmid && defined $parentnodeid && $filename; - - my $sto = Mgd::get_store(); - my $nodeid = $sto->plugin_filepaths_get_nodeid($dmid, $parentnodeid, $filename); - return undef unless $nodeid; - return $sto->plugin_filepaths_delete_node($nodeid); -} - # generic sub that converts a file path to a key name that # MogileFS will understand sub _path_to_key { From 7fabdd8a49ad03cedf89db009162a835f104869a Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 23 Feb 2012 17:37:47 -0500 Subject: [PATCH 05/20] get rid of code duplication --- lib/MogileFS/Plugin/FilePaths.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 1a6c2ee..b87b7e8 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -380,8 +380,7 @@ sub _path_to_key { return 1 unless _check_dmid($dmid); # ensure we got a valid seeming path and filename - my ($path, $filename) = - ($args->{key} =~ m!^(/(?:[\w\-\.]+/)*)([\w\-\.]+)$!) ? ($1, $2) : (undef, undef); + my ($path, $filename) = _parse_path($args->{key}); return 0 unless $path && $filename; # now try to get the end of the path From f599efdb0942e1b7fea6b6b27e6f98d86e6390d4 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 23 Feb 2012 17:55:23 -0500 Subject: [PATCH 06/20] reworked the file_stored hook to use less db queries and be safer about deleting the old FID if something fails --- lib/MogileFS/Plugin/FilePaths.pm | 50 +++++++++++++------------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index b87b7e8..ee2238a 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -75,19 +75,30 @@ sub load { my $parentnodeid = MogileFS::Plugin::FilePaths::vivify_path( $args->{dmid}, $path ); return 0 unless defined $parentnodeid; - # see if this file exists already + # find/create a node to store this file at, track the old FID in order + # to delete it after updating the node my $sto = Mgd::get_store(); - my $oldfid = $sto->plugin_filepaths_get_fid_by_mapping($args->{dmid}, $parentnodeid, $filename); + my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnodeid, $filename); + my $oldfid = $node ? $node->fid : undef; + if($node) { + $sto->plugin_filepaths_update_node($node->id, {'fid' => $args->{fid}}); + } else { + my $nodeid = $sto->plugin_filepaths_add_node( + 'dmid' => $args->{dmid}, + 'parentnodeid' => $parentnodeid, + 'nodename' => $filename, + 'fid' => $args->{fid}, + ); + $node = MogileFS::Plugin::FilePaths::Node->new($nodeid); + } + return 0 unless $node; + + # delete the old FID now that the new FID has been stored if ($oldfid) { $oldfid->delete; } - my $fid = $args->{fid}; - - # and now, setup the mapping - my $nodeid = MogileFS::Plugin::FilePaths::set_file_mapping( $args->{dmid}, $parentnodeid, $filename, $fid ); - return 0 unless $nodeid; - + # store metadata if (my $keys = $args->{"plugin.meta.keys"}) { my %metadata; for (my $i = 0; $i < $keys; $i++) { @@ -96,7 +107,7 @@ sub load { $metadata{$key} = $value; } - MogileFS::Plugin::MetaData::set_metadata($fid, \%metadata); + MogileFS::Plugin::MetaData::set_metadata($args->{fid}, \%metadata); } # we're successful, let's keep the file @@ -350,27 +361,6 @@ sub _find_node { return undef; } -# sets the mapping of a file from a name to a fid -sub set_file_mapping { - my ($dmid, $parentnodeid, $filename, $fid) = @_; - return undef unless $dmid && defined $parentnodeid && $filename && $fid; - - my $sto = Mgd::get_store(); - my $nodeid = $sto->plugin_filepaths_get_nodeid($dmid, $parentnodeid, $filename); - - unless ($nodeid) { - $nodeid = $sto->plugin_filepaths_add_node( - 'dmid' => $dmid, - 'parentnodeid' => $parentnodeid, - 'nodename' => $filename, - 'fid' => $fid, - ); - } else { - $sto->plugin_filepaths_update_node($nodeid, {'fid' => $fid}); - } - return $nodeid; -} - # generic sub that converts a file path to a key name that # MogileFS will understand sub _path_to_key { From fa55a7e63b96ca2dbad097f00fb4c49830ff82a1 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 23 Feb 2012 18:04:03 -0500 Subject: [PATCH 07/20] update the filepaths_rename command to use the get_node_by_parent method and not vivify the path if it can't find the source path --- lib/MogileFS/Plugin/FilePaths.pm | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index ee2238a..e8b0b91 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -268,12 +268,16 @@ sub load { # LOCK rename + # find the node being renamed my $old_parentid = load_path($dmid, $old_path); - my $new_parentid = vivify_path($dmid, $new_path); - my $sto = Mgd::get_store(); - my $nodeid = $sto->plugin_filepaths_get_nodeid($dmid, $old_parentid, $old_name); - my $rv = $sto->plugin_filepaths_update_node($nodeid, { + my $node = $sto->plugin_filepaths_get_node_by_parent($dmid, $old_parentid, $old_name); + return $self->err_line('path_not_found', 'Path provided was not found in database') + unless $node; + + # now vivify the destination path and rename the file + my $new_parentid = vivify_path($dmid, $new_path); + my $rv = $sto->plugin_filepaths_update_node($node->id, { 'parentnodeid' => $new_parentid, 'nodename' => $new_name, }); From c86867df3df1b5cf088c37ecc2a91f4647aab86c Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 13:14:15 -0500 Subject: [PATCH 08/20] make vivify_path return the node at the end of the path --- lib/MogileFS/Plugin/FilePaths.pm | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index e8b0b91..ef20886 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -72,20 +72,20 @@ sub load { return 0 unless defined($path) && length($path) && defined($filename) && length($filename); # great, let's vivify that path and get the node to it - my $parentnodeid = MogileFS::Plugin::FilePaths::vivify_path( $args->{dmid}, $path ); - return 0 unless defined $parentnodeid; + my $parentnode = vivify_path( $args->{dmid}, $path ); + return 0 unless defined $parentnode; # find/create a node to store this file at, track the old FID in order # to delete it after updating the node my $sto = Mgd::get_store(); - my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnodeid, $filename); + my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnode->id, $filename); my $oldfid = $node ? $node->fid : undef; if($node) { $sto->plugin_filepaths_update_node($node->id, {'fid' => $args->{fid}}); } else { my $nodeid = $sto->plugin_filepaths_add_node( 'dmid' => $args->{dmid}, - 'parentnodeid' => $parentnodeid, + 'parentnodeid' => $parentnode->id, 'nodename' => $filename, 'fid' => $args->{fid}, ); @@ -276,9 +276,10 @@ sub load { unless $node; # now vivify the destination path and rename the file - my $new_parentid = vivify_path($dmid, $new_path); + my $new_parent = vivify_path($dmid, $new_path); + return $self->err_line("rename_failed") unless $new_parent; my $rv = $sto->plugin_filepaths_update_node($node->id, { - 'parentnodeid' => $new_parentid, + 'parentnodeid' => $new_parent->id, 'nodename' => $new_name, }); @@ -311,8 +312,7 @@ sub unload { sub vivify_path { my ($dmid, $path) = @_; return undef unless $dmid && $path; - my $node = _traverse_path($dmid, $path, 1); - return $node ? $node->id : undef; + return _traverse_path($dmid, $path, 1); } # called to load the nodeid of the final element in a path, which is useful for finding From 3ecfe7993efe3a0f5c26a49f590e88f4a3492bba Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 13:44:42 -0500 Subject: [PATCH 09/20] rewrite the list_directory command to use less db queries --- lib/MogileFS/Plugin/FilePaths.pm | 78 +++++++++++++++++++++++---- lib/MogileFS/Plugin/FilePaths/Node.pm | 14 +++++ 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index ef20886..9080d5b 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -200,7 +200,7 @@ sub load { unless $args->{argcount} == 1 && $path && $path =~ /^\//; # now find the id of the path - my $nodeid = MogileFS::Plugin::FilePaths::load_path( $dmid, $path ); + my $nodeid = load_path( $dmid, $path ); return $self->err_line('path_not_found', 'Path provided was not found in database') unless defined $nodeid; @@ -214,25 +214,45 @@ sub load { my %res; my $ct = 0; my $sto = Mgd::get_store(); - my @nodes = $sto->plugin_filepaths_get_nodes_by_mapping($dmid, $nodeid); + my @nodes = $sto->plugin_filepaths_get_nodes_by_parent($dmid, $nodeid); - my $node_count = $res{'files'} = scalar @nodes; + # get FIDs for all the found nodes + my %fids = ( + map {($_->id, $_)} + $sto->plugin_filepaths_load_fids(map {$_->fidid} @nodes) + ); + + # add all nodes to the response + my $i = 0; + foreach my $node (@nodes) { + next if(!$node); - for(my $i = 0; $i < $node_count; $i++) { - my ($nodename, $fidid) = @{$nodes[$i]}; my $prefix = "file$i"; - $res{$prefix} = $nodename; + $res{$prefix} = $node->nodename; + + # This node is a directory + if ($node->is_directory) { + $res{"$prefix.type"} = "D"; + } + # This file is a regular file + elsif(my $fid = $fids{$node->fidid} || $node->fid) { + # skip this node unless the fid exists + next unless $fid->exists; - if ($fidid) { # This file is a regular file $res{"$prefix.type"} = "F"; - my $length = MogileFS::FID->new($fidid)->length; + my $length = $fid->length; $res{"$prefix.size"} = $length if defined($length); - my $metadata = MogileFS::Plugin::MetaData::get_metadata($fidid); + my $metadata = MogileFS::Plugin::MetaData::get_metadata($fid->id); $res{"$prefix.mtime"} = $metadata->{mtime} if $metadata->{mtime}; - } else { # This file is a directory - $res{"$prefix.type"} = "D"; } + # invalid node, don't include it in the response + else { + next; + } + + $i++; } + $res{'files'} = $i; return $self->ok_line( \%res ); }); @@ -540,6 +560,24 @@ sub plugin_filepaths_get_node_by_parent { return MogileFS::Plugin::FilePaths::Node->new_from_db_row($row); } +# get all the nodes that are child nodes of the specified parent node +sub plugin_filepaths_get_nodes_by_parent { + my $self = shift; + my ($dmid, $parentnodeid) = @_; + my $dbh = $self->dbh; + my $sth = $dbh->prepare('SELECT nodeid, dmid, parentnodeid, nodename, fid ' . + 'FROM plugin_filepaths_paths ' . + 'WHERE dmid = ? AND parentnodeid = ?'); + $sth->execute($dmid, $parentnodeid); + + my @nodes; + while (my $row = $sth->fetchrow_hashref()) { + push @nodes, MogileFS::Plugin::FilePaths::Node->new_from_db_row($row); + } + + return @nodes; +} + # return the nodeid for the specified node sub plugin_filepaths_get_nodeid { my $self = shift; @@ -569,6 +607,24 @@ sub plugin_filepaths_get_nodes_by_mapping { return @nodes; } +# load the specified fids from the database +sub plugin_filepaths_load_fids { + my $self = shift; + my @fids = @_; + return if(!@fids); + + my @ret; + my $dbh = $self->dbh; + my $sth = $dbh->prepare("SELECT fid, dmid, dkey, length, classid, devcount ". + "FROM file ". + "WHERE fid IN (". join(',', (('?') x scalar @fids)) . ")"); + $sth->execute(@fids); + while (my $row = $sth->fetchrow_hashref) { + push @ret, MogileFS::FID->new_from_db_row($row); + } + return @ret; +} + # takes a domain, parentnodeid, and filename and returns a MogileFS::FID object # for the found file sub plugin_filepaths_get_fid_by_mapping { diff --git a/lib/MogileFS/Plugin/FilePaths/Node.pm b/lib/MogileFS/Plugin/FilePaths/Node.pm index cdba0ba..68ac52c 100644 --- a/lib/MogileFS/Plugin/FilePaths/Node.pm +++ b/lib/MogileFS/Plugin/FilePaths/Node.pm @@ -27,6 +27,12 @@ sub new_from_db_row { sub id { $_[0]{nodeid} } +sub nodename { + my $self = shift; + $self->_load; + return $self->{nodename}; +} + sub fidid { my $self = shift; $self->_load; @@ -64,4 +70,12 @@ sub fid { return undef; } +sub is_directory { + return !($_[0]->is_file); +} + +sub is_file { + return defined($_[0]->fidid); +} + 1; From 8122cc4d1e528e19175e82f62239e5c7ac666efe Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 16:05:47 -0500 Subject: [PATCH 10/20] updated _path_to_key to utilize the node object --- lib/MogileFS/Plugin/FilePaths.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 9080d5b..25dbb05 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -403,11 +403,12 @@ sub _path_to_key { # great, find this file my $sto = Mgd::get_store(); - my $fid = $sto->plugin_filepaths_get_fid_by_mapping($dmid, $parentnodeid, $filename); - return 0 unless $fid; + my $node = $sto->plugin_filepaths_get_node_by_parent($dmid, $parentnodeid, $filename); + my $fidid = $node->fidid; + return 0 unless $fidid; # now pretend they asked for it and continue - $args->{key} = 'fid:' . $fid->id; + $args->{key} = 'fid:' . $fidid; return 1; } From 03123a7549ab73b87f7baa57e2eb4e94cd84cc61 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 16:07:11 -0500 Subject: [PATCH 11/20] remove a few unused MogileFS::Store methods --- lib/MogileFS/Plugin/FilePaths.pm | 43 -------------------------------- 1 file changed, 43 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 25dbb05..5a16f69 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -579,35 +579,6 @@ sub plugin_filepaths_get_nodes_by_parent { return @nodes; } -# return the nodeid for the specified node -sub plugin_filepaths_get_nodeid { - my $self = shift; - my ($dmid, $parentnodeid, $nodename) = @_; - my $dbh = $self->dbh; - my $nodeid = $dbh->selectrow_array('SELECT nodeid FROM plugin_filepaths_paths ' . - 'WHERE dmid = ? AND parentnodeid = ? AND nodename = ?', - undef, $dmid, $parentnodeid, $nodename); - return undef if $dbh->err; - return $nodeid; -} - -# get all the nodes that are child nodes of the specified parent node -sub plugin_filepaths_get_nodes_by_mapping { - my $self = shift; - my ($dmid, $parentnodeid) = @_; - my $dbh = $self->dbh; - my $sth = $dbh->prepare('SELECT nodename, fid FROM plugin_filepaths_paths ' . - 'WHERE dmid = ? AND parentnodeid = ?'); - $sth->execute($dmid, $parentnodeid); - - my @nodes; - while (my ($nodename, $fid) = $sth->fetchrow_array) { - push @nodes, [$nodename, $fid]; - } - - return @nodes; -} - # load the specified fids from the database sub plugin_filepaths_load_fids { my $self = shift; @@ -626,20 +597,6 @@ sub plugin_filepaths_load_fids { return @ret; } -# takes a domain, parentnodeid, and filename and returns a MogileFS::FID object -# for the found file -sub plugin_filepaths_get_fid_by_mapping { - my $self = shift; - my ($dmid, $parentnodeid, $filename) = @_; - - my $dbh = $self->dbh; - my ($fid) = $dbh->selectrow_array('SELECT fid FROM plugin_filepaths_paths ' . - 'WHERE dmid = ? AND parentnodeid = ? AND nodename = ?', - undef, $dmid, $parentnodeid, $filename); - return undef unless $fid; - return MogileFS::FID->new($fid); -} - __PACKAGE__->add_extra_tables("plugin_filepaths_paths", "plugin_filepaths_domains"); 1; From 6cd432ab9f5429c0e12f0455eb0d52fa1145e23c Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 16:17:54 -0500 Subject: [PATCH 12/20] update load_path to return a node object --- lib/MogileFS/Plugin/FilePaths.pm | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 5a16f69..beda98a 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -128,12 +128,12 @@ sub load { return 0 unless defined($path) && length($path) && defined($filename) && length($filename); # now try to get the end of the path - my $parentnodeid = MogileFS::Plugin::FilePaths::load_path( $args->{dmid}, $path ); - return 0 unless defined $parentnodeid; + my $parentnode = load_path( $args->{dmid}, $path ); + return 0 unless $parentnode; # get the fid of the file, bail out if it doesn't have one (directory nodes) my $sto = Mgd::get_store(); - my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnodeid, $filename); + my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnode->id, $filename); return 0 unless $node; my $fidid = $node->fidid; return 0 unless $fidid; @@ -200,9 +200,9 @@ sub load { unless $args->{argcount} == 1 && $path && $path =~ /^\//; # now find the id of the path - my $nodeid = load_path( $dmid, $path ); + my $node = load_path( $dmid, $path ); return $self->err_line('path_not_found', 'Path provided was not found in database') - unless defined $nodeid; + unless $node; # TODO This is wrong, but we should throw an error saying 'not a directory'. Requires refactoring # a bit of code to make the 'fid' value available from the last node we fetched. @@ -214,7 +214,7 @@ sub load { my %res; my $ct = 0; my $sto = Mgd::get_store(); - my @nodes = $sto->plugin_filepaths_get_nodes_by_parent($dmid, $nodeid); + my @nodes = $sto->plugin_filepaths_get_nodes_by_parent($dmid, $node->id); # get FIDs for all the found nodes my %fids = ( @@ -289,9 +289,9 @@ sub load { # LOCK rename # find the node being renamed - my $old_parentid = load_path($dmid, $old_path); my $sto = Mgd::get_store(); - my $node = $sto->plugin_filepaths_get_node_by_parent($dmid, $old_parentid, $old_name); + my $old_parent = load_path($dmid, $old_path); + my $node = $old_parent ? $sto->plugin_filepaths_get_node_by_parent($dmid, $old_parent->id, $old_name) : undef; return $self->err_line('path_not_found', 'Path provided was not found in database') unless $node; @@ -340,8 +340,7 @@ sub vivify_path { sub load_path { my ($dmid, $path) = @_; return undef unless $dmid && $path; - my $node = _traverse_path($dmid, $path, 0); - return $node ? $node->id : undef; + return _traverse_path($dmid, $path, 0); } # does the internal work of traversing a path @@ -398,12 +397,12 @@ sub _path_to_key { return 0 unless $path && $filename; # now try to get the end of the path - my $parentnodeid = MogileFS::Plugin::FilePaths::load_path( $dmid, $path ); - return 0 unless defined $parentnodeid; + my $parentnode = load_path( $dmid, $path ); + return 0 unless $parentnode; # great, find this file my $sto = Mgd::get_store(); - my $node = $sto->plugin_filepaths_get_node_by_parent($dmid, $parentnodeid, $filename); + my $node = $sto->plugin_filepaths_get_node_by_parent($dmid, $parentnode->id, $filename); my $fidid = $node->fidid; return 0 unless $fidid; From 2a84e2f217be0f8dfa55a3bb137d30b9d2566ac1 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 16:19:09 -0500 Subject: [PATCH 13/20] remove a couple redundant tests --- lib/MogileFS/Plugin/FilePaths.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index beda98a..e3ef35b 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -331,7 +331,6 @@ sub unload { # on error, else, 0-N is valid. sub vivify_path { my ($dmid, $path) = @_; - return undef unless $dmid && $path; return _traverse_path($dmid, $path, 1); } @@ -339,7 +338,6 @@ sub vivify_path { # out if a path exists. does NOT automatically create path elements that don't exist. sub load_path { my ($dmid, $path) = @_; - return undef unless $dmid && $path; return _traverse_path($dmid, $path, 0); } From 99346ccc04de7d3b488ef16870b3cd60e6611ecf Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 16:20:12 -0500 Subject: [PATCH 14/20] unregister all hooks --- lib/MogileFS/Plugin/FilePaths.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index e3ef35b..1344354 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -322,6 +322,10 @@ sub unload { MogileFS::unregister_global_hook( 'cmd_create_open' ); MogileFS::unregister_global_hook( 'cmd_create_close' ); MogileFS::unregister_global_hook( 'file_stored' ); + MogileFS::unregister_global_hook( 'cmd_get_paths' ); + MogileFS::unregister_global_hook( 'cmd_file_info' ); + MogileFS::unregister_global_hook( 'cmd_file_debug' ); + MogileFS::unregister_global_hook( 'cmd_delete' ); return 1; } From 13d37e10e96841650075eab0043697300e054ccd Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 16:21:46 -0500 Subject: [PATCH 15/20] make sure path traversal only processes directory nodes --- lib/MogileFS/Plugin/FilePaths.pm | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 1344354..0961d6d 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -204,12 +204,6 @@ sub load { return $self->err_line('path_not_found', 'Path provided was not found in database') unless $node; -# TODO This is wrong, but we should throw an error saying 'not a directory'. Requires refactoring -# a bit of code to make the 'fid' value available from the last node we fetched. -# if (get_file_mapping($nodeid)) { -# return $self->err_line('not_a_directory', 'Path provided is not a directory'); -# } - # get files in path, return as an array my %res; my $ct = 0; @@ -357,7 +351,7 @@ sub _traverse_path { foreach my $part (grep { $_ } split /\//, $path) { # look for the current path part $node = _find_node($dmid, $node->id, $part, $vivify); - return undef unless $node; + return undef unless $node && $node->is_directory; } # we're done, so return the most recent node From 20e0ab0c3a4a53f81e9cee4bd5599f2341cedf4d Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 24 Feb 2012 16:43:18 -0500 Subject: [PATCH 16/20] do a bit more validation of when we are working on a file vs a directory --- lib/MogileFS/Plugin/FilePaths.pm | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 0961d6d..0e0c653 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -75,10 +75,13 @@ sub load { my $parentnode = vivify_path( $args->{dmid}, $path ); return 0 unless defined $parentnode; - # find/create a node to store this file at, track the old FID in order - # to delete it after updating the node + # find any existing node, bail if we find a directory my $sto = Mgd::get_store(); my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnode->id, $filename); + return 0 if $node && $node->is_directory; + + # update/create node to store this file at, track the old FID in order + # to delete it after updating the node my $oldfid = $node ? $node->fid : undef; if($node) { $sto->plugin_filepaths_update_node($node->id, {'fid' => $args->{fid}}); @@ -131,19 +134,17 @@ sub load { my $parentnode = load_path( $args->{dmid}, $path ); return 0 unless $parentnode; - # get the fid of the file, bail out if it doesn't have one (directory nodes) + # find the file node and bail if it doesn't exist my $sto = Mgd::get_store(); my $node = $sto->plugin_filepaths_get_node_by_parent($args->{dmid}, $parentnode->id, $filename); - return 0 unless $node; - my $fidid = $node->fidid; - return 0 unless $fidid; + return 0 unless $node && $node->is_file; - # great, delete this file + # update the key + $args->{key} = 'fid:' . $node->fidid; + + # great, delete this node $sto->plugin_filepaths_delete_node($node->id); # FIXME What should happen if this delete fails? - - # now pretend they asked for it and continue - $args->{key} = 'fid:' . $fidid; }); MogileFS::register_worker_command( 'filepaths_enable', sub { @@ -211,10 +212,7 @@ sub load { my @nodes = $sto->plugin_filepaths_get_nodes_by_parent($dmid, $node->id); # get FIDs for all the found nodes - my %fids = ( - map {($_->id, $_)} - $sto->plugin_filepaths_load_fids(map {$_->fidid} @nodes) - ); + my %fids = map {($_->id => $_)} $sto->plugin_filepaths_load_fids(map {$_->fidid} @nodes); # add all nodes to the response my $i = 0; @@ -229,7 +227,7 @@ sub load { $res{"$prefix.type"} = "D"; } # This file is a regular file - elsif(my $fid = $fids{$node->fidid} || $node->fid) { + elsif($node->is_file && (my $fid = $fids{$node->fidid} || $node->fid)) { # skip this node unless the fid exists next unless $fid->exists; From 32dc338e4da904713f64c62475343c6f54d4c5df Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Mon, 27 Feb 2012 13:39:35 -0500 Subject: [PATCH 17/20] added a remove_directory command that can be used to delete empty directories --- lib/MogileFS/Plugin/FilePaths.pm | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 0e0c653..99089ce 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -302,6 +302,43 @@ sub load { return $self->ok_line(); }); + MogileFS::register_worker_command( 'filepaths_remove_directory', sub { + my MogileFS::Worker::Query $self = shift; + my $args = shift; + + # verify domain firstly + my $dmid = $self->check_domain($args) + or return $self->err_line('domain_not_found'); + + return $self->err_line("plugin_not_active_for_domain") + unless _check_dmid($dmid); + + # verify arguments - make sure we have the right amount or arguments + return $self->err_line('bad_params') + unless $args->{argcount} == 1; + + # verify arguments - path, make sure it starts with a / + my $path = $args->{arg1}; + return $self->err_line('bad_params') + unless $path && $path =~ /^\//; + + # now find the specified node + my $node = load_path( $dmid, $path ); + return $self->err_line('path_not_found', 'Path provided was not found in database') + unless $node && $node->is_directory; + + # check to see if the directory is empty + my $sto = Mgd::get_store(); + my @nodes = $sto->plugin_filepaths_get_nodes_by_parent($dmid, $node->id); + return $self->err_line('directory_not_empty', 'Provided path was not empty') + if @nodes; + + # delete this node + $sto->plugin_filepaths_delete_node($node->id); + + return $self->ok_line(); + }); + return 1; } From 111845ad774000552befdf63871bd4cb3f2e74b2 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Thu, 29 Nov 2012 22:39:32 -0500 Subject: [PATCH 18/20] only retrieve the fid if the node exists --- lib/MogileFS/Plugin/FilePaths.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 0e0c653..5f2223c 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -397,7 +397,7 @@ sub _path_to_key { # great, find this file my $sto = Mgd::get_store(); my $node = $sto->plugin_filepaths_get_node_by_parent($dmid, $parentnode->id, $filename); - my $fidid = $node->fidid; + my $fidid = $node ? $node->fidid : undef; return 0 unless $fidid; # now pretend they asked for it and continue From 9006fd08848a1112753f2e6500ae9bcd17e27763 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Sat, 1 Dec 2012 10:58:06 -0500 Subject: [PATCH 19/20] update the key for the updateclass cmd --- lib/MogileFS/Plugin/FilePaths.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 5f2223c..9b4bdf3 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -122,6 +122,7 @@ sub load { MogileFS::register_global_hook( 'cmd_get_paths', \&_path_to_key ); MogileFS::register_global_hook( 'cmd_file_info', \&_path_to_key ); MogileFS::register_global_hook( 'cmd_file_debug', \&_path_to_key ); + MogileFS::register_global_hook( 'cmd_updateclass', \&_path_to_key ); MogileFS::register_global_hook( 'cmd_delete', sub { my $args = shift; return 1 unless _check_dmid($args->{dmid}); From d32daee2d7e9660a64c9f479b1619699cce8f7e4 Mon Sep 17 00:00:00 2001 From: Daniel Frett Date: Fri, 17 May 2013 22:41:13 -0400 Subject: [PATCH 20/20] utilize the get_bulk_metadata method --- lib/MogileFS/Plugin/FilePaths.pm | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/MogileFS/Plugin/FilePaths.pm b/lib/MogileFS/Plugin/FilePaths.pm index 5f2223c..9bec1ec 100644 --- a/lib/MogileFS/Plugin/FilePaths.pm +++ b/lib/MogileFS/Plugin/FilePaths.pm @@ -210,9 +210,13 @@ sub load { my $ct = 0; my $sto = Mgd::get_store(); my @nodes = $sto->plugin_filepaths_get_nodes_by_parent($dmid, $node->id); + my @fidids = grep {$_} map {$_->fidid} @nodes; # get FIDs for all the found nodes - my %fids = map {($_->id => $_)} $sto->plugin_filepaths_load_fids(map {$_->fidid} @nodes); + my %fids = map {($_->id => $_)} $sto->plugin_filepaths_load_fids(@fidids); + + # load the meta-data for all the fids + my $metadata = MogileFS::Plugin::MetaData::get_bulk_metadata(@fidids); # add all nodes to the response my $i = 0; @@ -234,8 +238,8 @@ sub load { $res{"$prefix.type"} = "F"; my $length = $fid->length; $res{"$prefix.size"} = $length if defined($length); - my $metadata = MogileFS::Plugin::MetaData::get_metadata($fid->id); - $res{"$prefix.mtime"} = $metadata->{mtime} if $metadata->{mtime}; + my $mtime = $metadata->{$fid->id}->{'mtime'}; + $res{"$prefix.mtime"} = $mtime if $mtime; } # invalid node, don't include it in the response else {