From c6ae612f3022e4f010dc8ecce7b19e11e6b12f93 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 10 Nov 2011 10:16:23 +0000 Subject: documentation/packaging updates Trying to find a happy medium within Hoe while keeping my preference for gmake and forcing wrongdoc on readers: JavaScript and GUIs all suck :P --- .document | 10 +++++++ .gitignore | 3 +++ .wrongdoc.yml | 5 ++++ GNUmakefile | 35 +++++++++++++++++++++++++ History | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ History.txt | 81 --------------------------------------------------------- LICENSE | 27 +++++++++++++++++++ LICENSE.txt | 28 -------------------- Manifest.txt | 4 +++ README | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.txt | 80 -------------------------------------------------------- Rakefile | 16 +++++------- lib/mogilefs.rb | 2 ++ 13 files changed, 252 insertions(+), 198 deletions(-) create mode 100644 .document create mode 100644 .wrongdoc.yml create mode 100644 History delete mode 100644 History.txt create mode 100644 LICENSE delete mode 100644 LICENSE.txt create mode 100644 README delete mode 100644 README.txt diff --git a/.document b/.document new file mode 100644 index 0000000..eef8ef0 --- /dev/null +++ b/.document @@ -0,0 +1,10 @@ +History +ChangeLog +LICENSE +README +lib/mogilefs.rb +lib/mogilefs/admin.rb +lib/mogilefs/backend.rb +lib/mogilefs/client.rb +lib/mogilefs/mogilefs.rb +lib/mogilefs/pool.rb diff --git a/.gitignore b/.gitignore index 341d3b4..3cfc9e6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ /doc /.config /InstalledFiles +/ChangeLog +/NEWS +/LATEST diff --git a/.wrongdoc.yml b/.wrongdoc.yml new file mode 100644 index 0000000..d62f82d --- /dev/null +++ b/.wrongdoc.yml @@ -0,0 +1,5 @@ +--- +cgit_url: http://bogomips.org/mogilefs-client.git +git_url: git://bogomips.org/mogilefs-client.git +rdoc_url: http://bogomips.org/mogilefs-client +changelog_start: v1.2.1 diff --git a/GNUmakefile b/GNUmakefile index da1e1e3..574e77a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,5 +1,6 @@ # use GNU Make to run tests in parallel, and without depending on Rubygems all:: test +RSYNC_DEST := bogomips.org:/srv/bogomips/mogilefs-client T := $(wildcard test/test*.rb) TO := $(subst .rb,.log,$(T)) @@ -56,3 +57,37 @@ flog: $(libs) check-warnings: @(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \ do ruby -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || : +RSYNC = rsync +WRONGDOC = wrongdoc + +doc:: .document .wrongdoc.yml $(pkg_extra) + -find lib -type f -name '*.rbc' -exec rm -f '{}' ';' + $(RM) -r doc + $(WRONGDOC) all + install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/ + cd doc && \ + ln -s README README.txt && \ + ln -s README.html README_txt.html && \ + ln -s LICENSE LICENSE.txt && \ + ln -s LICENSE.html LICENSE_txt.html && \ + ln -s History History.txt && \ + ln -s History.html History_txt.html + +# Create gzip variants of the same timestamp as the original so nginx +# "gzip_static on" can serve the gzipped versions directly. +doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$') +doc_gz: + for i in $(docs); do \ + gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done + +# this requires GNU coreutils variants +ifneq ($(RSYNC_DEST),) +publish_doc: + -git set-file-times + $(MAKE) doc + find doc/images -type f | \ + TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css + $(MAKE) doc_gz + $(RSYNC) -av doc/ $(RSYNC_DEST)/ + git ls-files | xargs touch +endif diff --git a/History b/History new file mode 100644 index 0000000..54ad991 --- /dev/null +++ b/History @@ -0,0 +1,80 @@ += 2.2.0 +* internal cleanups (no public API breakage) +* refactor backend socket/connection handling for reliability + += 2.1.0 +* MySQL interface returns integer length and devcount (API change) +* Ensure store_{content,file} always returns size (API fix) +* Add get_uris API method +* Respect timeout when doing get_file_data +* MySQL interface filters out URLs for down/dead hosts/devices +* Really remove all NFS support +* get_file in slurp mode slurps all output correctly + += 2.0.2 +* README.txt: update documentation URL +* backend: raise exceptions with the error message +* backend: don't blindly add "Error" to exception class names +* Update Manifest.txt with setup.rb +* tasks to enforce permissions are set correctly + += 2.0.1 +* verify_uris method made more robust +* preliminary Ruby 1.9 compatibility, tests still need some work +* allow store_content to be used with a streamable object of known length +* add setup.rb for non-RubyGems users + += 2.0.0 +* use a set of standard exceptions based on MogileFS::Error, + this is an incompatible API change (hence the 2.0.0 version number). +* remove NFS support since it's gone in MogileFS 2.x and NFS is horrible +* mog timeouts and retries increased +* more consistent handling of bad sockets, all sockets used internally + are now explicitly non-blocking and IO.select is used for timeouts + instead of using threads behind our backs +* remove open-uri and net/http dependencies, they were bad with large files +* add paths_size() method, allowing single mogilefsd call to get size and paths +* add read-only MogileFS::Mysql driver, allowing mogilefsd to be bypassed +* use TCP_NODELAY when available on sockets that require low latency +* use TCP_CORK on bulk transfers to improve bandwidth usage +* better HTTP error handling +* verify_uris method in new MogileFS::Network module which allows + async verification of several URIs at once on the client side. +* handle multiple device failover correctly on HTTP uploads +* initial big_file read support (should be mogtool(1)-compatible) +* unit tests can be run in parallel using GNU Make, 3x faster on a Core2 Duo +* unit tests modified to use real sockets for easier verification of + timeout and error condition handling. + += 1.3.1 + +* Fix missing MogileFS::Util include for sysrwloop in MogileFS::MogileFS + += 1.3.0 + +* Fixed MogileFS#rename. Bug #14465 submitted by Justin Dossey. +* Removed infinite loop in MogileFS::HTTPFile#store_file. Patch #13789 + submitted by Andy Lo-A-Foe. +* Made MogileFS#get_file_data timeout configurable. Bug #13490 submitted by + Andy Lo-A-Foe. +* Add MogileFS#size. Feature Request #14484 submitted by Justin Dossey. +* Fix MogileFS#get_file_data to return the data for HTTP mode. Bug #7133 + submitted by John Wanko. +* New maintainer: Eric Wong +* Add `mog' command-line tool as a demo/example +* Lower memory consumption with large files +* Allow get_file_data to accept a block for large files +* Fix each_keys loop termination condition +* Apply error handling patch from Matthew Willson. Bug #15987 +* Merge large file patch from Andy Lo-A-Foe. Bug #13764 + += 1.2.1 + +* Switched to Hoe. +* Moved to p4. +* Fixed bug #7273 in HTTP mode of client where data would not get + returned. Submitted by Matthew Willson. + += 1.2.0 + +* Changes lost to time. diff --git a/History.txt b/History.txt deleted file mode 100644 index 3713a87..0000000 --- a/History.txt +++ /dev/null @@ -1,81 +0,0 @@ -= 2.2.0 -* internal cleanups (no public API breakage) -* refactor backend socket/connection handling for reliability - -= 2.1.0 -* MySQL interface returns integer length and devcount (API change) -* Ensure store_{content,file} always returns size (API fix) -* Add get_uris API method -* Respect timeout when doing get_file_data -* MySQL interface filters out URLs for down/dead hosts/devices -* Really remove all NFS support -* get_file in slurp mode slurps all output correctly - -= 2.0.2 -* README.txt: update documentation URL -* backend: raise exceptions with the error message -* backend: don't blindly add "Error" to exception class names -* Update Manifest.txt with setup.rb -* tasks to enforce permissions are set correctly - -= 2.0.1 -* verify_uris method made more robust -* preliminary Ruby 1.9 compatibility, tests still need some work -* allow store_content to be used with a streamable object of known length -* add setup.rb for non-RubyGems users - -= 2.0.0 -* use a set of standard exceptions based on MogileFS::Error, - this is an incompatible API change (hence the 2.0.0 version number). -* remove NFS support since it's gone in MogileFS 2.x and NFS is horrible -* mog timeouts and retries increased -* more consistent handling of bad sockets, all sockets used internally - are now explicitly non-blocking and IO.select is used for timeouts - instead of using threads behind our backs -* remove open-uri and net/http dependencies, they were bad with large files -* add paths_size() method, allowing single mogilefsd call to get size and paths -* add read-only MogileFS::Mysql driver, allowing mogilefsd to be bypassed -* use TCP_NODELAY when available on sockets that require low latency -* use TCP_CORK on bulk transfers to improve bandwidth usage -* better HTTP error handling -* verify_uris method in new MogileFS::Network module which allows - async verification of several URIs at once on the client side. -* handle multiple device failover correctly on HTTP uploads -* initial big_file read support (should be mogtool(1)-compatible) -* unit tests can be run in parallel using GNU Make, 3x faster on a Core2 Duo -* unit tests modified to use real sockets for easier verification of - timeout and error condition handling. - -= 1.3.1 - -* Fix missing MogileFS::Util include for sysrwloop in MogileFS::MogileFS - -= 1.3.0 - -* Fixed MogileFS#rename. Bug #14465 submitted by Justin Dossey. -* Removed infinite loop in MogileFS::HTTPFile#store_file. Patch #13789 - submitted by Andy Lo-A-Foe. -* Made MogileFS#get_file_data timeout configurable. Bug #13490 submitted by - Andy Lo-A-Foe. -* Add MogileFS#size. Feature Request #14484 submitted by Justin Dossey. -* Fix MogileFS#get_file_data to return the data for HTTP mode. Bug #7133 - submitted by John Wanko. -* New maintainer: Eric Wong -* Add `mog' command-line tool as a demo/example -* Lower memory consumption with large files -* Allow get_file_data to accept a block for large files -* Fix each_keys loop termination condition -* Apply error handling patch from Matthew Willson. Bug #15987 -* Merge large file patch from Andy Lo-A-Foe. Bug #13764 - -= 1.2.1 - -* Switched to Hoe. -* Moved to p4. -* Fixed bug #7273 in HTTP mode of client where data would not get - returned. Submitted by Matthew Willson. - -= 1.2.0 - -* Changes lost to time. - diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..12a2ea5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright 2005 Eric Hodel, The Robot Co-op. All rights reserved. +Copyright 2008 Eric Wong. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the names of the authors nor the names of their contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 0133510..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2005 Eric Hodel, The Robot Co-op. All rights reserved. -Copyright 2008 Eric Wong. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -3. Neither the names of the authors nor the names of their contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, -OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT -OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/Manifest.txt b/Manifest.txt index 1c04e44..df7f614 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -1,4 +1,7 @@ +.CHANGELOG.old +.document .gitignore +.wrongdoc.yml GNUmakefile History.txt LICENSE.txt @@ -38,6 +41,7 @@ test/test_bigfile.rb test/test_client.rb test/test_db_backend.rb test/test_fresh.rb +test/test_http_reader.rb test/test_mogilefs.rb test/test_mogilefs_integration.rb test/test_mogilefs_integration_large_pipe.rb diff --git a/README b/README new file mode 100644 index 0000000..d72802d --- /dev/null +++ b/README @@ -0,0 +1,79 @@ += mogilefs-client - MogileFS client library for Ruby + +A MogileFS client library for Ruby. MogileFS is an open source +distributed filesystem, see: http://mogilefs.org for more details. This +library allows any Ruby application to read, write and delete files in a +MogileFS instance. + +== Links + +rdoc :: http://bogomips.org/mogilefs-client +mogilefs :: http://mogilefs.org/ +list :: mailto:mogile@googlegroups.com +email :: mailto:normalperson@yhbt.net +repo :: git://bogomips.org/mogilefs-client.git +cgit :: http://bogomips.org/mogilefs-client.git +gitweb :: http://repo.or.cz/w/ruby-mogilefs-client.git + +== Install + +First you need a MogileFS 2.x installation. You can find information on +how to do that at http://mogilefs.org + +Then install the RubyGem: + + $ gem install mogilefs-client + +This library supports Ruby 1.8.7 and later, but Ruby 1.9.3 is +recommended. No other libraries are required on the client. + +== Use + + # Create a new instance that will communicate with these trackers: + hosts = %w[192.168.1.69:6001 192.168.1.70:6001] + mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts) + + # Stores "A bunch of text to store" into 'some_key' with a class of 'text'. + mg.store_content 'some_key', 'text', "A bunch of text to store" + + # Retrieve data from 'some_key' + data = mg.get_file_data 'some_key' + + # Store the contents of 'image.jpeg' into the key 'my_image' with a class of + # 'image'. + mg.store_file 'my_image', 'image', 'image.jpeg' + + # Store the contents of 'image.jpeg' into the key 'my_image' with a class of + # 'image' using an open IO. + File.open 'image.jpeg', 'rb' do |fp| + mg.store_file 'my_image', 'image', fp + end + + # Retrieve the contents of 'my_image' into '/path/to/huge_file' + # without slurping the entire contents into memory: + File.open '/path/to/huge_file', 'w' do |fp| + mg.get_file_data 'my_image', wr + end + + # Remove the key 'my_image' and 'some_key'. + mg.delete 'my_image' + mg.delete 'some_key' + +== Contact + +Feedback (bug reports, user/development dicussion, patches, pull +requests) are greatly appreciated and handled via email. We currently +piggy-back onto the public MogileFS +{mailing list}[mailto:mogile@googlegroups.com]. + +If you do not want to deal with the corporate host of the MogileFS +mailing list, or if you wish to keep your issue secret, feel free to +email {Eric Wong at}[mailto:normalperson@yhbt.net]. + +Do not expect Eric Wong to read HTML mail under any circumstances. + +== WARNING! + +This client is only supported in HTTP mode. NFS mode was previously +supported in 1.3.x, but since MogileFS 2.x has dropped support for +NFS, this client has removed support for it. diff --git a/README.txt b/README.txt deleted file mode 100644 index 0def5e9..0000000 --- a/README.txt +++ /dev/null @@ -1,80 +0,0 @@ -= mogilefs-client - -A Ruby MogileFS client - -Rubyforge Project: - -http://rubyforge.org/projects/seattlerb/ - -Documentation: - -http://seattlerb.rubyforge.org/mogilefs-client - -File bugs: - -http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921 - -Please email Eric Wong at normalperson@yhbt.net as well since -he finds web interfaces difficult to use. - -Source repository (git): - - git://bogomips.org/mogilefs-client.git - http://bogomips.org/mogilefs-client.git - git://repo.or.cz/ruby-mogilefs-client.git (mirror) - http://repo.or.cz/r/ruby-mogilefs-client.git (mirror) - -Repository browsers: - -* http://bogomips.org/mogilefs-client.git (cgit) -* http://repo.or.cz/w/ruby-mogilefs-client.git (gitweb mirror) - -== About - -A Ruby MogileFS client. MogileFS is a distributed filesystem written -by Danga Interactive. This client only supports HTTP. - -For information on MogileFS see: - -http://danga.com/mogilefs/ - -== Installing mogilefs-client - -First you need a MogileFS setup. You can find information on how to do -that at the above URL. - -Then install the gem: - - $ sudo gem install mogilefs-client - -== Using mogilefs-client - - # Create a new instance that will communicate with these trackers: - hosts = %w[192.168.1.69:6001 192.168.1.70:6001] - mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts) - - # Stores "A bunch of text to store" into 'some_key' with a class of 'text'. - mg.store_content 'some_key', 'text', "A bunch of text to store" - - # Retrieve data from 'some_key' - data = mg.get_file_data 'some_key' - - # Store the contents of 'image.jpeg' into the key 'my_image' with a class of - # 'image'. - mg.store_file 'my_image', 'image', 'image.jpeg' - - # Store the contents of 'image.jpeg' into the key 'my_image' with a class of - # 'image' using an open IO. - File.open 'image.jpeg', 'rb' do |fp| - mg.store_file 'my_image', 'image', fp - end - - # Remove the key 'my_image' and 'some_key'. - mg.delete 'my_image' - mg.delete 'some_key' - -== WARNING! - -This client is only supported in HTTP mode. NFS mode was previously -supported in 1.3.x, but since MogileFS 2.x has dropped support for -NFS, this client has removed support for it. diff --git a/Rakefile b/Rakefile index 5d434b3..f18cffa 100644 --- a/Rakefile +++ b/Rakefile @@ -3,16 +3,14 @@ require 'hoe' $:.unshift 'lib' require 'mogilefs' +Hoe.plugin :seattlerb -Hoe.new 'mogilefs-client', MogileFS::VERSION do |p| - p.rubyforge_name = 'seattlerb' - p.author = [ 'Eric Wong', 'Eric Hodel' ] - p.email = 'normalperson@yhbt.net' # (Eric Wong) - # p.email = 'drbrain@segment7.net' # (Eric Hodel) - p.summary = p.paragraphs_of('README.txt', 1).first - p.description = p.paragraphs_of('README.txt', 9).first - p.url = p.paragraphs_of('README.txt', 5).first - p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n") +Hoe.spec 'mogilefs-client' do + self.rubyforge_name = 'seattlerb' + developer 'Eric Wong', 'normalperson@yhbt.net' + # developer 'drbrain@segment7.net', 'Eric Hodel' + self.readme_file = "README" + self.history_file = "History" end task :fix_perms do diff --git a/lib/mogilefs.rb b/lib/mogilefs.rb index 506d64b..e34d906 100644 --- a/lib/mogilefs.rb +++ b/lib/mogilefs.rb @@ -6,6 +6,7 @@ # To read more about Danga's MogileFS: http://danga.com/mogilefs/ module MogileFS + # :stopdoc: VERSION = '2.2.0' @@ -37,6 +38,7 @@ module MogileFS autoload :Mysql, 'mogilefs/mysql' autoload :Pool, 'mogilefs/pool' autoload :Admin, 'mogilefs/admin' + # :startdoc: end require 'mogilefs/util' -- cgit v1.2.3-24-ge0c7