diff options
| author | Eric Wong <normalperson@yhbt.net> | 2010-11-22 15:36:54 -0800 |
|---|---|---|
| committer | Eric Wong <normalperson@yhbt.net> | 2010-11-22 19:56:05 -0800 |
| commit | 5eb8bf5e005d0d800af0c7e5e2dfd6b2526dd218 (patch) | |
| tree | e1665fd589f13174a6ff0919946cc96e9308772d | |
| download | metropolis-5eb8bf5e005d0d800af0c7e5e2dfd6b2526dd218.tar.gz | |
initial TokyoCabinet hash database support
Basically working, for now
| -rw-r--r-- | .document | 5 | ||||
| -rw-r--r-- | .gitignore | 20 | ||||
| -rw-r--r-- | COPYING | 661 | ||||
| -rwxr-xr-x | GIT-VERSION-GEN | 40 | ||||
| -rw-r--r-- | GNUmakefile | 181 | ||||
| -rw-r--r-- | LICENSE | 17 | ||||
| -rw-r--r-- | README | 46 | ||||
| -rw-r--r-- | Rakefile | 144 | ||||
| -rw-r--r-- | lib/metropolis.rb | 28 | ||||
| -rw-r--r-- | lib/metropolis/tokyocabinet.rb | 6 | ||||
| -rw-r--r-- | lib/metropolis/tokyocabinet/hdb.rb | 160 | ||||
| -rw-r--r-- | lib/metropolis/tokyocabinet/hdb/ro.rb | 34 | ||||
| -rw-r--r-- | metropolis.gemspec | 46 | ||||
| -rw-r--r-- | setup.rb | 1586 | ||||
| -rw-r--r-- | test/test_tokyocabinet_hdb.rb | 200 |
15 files changed, 3174 insertions, 0 deletions
diff --git a/.document b/.document new file mode 100644 index 0000000..ccf40b1 --- /dev/null +++ b/.document @@ -0,0 +1,5 @@ +README +NEWS +ChangeLog +COPYING +LICENSE diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b98bf2e --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +*.o +*.bundle +*.log +*.so +*.rbc +/.config +/InstalledFiles +/doc +/local.mk +/test/rbx-* +/test/ruby-* +log/ +pkg/ +/vendor +/NEWS +/ChangeLog +/.manifest +/GIT-VERSION-FILE +/man +/tmp @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +<http://www.gnu.org/licenses/>. diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN new file mode 100755 index 0000000..92e03e3 --- /dev/null +++ b/GIT-VERSION-GEN @@ -0,0 +1,40 @@ +#!/bin/sh + +GVF=GIT-VERSION-FILE +DEF_VER=v0.0.0.GIT + +LF=' +' + +# First see if there is a version file (included in release tarballs), +# then try git-describe, then default. +if test -f version +then + VN=$(cat version) || VN="$DEF_VER" +elif test -d .git -o -f .git && + VN=$(git describe --abbrev=4 HEAD 2>/dev/null) && + case "$VN" in + *$LF*) (exit 1) ;; + v[0-9]*) + git update-index -q --refresh + test -z "$(git diff-index --name-only HEAD --)" || + VN="$VN-dirty" ;; + esac +then + VN=$(echo "$VN" | sed -e 's/-/./g'); +else + VN="$DEF_VER" +fi + +VN=$(expr "$VN" : v*'\(.*\)') + +if test -r $GVF +then + VC=$(sed -e 's/^GIT_VERSION = //' <$GVF) +else + VC=unset +fi +test "$VN" = "$VC" || { + echo >&2 "GIT_VERSION = $VN" + echo "GIT_VERSION = $VN" >$GVF +} diff --git a/GNUmakefile b/GNUmakefile new file mode 100644 index 0000000..e035434 --- /dev/null +++ b/GNUmakefile @@ -0,0 +1,181 @@ +# use GNU Make to run tests in parallel, and without depending on RubyGems +all:: +RUBY = ruby +RAKE = rake +GIT_URL = git://git.bogomips.org/metropolis.git +RSYNC = rsync + +GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE + @./GIT-VERSION-GEN +-include GIT-VERSION-FILE +-include local.mk +ifeq ($(RUBY_VERSION),) + RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION') +endif + +install: $(bins) + $(prep_setup_rb) + $(RM) -r .install-tmp + mkdir .install-tmp + cp -p bin/* .install-tmp + $(RUBY) setup.rb all + $(RM) $^ + mv .install-tmp/* bin/ + $(RM) -r .install-tmp + $(prep_setup_rb) + +setup_rb_files := .config InstalledFiles +prep_setup_rb := @-$(RM) $(setup_rb_files);$(MAKE) -C $(ext) clean + +clean: + $(RM) $(setup_rb_files) + +pkg_extra := GIT-VERSION-FILE NEWS ChangeLog +manifest: $(pkg_extra) + $(RM) .manifest + $(MAKE) .manifest + +.manifest: + (git ls-files && \ + for i in $@ $(pkg_extra) $(man1_paths); \ + do echo $$i; done) | LC_ALL=C sort > $@+ + cmp $@+ $@ || mv $@+ $@ + $(RM) $@+ + +NEWS: GIT-VERSION-FILE + $(RAKE) -s news_rdoc > $@+ + mv $@+ $@ + +latest: NEWS + @awk 'BEGIN{RS="=== ";ORS=""}NR==2{sub(/\n$$/,"");print RS""$$0 }' $< + +SINCE = +ChangeLog: LOG_VERSION = \ + $(shell git rev-parse -q "$(GIT_VERSION)" >/dev/null 2>&1 && \ + echo $(GIT_VERSION) || git describe) +ifneq ($(SINCE),) +ChangeLog: log_range = v$(SINCE)..$(LOG_VERSION) +endif +ChangeLog: GIT-VERSION-FILE + @echo "ChangeLog from $(GIT_URL) ($(log_range))" > $@+ + @echo >> $@+ + git log $(log_range) | sed -e 's/^/ /' >> $@+ + mv $@+ $@ + +news_atom := http://metropolis.bogomips.org/NEWS.atom.xml +cgit_atom := http://git.bogomips.org/cgit/metropolis.git/atom/?h=master +atom = <link rel="alternate" title="Atom feed" href="$(1)" \ + type="application/atom+xml"/> + +# using rdoc 2.5.x +doc: .document NEWS ChangeLog + rdoc -t "$(shell sed -ne '1s/^= //p' README)" + install -m644 COPYING doc/COPYING + install -m644 $(shell grep '^[A-Z]' .document) doc/ + $(RUBY) -i -p -e \ + '$$_.gsub!("</title>",%q{\&$(call atom,$(cgit_atom))})' \ + doc/ChangeLog.html + $(RUBY) -i -p -e \ + '$$_.gsub!("</title>",%q{\&$(call atom,$(news_atom))})' \ + doc/NEWS.html doc/README.html + $(RAKE) -s news_atom > doc/NEWS.atom.xml + cd doc && ln README.html tmp && mv tmp index.html + +ifneq ($(VERSION),) +rfproject := rainbows +rfpackage := metropolis +pkggem := pkg/$(rfpackage)-$(VERSION).gem +pkgtgz := pkg/$(rfpackage)-$(VERSION).tgz +release_notes := release_notes-$(VERSION) +release_changes := release_changes-$(VERSION) + +release-notes: $(release_notes) +release-changes: $(release_changes) +$(release_changes): + $(RAKE) -s release_changes > $@+ + $(VISUAL) $@+ && test -s $@+ && mv $@+ $@ +$(release_notes): + GIT_URL=$(GIT_URL) $(RAKE) -s release_notes > $@+ + $(VISUAL) $@+ && test -s $@+ && mv $@+ $@ + +# ensures we're actually on the tagged $(VERSION), only used for release +verify: + test x"$(shell umask)" = x0022 + git rev-parse --verify refs/tags/v$(VERSION)^{} + git diff-index --quiet HEAD^0 + test `git rev-parse --verify HEAD^0` = \ + `git rev-parse --verify refs/tags/v$(VERSION)^{}` + +fix-perms: + -git ls-tree -r HEAD | awk '/^100644 / {print $$NF}' | xargs chmod 644 + -git ls-tree -r HEAD | awk '/^100755 / {print $$NF}' | xargs chmod 755 + +gem: $(pkggem) + +install-gem: $(pkggem) + gem install $(CURDIR)/$< + +$(pkggem): manifest fix-perms + gem build $(rfpackage).gemspec + mkdir -p pkg + mv $(@F) $@ + +$(pkgtgz): distdir = $(basename $@) +$(pkgtgz): HEAD = v$(VERSION) +$(pkgtgz): manifest fix-perms + @test -n "$(distdir)" + $(RM) -r $(distdir) + mkdir -p $(distdir) + tar cf - `cat .manifest` | (cd $(distdir) && tar x) + cd pkg && tar cf - $(basename $(@F)) | gzip -9 > $(@F)+ + mv $@+ $@ + +package: $(pkgtgz) $(pkggem) + +test-release: verify package $(release_notes) $(release_changes) +release: verify package $(release_notes) $(release_changes) + # make tgz release on RubyForge + rubyforge add_release -f -n $(release_notes) -a $(release_changes) \ + $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz) + # push gem to RubyGems.org + gem push $(pkggem) + # in case of gem downloads from RubyForge releases page + -rubyforge add_file \ + $(rfproject) $(rfpackage) $(VERSION) $(pkggem) + $(RAKE) raa_update VERSION=$(VERSION) + $(RAKE) publish_news VERSION=$(VERSION) +else +gem install-gem: GIT-VERSION-FILE + $(MAKE) $@ VERSION=$(GIT_VERSION) +endif + +all:: test + +test_units := $(wildcard test/test_*.rb) +test: test-unit +test-unit: $(test_units) +$(test_units): + $(RUBY) -I lib $@ + +# this requires GNU coreutils variants +publish_doc: + -git set-file-times + $(RM) -r doc ChangeLog NEWS + $(MAKE) doc LOG_VERSION=$(shell git tag -l | tail -1) + $(MAKE) -s latest > doc/LATEST + find doc/images doc/js -type f | \ + TZ=UTC xargs touch -d '1970-01-01 00:00:00' doc/rdoc.css + $(MAKE) doc_gz + chmod 644 $$(find doc -type f) + $(RSYNC) -av doc/ metropolis.bogomips.org:/srv/metropolis/ + git ls-files | xargs touch + +# Create gzip variants of the same timestamp as the original so nginx +# "gzip_static on" can serve the gzipped versions directly. +doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$') +doc_gz: + touch doc/NEWS.atom.xml -d "$$(awk 'NR==1{print $$4,$$5,$$6}' NEWS)" + for i in $(docs); do \ + gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done + +.PHONY: .FORCE-GIT-VERSION-FILE doc manifest man test $(test_units) @@ -0,0 +1,17 @@ +\Metropolis is copyrighted Free Software by all contributors, see logs in +revision control for names and email addresses of all of them. + +You can redistribute it and/or modify it under the terms of the GNU +Affero General Public License, version 3 ({AGPLv3}[link:COPYING]) as +published by the Free Software Foundation. The \Metropolis project leader +(Eric Wong) reserves the right to relicense \Metropolis under future versions +of the AGPL. + +\Metropolis is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public +License for more details. + +You should have received a copy of the GNU Affero General Public License +along with \BARFS; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 @@ -0,0 +1,46 @@ += \Metropolis: a key-value store for Rack + +\Metropolis is a Rack application used to store key value pairs. It +seeks to emulate the TokyoTyrant HTTP interface, but allows access to +multiple databases through a single TCP port. It hashes keys to split +keys between different on-disk databases to work around scalability +limitations of the kernel, hardware and filesystem. + +== Features + +* Currently supports the TokyoCabinet hash database format, other + database formats will be supported along with KyotoCabinet, as + well as other key-value stores. + +* Compatible with all Rack-compatible servers + +== Hacking + +You can get the latest source via git from the following locations: + + git://git.bogomips.org/metropolis.git + git://repo.or.cz/metropolis.git (mirror) + +You may browse the code from the web and download the latest snapshot +tarballs here: + +* http://git.bogomips.org/cgit/metropolis.git (cgit) +* http://repo.or.cz/w/metropolis.git (gitweb) + +Inline patches (from "git format-patch") to the mailing list are +preferred because they allow code review and comments in the reply to +the patch. + +We will adhere to mostly the same conventions for patch submissions as +git itself. See the Documentation/SubmittingPatches document +distributed with git on on patch submission guidelines to follow. Just +don't email the git mailing list or maintainer with \Metropolis patches. + +== Contact + +All feedback (bug reports, user/development discussion, patches, pull +requests) go to the mailing list: mailto:metropolis@librelist.org + +Mailing list archives in mbox format may be downloaded here: + +http://metropolis.bogomips.org/archives/ diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..5759974 --- /dev/null +++ b/Rakefile @@ -0,0 +1,144 @@ +# -*- encoding: binary -*- +autoload :Gem, 'rubygems' + +# most tasks are in the GNUmakefile which offers better parallelism + +TIMEFMT = '%Y-%m-%dT%H:%M:%SZ' +def tags + @tags ||= `git tag -l`.split(/\n/).map do |tag| + if %r{\Av[\d\.]+} =~ tag + header, subject, body = `git cat-file tag #{tag}`.split(/\n\n/, 3) + header = header.split(/\n/) + tagger = header.grep(/\Atagger /).first + body ||= "initial" + { + :time => Time.at(tagger.split(/ /)[-2].to_i).utc.strftime(TIMEFMT), + :tagger_name => %r{^tagger ([^<]+)}.match(tagger)[1].strip, + :tagger_email => %r{<([^>]+)>}.match(tagger)[1].strip, + :id => `git rev-parse refs/tags/#{tag}`.chomp!, + :tag => tag, + :subject => subject, + :body => body, + } + end + end.compact.sort { |a,b| b[:time] <=> a[:time] } +end + +cgit_url = "http://git.bogomips.org/cgit/metropolis.git" +git_url = ENV['GIT_URL'] || 'git://git.bogomips.org/metropolis.git' + +desc 'prints news as an Atom feed' +task :news_atom do + require 'nokogiri' + new_tags = tags[0,10] + puts(Nokogiri::XML::Builder.new do + feed :xmlns => "http://www.w3.org/2005/Atom" do + id! "http://metropolis.bogomips.org/NEWS.atom.xml" + title "Metropolis news" + subtitle "Rack HTTP key value store" + link! :rel => 'alternate', :type => 'text/html', + :href => 'http://metropolis.bogomips.org/NEWS.html' + if new_tags[0] + updated(new_tags[0][:time]) + else + updated(Time.now.utc.strftime(TIMEFMT)) + end + new_tags.each do |tag| + entry do + title tag[:subject] + updated tag[:time] + published tag[:time] + author { + name tag[:tagger_name] + email tag[:tagger_email] + } + url = "#{cgit_url}/tag/?id=#{tag[:tag]}" + link! :rel => "alternate", :type => "text/html", :href =>url + id! url + message_only = tag[:body].split(/\n.+\(\d+\):\n {6}/s).first.strip + content({:type =>:text}, message_only) + content(:type =>:xhtml) { pre tag[:body] } + end + end + end + end.to_xml) +end + +desc 'prints RDoc-formatted news' +task :news_rdoc do + tags.each do |tag| + time = tag[:time].tr!('T', ' ').gsub!(/:\d\dZ/, ' UTC') + puts "=== #{tag[:tag].sub(/^v/, '')} / #{time}" + puts "" + + body = tag[:body] + puts tag[:body].gsub(/^/sm, " ").gsub(/[ \t]+$/sm, "") + puts "" + end +end + +desc "print release changelog for Rubyforge" +task :release_changes do + version = ENV['VERSION'] or abort "VERSION= needed" + version = "v#{version}" + vtags = tags.map { |tag| tag[:tag] =~ /\Av/ and tag[:tag] }.sort + prev = vtags[vtags.index(version) - 1] + if prev + system('git', 'diff', '--stat', prev, version) or abort $? + puts "" + system('git', 'log', "#{prev}..#{version}") or abort $? + else + system('git', 'log', version) or abort $? + end +end + +desc "print release notes for Rubyforge" +task :release_notes do + spec = Gem::Specification.load('metropolis.gemspec') + puts spec.description.strip + puts "" + puts "* #{spec.homepage}" + puts "* #{spec.email}" + puts "* #{git_url}" + + _, _, body = `git cat-file tag v#{spec.version}`.split(/\n\n/, 3) + print "\nChanges:\n\n" + puts body +end + +desc "post to RAA" +task :raa_update do + require 'net/http' + require 'net/netrc' + rc = Net::Netrc.locate('metropolis-raa') or abort "~/.netrc not found" + password = rc.password + + s = Gem::Specification.load('metropolis.gemspec') + desc = [ s.description.strip ] + desc << "" + desc << "* #{s.email}" + desc << "* #{git_url}" + desc << "* #{cgit_url}" + desc = desc.join("\n") + uri = URI.parse('http://raa.ruby-lang.org/regist.rhtml') + form = { + :name => s.name, + :short_description => s.summary, + :version => s.version.to_s, + :status => 'stable', + :owner => s.authors.first, + :email => s.email, + :category_major => 'Library', + :category_minor => 'Web', + :url => s.homepage, + :download => "http://rubyforge.org/frs/?group_id=1306", + :license => "Ruby's", + :description_style => 'Plain', + :description => desc, + :pass => password, + :submit => "Update", + } + res = Net::HTTP.post_form(uri, form) + p res + puts res.body +end diff --git a/lib/metropolis.rb b/lib/metropolis.rb new file mode 100644 index 0000000..987040d --- /dev/null +++ b/lib/metropolis.rb @@ -0,0 +1,28 @@ +# -*- encoding: binary -*- +require 'rack' +require 'uri' + +module Metropolis + autoload :TokyoCabinet, 'metropolis/tokyocabinet' + + def self.new(opts = {}) + opts = opts.dup + rv = Object.new + uri = URI.parse(opts[:uri]) + case uri.scheme + when 'tc' + opts[:path_pattern] = uri.path + opts[:query] = Rack::Utils.parse_query(uri.query) if uri.query + case ext = File.extname(uri.path) + when '.tch' + rv.extend Metropolis::TokyoCabinet::HDB + else + raise ArgumentError, "unsupported suffix: #{ext}" + end + else + raise ArgumentError, "unsupported URI scheme: #{uri.scheme}" + end + rv.setup(opts) + rv + end +end diff --git a/lib/metropolis/tokyocabinet.rb b/lib/metropolis/tokyocabinet.rb new file mode 100644 index 0000000..86b8104 --- /dev/null +++ b/lib/metropolis/tokyocabinet.rb @@ -0,0 +1,6 @@ +# -*- encoding: binary -*- +require 'tokyocabinet' + +module Metropolis::TokyoCabinet + autoload :HDB, 'metropolis/tokyocabinet/hdb' +end diff --git a/lib/metropolis/tokyocabinet/hdb.rb b/lib/metropolis/tokyocabinet/hdb.rb new file mode 100644 index 0000000..ac8d58f --- /dev/null +++ b/lib/metropolis/tokyocabinet/hdb.rb @@ -0,0 +1,160 @@ +# -*- encoding: binary -*- + +# this module is NOT thread-safe, all performance is dependent on the +# local machine so there is never anything that needs yielding to threads. +module Metropolis::TokyoCabinet::HDB + autoload :RO, 'metropolis/tokyocabinet/hdb/ro' + + TCHDB = ::TokyoCabinet::HDB # :nodoc + include Rack::Utils # unescape + + def r(code) + body = "#{HTTP_STATUS_CODES[code]}\n" + [ code, + { 'Content-Length' => body.size.to_s, 'Content-Type' => 'text/plain' }, + [ body ] ] + end + + def setup(opts) + @headers = { 'Content-Type' => 'application/octet-stream' } + @headers.merge!(opts[:response_headers] || {}) + @nr_slots = opts[:nr_slots] || 3 + path_pattern = opts[:path_pattern] + path_pattern.scan(/%\d*x/).size == 1 or + raise ArgumentError, "only one '/%\d*x/' may appear in #{path_pattern}" + @optimize = nil + if query = opts[:query] + flags = 0 + @optimize = %w(bnum apow fpow).map do |x| + v = query[x] + v ? v.to_i : nil + end + case large = query['large'] + when 'false', nil + when 'true' + flags |= TCHDB::TLARGE + else + raise ArgumentError, "invalid 'large' value: #{large}" + end + case compress = query['compress'] + when nil + when 'deflate', 'bzip', 'tcbs' + flags |= TCHDB.const_get("T#{compress.upcase}") + else + raise ArgumentError, "invalid 'compress' value: #{compress}" + end + @optimize << flags + end + @hdbv = (0...@nr_slots).to_a.map do |slot| + path = sprintf(path_pattern, slot) + hdb = TCHDB.new + unless opts[:read_only] + hdb.open(path, TCHDB::OWRITER | TCHDB::OCREAT) or ex!(:open, hdb) + if @optimize + hdb.optimize(*@optimize) or ex!(:optimize, hdb) + end + hdb.close or ex!(:close, hdb) + end + [ hdb, path ] + end + @rd_flags = TCHDB::OREADER + @wr_flags = TCHDB::OWRITER + if opts[:read_only] + extend(RO) + end + end + + def call(env) + if %r{\A/(.*)\z} =~ env["PATH_INFO"] + key = unescape($1) + case env["REQUEST_METHOD"] + when "GET" + get(key) + when "HEAD" + head(key) + when "DELETE" + delete(key) + when "PUT" + put(key, env) + else + [ 405, {}, [] ] + end + else # OPTIONS + [ 405, {}, [] ] + end + end + + def ex!(msg, hdb) + raise "#{msg}: #{hdb.errmsg(hdb.ecode)}" + end + + def writer(key, &block) + hdb, path = @hdbv[key.hash % @nr_slots] + hdb.open(path, @wr_flags) or ex!(:open, hdb) + yield hdb + ensure + hdb.close or ex!(:close, hdb) + end + + def reader(key) + hdb, path = @hdbv[key.hash % @nr_slots] + hdb.open(path, @rd_flags) or ex!(:open, hdb) + yield hdb + ensure + hdb.close or ex!(:close, hdb) + end + + def put(key, env) + value = env["rack.input"].read + writer(key) do |hdb| + case env['HTTP_X_TT_PDMODE'] + when '1' + unless hdb.putkeep(key, value) + TCHDB::EKEEP == hdb.ecode and return r(409) + ex!(:putkeep, hdb) + end + when '2' + hdb.putcat(key, value) or ex!(:putcat, hdb) + else + # ttserver does not care for other PDMODE values, so we don't, either + hdb.put(key, value) or ex!(:put, hdb) + end + end + r(201) + end + + def delete(key) + writer(key) do |hdb| + unless hdb.delete(key) + TCHDB::ENOREC == hdb.ecode and return r(404) + ex!(:delete, hdb) + end + end + r(200) + end + + def head(key) + size = reader(key) { |hdb| hdb.vsiz(key) or ex!(:vsiz, hdb) } + 0 > size and return r(404) + [ 200, { + 'Content-Length' => size.to_s, + }.merge!(@headers), [] ] + end + + def get(key) + value = nil + reader(key) do |hdb| + unless value = hdb.get(key) + TCHDB::ENOREC == hdb.ecode and return r(404) + ex!(:get, hdb) + end + end + [ 200, { + 'Content-Length' => value.size.to_s, + }.merge!(@headers), [ value ] ] + end + + def close! + @hdbv.each { |(hdb,_)| hdb.close } + end +end diff --git a/lib/metropolis/tokyocabinet/hdb/ro.rb b/lib/metropolis/tokyocabinet/hdb/ro.rb new file mode 100644 index 0000000..b7a1b40 --- /dev/null +++ b/lib/metropolis/tokyocabinet/hdb/ro.rb @@ -0,0 +1,34 @@ +# -*- encoding: binary -*- + +module Metropolis::TokyoCabinet::HDB::RO + def self.extended(obj) + obj.instance_eval do + @wr_flags = nil + @rd_flags |= ::TokyoCabinet::HDB::ONOLCK + @hdbv.each { |(hdb, path)| + hdb.open(path, @rd_flags) or ex!(:open, path) + } + @ro_hdbv = @hdbv.map { |(hdb,_)| hdb } + end + end + + def call(env) + if %r{\A/(.*)\z} =~ env["PATH_INFO"] + key = unescape($1) + case env["REQUEST_METHOD"] + when "GET" + get(key) + when "HEAD" + head(key) + else + [ 405, {}, [] ] + end + else # OPTIONS + [ 405, {}, [] ] + end + end + + def reader(key) + yield @ro_hdbv[key.hash % @nr_slots] + end +end diff --git a/metropolis.gemspec b/metropolis.gemspec new file mode 100644 index 0000000..f937f4f --- /dev/null +++ b/metropolis.gemspec @@ -0,0 +1,46 @@ +# -*- encoding: binary -*- + +ENV["VERSION"] or abort "VERSION= must be specified" +manifest = File.readlines('.manifest').map! { |x| x.chomp! } + +# don't bother with tests that fork, not worth our time to get working +# with `gem check -t` ... (of course we care for them when testing with +# GNU make when they can run in parallel) +test_files = manifest.grep(%r{\Atest/test_.*\.rb\z}) + +Gem::Specification.new do |s| + s.name = %q{metropolis} + s.version = ENV["VERSION"].dup + + s.authors = ["The Sleeper"] + s.date = Time.now.utc.strftime('%Y-%m-%d') + s.description = File.read("README").split(/\n\n/)[1].delete('\\') + s.email = %q{metropolis@librelist.org} + s.executables = [] + + s.extra_rdoc_files = File.readlines('.document').map! do |x| + x.chomp! + if File.directory?(x) + manifest.grep(%r{\A#{x}/}) + elsif File.file?(x) + x + else + nil + end + end.flatten.compact + + s.files = manifest + s.homepage = %q{http://metropolis.bogomips.org/} + + summary = File.readlines("README")[0].delete('\'') + s.rdoc_options = [ "-t", summary ] + s.require_paths = %w(lib) + s.rubyforge_project = %q{rainbows} + s.summary = summary + + s.test_files = test_files + s.add_dependency(%q<rack>) + s.add_dependency(%q<tokyocabinet>) + + # s.licenses = %w(AGPL) # licenses= method is not in older RubyGems +end diff --git a/setup.rb b/setup.rb new file mode 100644 index 0000000..5eb5006 --- /dev/null +++ b/setup.rb @@ -0,0 +1,1586 @@ +# -*- encoding: binary -*- +# +# setup.rb +# +# Copyright (c) 2000-2005 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the terms of +# the GNU LGPL, Lesser General Public License version 2.1. +# + +unless Enumerable.method_defined?(:map) # Ruby 1.4.6 + module Enumerable + alias map collect + end +end + +unless File.respond_to?(:read) # Ruby 1.6 + def File.read(fname) + open(fname) {|f| + return f.read + } + end +end + +unless Errno.const_defined?(:ENOTEMPTY) # Windows? + module Errno + class ENOTEMPTY + # We do not raise this exception, implementation is not needed. + end + end +end + +def File.binread(fname) + open(fname, 'rb') {|f| + return f.read + } +end + +# for corrupted Windows' stat(2) +def File.dir?(path) + File.directory?((path[-1,1] == '/') ? path : path + '/') +end + + +class ConfigTable + + include Enumerable + + def initialize(rbconfig) + @rbconfig = rbconfig + @items = [] + @table = {} + # options + @install_prefix = nil + @config_opt = nil + @verbose = true + @no_harm = false + end + + attr_accessor :install_prefix + attr_accessor :config_opt + + attr_writer :verbose + + def verbose? + @verbose + end + + attr_writer :no_harm + + def no_harm? + @no_harm + end + + def [](key) + lookup(key).resolve(self) + end + + def []=(key, val) + lookup(key).set val + end + + def names + @items.map {|i| i.name } + end + + def each(&block) + @items.each(&block) + end + + def key?(name) + @table.key?(name) + end + + def lookup(name) + @table[name] or setup_rb_error "no such config item: #{name}" + end + + def add(item) + @items.push item + @table[item.name] = item + end + + def remove(name) + item = lookup(name) + @items.delete_if {|i| i.name == name } + @table.delete_if {|name, i| i.name == name } + item + end + + def load_script(path, inst = nil) + if File.file?(path) + MetaConfigEnvironment.new(self, inst).instance_eval File.read(path), path + end + end + + def savefile + '.config' + end + + def load_savefile + begin + File.foreach(savefile()) do |line| + k, v = *line.split(/=/, 2) + self[k] = v.strip + end + rescue Errno::ENOENT + setup_rb_error $!.message + "\n#{File.basename($0)} config first" + end + end + + def save + @items.each {|i| i.value } + File.open(savefile(), 'w') {|f| + @items.each do |i| + f.printf "%s=%s\n", i.name, i.value if i.value? and i.value + end + } + end + + def load_standard_entries + standard_entries(@rbconfig).each do |ent| + add ent + end + end + + def standard_entries(rbconfig) + c = rbconfig + + rubypath = File.join(c['bindir'], c['ruby_install_name'] + c['EXEEXT']) + + major = c['MAJOR'].to_i + minor = c['MINOR'].to_i + teeny = c['TEENY'].to_i + version = "#{major}.#{minor}" + + # ruby ver. >= 1.4.4? + newpath_p = ((major >= 2) or + ((major == 1) and + ((minor >= 5) or + ((minor == 4) and (teeny >= 4))))) + + if c['rubylibdir'] + # V > 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = c['rubylibdir'] + librubyverarch = c['archdir'] + siteruby = c['sitedir'] + siterubyver = c['sitelibdir'] + siterubyverarch = c['sitearchdir'] + elsif newpath_p + # 1.4.4 <= V <= 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = c['sitedir'] + siterubyver = "$siteruby/#{version}" + siterubyverarch = "$siterubyver/#{c['arch']}" + else + # V < 1.4.4 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = "#{c['prefix']}/lib/ruby/#{version}/site_ruby" + siterubyver = siteruby + siterubyverarch = "$siterubyver/#{c['arch']}" + end + parameterize = lambda {|path| + path.sub(/\A#{Regexp.quote(c['prefix'])}/, '$prefix') + } + + if arg = c['configure_args'].split.detect {|arg| /--with-make-prog=/ =~ arg } + makeprog = arg.sub(/'/, '').split(/=/, 2)[1] + else + makeprog = 'make' + end + + [ + ExecItem.new('installdirs', 'std/site/home', + 'std: install under libruby; site: install under site_ruby; home: install under $HOME')\ + {|val, table| + case val + when 'std' + table['rbdir'] = '$librubyver' + table['sodir'] = '$librubyverarch' + when 'site' + table['rbdir'] = '$siterubyver' + table['sodir'] = '$siterubyverarch' + when 'home' + setup_rb_error '$HOME was not set' unless ENV['HOME'] + table['prefix'] = ENV['HOME'] + table['rbdir'] = '$libdir/ruby' + table['sodir'] = '$libdir/ruby' + end + }, + PathItem.new('prefix', 'path', c['prefix'], + 'path prefix of target environment'), + PathItem.new('bindir', 'path', parameterize.call(c['bindir']), + 'the directory for commands'), + PathItem.new('libdir', 'path', parameterize.call(c['libdir']), + 'the directory for libraries'), + PathItem.new('datadir', 'path', parameterize.call(c['datadir']), + 'the directory for shared data'), + PathItem.new('mandir', 'path', parameterize.call(c['mandir']), + 'the directory for man pages'), + PathItem.new('sysconfdir', 'path', parameterize.call(c['sysconfdir']), + 'the directory for system configuration files'), + PathItem.new('localstatedir', 'path', parameterize.call(c['localstatedir']), + 'the directory for local state data'), + PathItem.new('libruby', 'path', libruby, + 'the directory for ruby libraries'), + PathItem.new('librubyver', 'path', librubyver, + 'the directory for standard ruby libraries'), + PathItem.new('librubyverarch', 'path', librubyverarch, + 'the directory for standard ruby extensions'), + PathItem.new('siteruby', 'path', siteruby, + 'the directory for version-independent aux ruby libraries'), + PathItem.new('siterubyver', 'path', siterubyver, + 'the directory for aux ruby libraries'), + PathItem.new('siterubyverarch', 'path', siterubyverarch, + 'the directory for aux ruby binaries'), + PathItem.new('rbdir', 'path', '$siterubyver', + 'the directory for ruby scripts'), + PathItem.new('sodir', 'path', '$siterubyverarch', + 'the directory for ruby extentions'), + PathItem.new('rubypath', 'path', rubypath, + 'the path to set to #! line'), + ProgramItem.new('rubyprog', 'name', rubypath, + 'the ruby program using for installation'), + ProgramItem.new('makeprog', 'name', makeprog, + 'the make program to compile ruby extentions'), + SelectItem.new('shebang', 'all/ruby/never', 'ruby', + 'shebang line (#!) editing mode'), + BoolItem.new('without-ext', 'yes/no', 'no', + 'does not compile/install ruby extentions') + ] + end + private :standard_entries + + def load_multipackage_entries + multipackage_entries().each do |ent| + add ent + end + end + + def multipackage_entries + [ + PackageSelectionItem.new('with', 'name,name...', '', 'ALL', + 'package names that you want to install'), + PackageSelectionItem.new('without', 'name,name...', '', 'NONE', + 'package names that you do not want to install') + ] + end + private :multipackage_entries + + ALIASES = { + 'std-ruby' => 'librubyver', + 'stdruby' => 'librubyver', + 'rubylibdir' => 'librubyver', + 'archdir' => 'librubyverarch', + 'site-ruby-common' => 'siteruby', # For backward compatibility + 'site-ruby' => 'siterubyver', # For backward compatibility + 'bin-dir' => 'bindir', + 'bin-dir' => 'bindir', + 'rb-dir' => 'rbdir', + 'so-dir' => 'sodir', + 'data-dir' => 'datadir', + 'ruby-path' => 'rubypath', + 'ruby-prog' => 'rubyprog', + 'ruby' => 'rubyprog', + 'make-prog' => 'makeprog', + 'make' => 'makeprog' + } + + def fixup + ALIASES.each do |ali, name| + @table[ali] = @table[name] + end + @items.freeze + @table.freeze + @options_re = /\A--(#{@table.keys.join('|')})(?:=(.*))?\z/ + end + + def parse_opt(opt) + m = @options_re.match(opt) or setup_rb_error "config: unknown option #{opt}" + m.to_a[1,2] + end + + def dllext + @rbconfig['DLEXT'] + end + + def value_config?(name) + lookup(name).value? + end + + class Item + def initialize(name, template, default, desc) + @name = name.freeze + @template = template + @value = default + @default = default + @description = desc + end + + attr_reader :name + attr_reader :description + + attr_accessor :default + alias help_default default + + def help_opt + "--#{@name}=#{@template}" + end + + def value? + true + end + + def value + @value + end + + def resolve(table) + @value.gsub(%r<\$([^/]+)>) { table[$1] } + end + + def set(val) + @value = check(val) + end + + private + + def check(val) + setup_rb_error "config: --#{name} requires argument" unless val + val + end + end + + class BoolItem < Item + def config_type + 'bool' + end + + def help_opt + "--#{@name}" + end + + private + + def check(val) + return 'yes' unless val + case val + when /\Ay(es)?\z/i, /\At(rue)?\z/i then 'yes' + when /\An(o)?\z/i, /\Af(alse)\z/i then 'no' + else + setup_rb_error "config: --#{@name} accepts only yes/no for argument" + end + end + end + + class PathItem < Item + def config_type + 'path' + end + + private + + def check(path) + setup_rb_error "config: --#{@name} requires argument" unless path + path[0,1] == '$' ? path : File.expand_path(path) + end + end + + class ProgramItem < Item + def config_type + 'program' + end + end + + class SelectItem < Item + def initialize(name, selection, default, desc) + super + @ok = selection.split('/') + end + + def config_type + 'select' + end + + private + + def check(val) + unless @ok.include?(val.strip) + setup_rb_error "config: use --#{@name}=#{@template} (#{val})" + end + val.strip + end + end + + class ExecItem < Item + def initialize(name, selection, desc, &block) + super name, selection, nil, desc + @ok = selection.split('/') + @action = block + end + + def config_type + 'exec' + end + + def value? + false + end + + def resolve(table) + setup_rb_error "$#{name()} wrongly used as option value" + end + + undef set + + def evaluate(val, table) + v = val.strip.downcase + unless @ok.include?(v) + setup_rb_error "invalid option --#{@name}=#{val} (use #{@template})" + end + @action.call v, table + end + end + + class PackageSelectionItem < Item + def initialize(name, template, default, help_default, desc) + super name, template, default, desc + @help_default = help_default + end + + attr_reader :help_default + + def config_type + 'package' + end + + private + + def check(val) + unless File.dir?("packages/#{val}") + setup_rb_error "config: no such package: #{val}" + end + val + end + end + + class MetaConfigEnvironment + def initialize(config, installer) + @config = config + @installer = installer + end + + def config_names + @config.names + end + + def config?(name) + @config.key?(name) + end + + def bool_config?(name) + @config.lookup(name).config_type == 'bool' + end + + def path_config?(name) + @config.lookup(name).config_type == 'path' + end + + def value_config?(name) + @config.lookup(name).config_type != 'exec' + end + + def add_config(item) + @config.add item + end + + def add_bool_config(name, default, desc) + @config.add BoolItem.new(name, 'yes/no', default ? 'yes' : 'no', desc) + end + + def add_path_config(name, default, desc) + @config.add PathItem.new(name, 'path', default, desc) + end + + def set_config_default(name, default) + @config.lookup(name).default = default + end + + def remove_config(name) + @config.remove(name) + end + + # For only multipackage + def packages + raise '[setup.rb fatal] multi-package metaconfig API packages() called for single-package; contact application package vendor' unless @installer + @installer.packages + end + + # For only multipackage + def declare_packages(list) + raise '[setup.rb fatal] multi-package metaconfig API declare_packages() called for single-package; contact application package vendor' unless @installer + @installer.packages = list + end + end + +end # class ConfigTable + + +# This module requires: #verbose?, #no_harm? +module FileOperations + + def mkdir_p(dirname, prefix = nil) + dirname = prefix + File.expand_path(dirname) if prefix + $stderr.puts "mkdir -p #{dirname}" if verbose? + return if no_harm? + + # Does not check '/', it's too abnormal. + dirs = File.expand_path(dirname).split(%r<(?=/)>) + if /\A[a-z]:\z/i =~ dirs[0] + disk = dirs.shift + dirs[0] = disk + dirs[0] + end + dirs.each_index do |idx| + path = dirs[0..idx].join('') + Dir.mkdir path unless File.dir?(path) + end + end + + def rm_f(path) + $stderr.puts "rm -f #{path}" if verbose? + return if no_harm? + force_remove_file path + end + + def rm_rf(path) + $stderr.puts "rm -rf #{path}" if verbose? + return if no_harm? + remove_tree path + end + + def remove_tree(path) + if File.symlink?(path) + remove_file path + elsif File.dir?(path) + remove_tree0 path + else + force_remove_file path + end + end + + def remove_tree0(path) + Dir.foreach(path) do |ent| + next if ent == '.' + next if ent == '..' + entpath = "#{path}/#{ent}" + if File.symlink?(entpath) + remove_file entpath + elsif File.dir?(entpath) + remove_tree0 entpath + else + force_remove_file entpath + end + end + begin + Dir.rmdir path + rescue Errno::ENOTEMPTY + # directory may not be empty + end + end + + def move_file(src, dest) + force_remove_file dest + begin + File.rename src, dest + rescue + File.open(dest, 'wb') {|f| + f.write File.binread(src) + } + File.chmod File.stat(src).mode, dest + File.unlink src + end + end + + def force_remove_file(path) + begin + remove_file path + rescue + end + end + + def remove_file(path) + File.chmod 0777, path + File.unlink path + end + + def install(from, dest, mode, prefix = nil) + $stderr.puts "install #{from} #{dest}" if verbose? + return if no_harm? + + realdest = prefix ? prefix + File.expand_path(dest) : dest + realdest = File.join(realdest, File.basename(from)) if File.dir?(realdest) + str = File.binread(from) + if diff?(str, realdest) + verbose_off { + rm_f realdest if File.exist?(realdest) + } + File.open(realdest, 'wb') {|f| + f.write str + } + File.chmod mode, realdest + + File.open("#{objdir_root()}/InstalledFiles", 'a') {|f| + if prefix + f.puts realdest.sub(prefix, '') + else + f.puts realdest + end + } + end + end + + def diff?(new_content, path) + return true unless File.exist?(path) + new_content != File.binread(path) + end + + def command(*args) + $stderr.puts args.join(' ') if verbose? + system(*args) or raise RuntimeError, + "system(#{args.map{|a| a.inspect }.join(' ')}) failed" + end + + def ruby(*args) + command config('rubyprog'), *args + end + + def make(task = nil) + command(*[config('makeprog'), task].compact) + end + + def extdir?(dir) + File.exist?("#{dir}/MANIFEST") or File.exist?("#{dir}/extconf.rb") + end + + def files_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.file?("#{dir}/#{ent}") } + } + end + + DIR_REJECT = %w( . .. CVS SCCS RCS CVS.adm .svn ) + + def directories_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.dir?("#{dir}/#{ent}") } - DIR_REJECT + } + end + +end + + +# This module requires: #srcdir_root, #objdir_root, #relpath +module HookScriptAPI + + def get_config(key) + @config[key] + end + + alias config get_config + + # obsolete: use metaconfig to change configuration + def set_config(key, val) + @config[key] = val + end + + # + # srcdir/objdir (works only in the package directory) + # + + def curr_srcdir + "#{srcdir_root()}/#{relpath()}" + end + + def curr_objdir + "#{objdir_root()}/#{relpath()}" + end + + def srcfile(path) + "#{curr_srcdir()}/#{path}" + end + + def srcexist?(path) + File.exist?(srcfile(path)) + end + + def srcdirectory?(path) + File.dir?(srcfile(path)) + end + + def srcfile?(path) + File.file?(srcfile(path)) + end + + def srcentries(path = '.') + Dir.open("#{curr_srcdir()}/#{path}") {|d| + return d.to_a - %w(. ..) + } + end + + def srcfiles(path = '.') + srcentries(path).select {|fname| + File.file?(File.join(curr_srcdir(), path, fname)) + } + end + + def srcdirectories(path = '.') + srcentries(path).select {|fname| + File.dir?(File.join(curr_srcdir(), path, fname)) + } + end + +end + + +class ToplevelInstaller + + Version = '3.4.1' + Copyright = 'Copyright (c) 2000-2005 Minero Aoki' + + TASKS = [ + [ 'all', 'do config, setup, then install' ], + [ 'config', 'saves your configurations' ], + [ 'show', 'shows current configuration' ], + [ 'setup', 'compiles ruby extentions and others' ], + [ 'install', 'installs files' ], + [ 'test', 'run all tests in test/' ], + [ 'clean', "does `make clean' for each extention" ], + [ 'distclean',"does `make distclean' for each extention" ] + ] + + def ToplevelInstaller.invoke + config = ConfigTable.new(load_rbconfig()) + config.load_standard_entries + config.load_multipackage_entries if multipackage? + config.fixup + klass = (multipackage?() ? ToplevelInstallerMulti : ToplevelInstaller) + klass.new(File.dirname($0), config).invoke + end + + def ToplevelInstaller.multipackage? + File.dir?(File.dirname($0) + '/packages') + end + + def ToplevelInstaller.load_rbconfig + if arg = ARGV.detect {|arg| /\A--rbconfig=/ =~ arg } + ARGV.delete(arg) + load File.expand_path(arg.split(/=/, 2)[1]) + $".push 'rbconfig.rb' + else + require 'rbconfig' + end + ::Config::CONFIG + end + + def initialize(ardir_root, config) + @ardir = File.expand_path(ardir_root) + @config = config + # cache + @valid_task_re = nil + end + + def config(key) + @config[key] + end + + def inspect + "#<#{self.class} #{__id__()}>" + end + + def invoke + run_metaconfigs + case task = parsearg_global() + when nil, 'all' + parsearg_config + init_installers + exec_config + exec_setup + exec_install + else + case task + when 'config', 'test' + ; + when 'clean', 'distclean' + @config.load_savefile if File.exist?(@config.savefile) + else + @config.load_savefile + end + __send__ "parsearg_#{task}" + init_installers + __send__ "exec_#{task}" + end + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig" + end + + def init_installers + @installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + # + # Hook Script API bases + # + + def srcdir_root + @ardir + end + + def objdir_root + '.' + end + + def relpath + '.' + end + + # + # Option Parsing + # + + def parsearg_global + while arg = ARGV.shift + case arg + when /\A\w+\z/ + setup_rb_error "invalid task: #{arg}" unless valid_task?(arg) + return arg + when '-q', '--quiet' + @config.verbose = false + when '--verbose' + @config.verbose = true + when '--help' + print_usage $stdout + exit 0 + when '--version' + puts "#{File.basename($0)} version #{Version}" + exit 0 + when '--copyright' + puts Copyright + exit 0 + else + setup_rb_error "unknown global option '#{arg}'" + end + end + nil + end + + def valid_task?(t) + valid_task_re() =~ t + end + + def valid_task_re + @valid_task_re ||= /\A(?:#{TASKS.map {|task,desc| task }.join('|')})\z/ + end + + def parsearg_no_options + unless ARGV.empty? + task = caller(0).first.slice(%r<`parsearg_(\w+)'>, 1) + setup_rb_error "#{task}: unknown options: #{ARGV.join(' ')}" + end + end + + alias parsearg_show parsearg_no_options + alias parsearg_setup parsearg_no_options + alias parsearg_test parsearg_no_options + alias parsearg_clean parsearg_no_options + alias parsearg_distclean parsearg_no_options + + def parsearg_config + evalopt = [] + set = [] + @config.config_opt = [] + while i = ARGV.shift + if /\A--?\z/ =~ i + @config.config_opt = ARGV.dup + break + end + name, value = *@config.parse_opt(i) + if @config.value_config?(name) + @config[name] = value + else + evalopt.push [name, value] + end + set.push name + end + evalopt.each do |name, value| + @config.lookup(name).evaluate value, @config + end + # Check if configuration is valid + set.each do |n| + @config[n] if @config.value_config?(n) + end + end + + def parsearg_install + @config.no_harm = false + @config.install_prefix = '' + while a = ARGV.shift + case a + when '--no-harm' + @config.no_harm = true + when /\A--prefix=/ + path = a.split(/=/, 2)[1] + path = File.expand_path(path) unless path[0,1] == '/' + @config.install_prefix = path + else + setup_rb_error "install: unknown option #{a}" + end + end + end + + def print_usage(out) + out.puts 'Typical Installation Procedure:' + out.puts " $ ruby #{File.basename $0} config" + out.puts " $ ruby #{File.basename $0} setup" + out.puts " # ruby #{File.basename $0} install (may require root privilege)" + out.puts + out.puts 'Detailed Usage:' + out.puts " ruby #{File.basename $0} <global option>" + out.puts " ruby #{File.basename $0} [<global options>] <task> [<task options>]" + + fmt = " %-24s %s\n" + out.puts + out.puts 'Global options:' + out.printf fmt, '-q,--quiet', 'suppress message outputs' + out.printf fmt, ' --verbose', 'output messages verbosely' + out.printf fmt, ' --help', 'print this message' + out.printf fmt, ' --version', 'print version and quit' + out.printf fmt, ' --copyright', 'print copyright and quit' + out.puts + out.puts 'Tasks:' + TASKS.each do |name, desc| + out.printf fmt, name, desc + end + + fmt = " %-24s %s [%s]\n" + out.puts + out.puts 'Options for CONFIG or ALL:' + @config.each do |item| + out.printf fmt, item.help_opt, item.description, item.help_default + end + out.printf fmt, '--rbconfig=path', 'rbconfig.rb to load',"running ruby's" + out.puts + out.puts 'Options for INSTALL:' + out.printf fmt, '--no-harm', 'only display what to do if given', 'off' + out.printf fmt, '--prefix=path', 'install path prefix', '' + out.puts + end + + # + # Task Handlers + # + + def exec_config + @installer.exec_config + @config.save # must be final + end + + def exec_setup + @installer.exec_setup + end + + def exec_install + @installer.exec_install + end + + def exec_test + @installer.exec_test + end + + def exec_show + @config.each do |i| + printf "%-20s %s\n", i.name, i.value if i.value? + end + end + + def exec_clean + @installer.exec_clean + end + + def exec_distclean + @installer.exec_distclean + end + +end # class ToplevelInstaller + + +class ToplevelInstallerMulti < ToplevelInstaller + + include FileOperations + + def initialize(ardir_root, config) + super + @packages = directories_of("#{@ardir}/packages") + raise 'no package exists' if @packages.empty? + @root_installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig", self + @packages.each do |name| + @config.load_script "#{@ardir}/packages/#{name}/metaconfig" + end + end + + attr_reader :packages + + def packages=(list) + raise 'package list is empty' if list.empty? + list.each do |name| + raise "directory packages/#{name} does not exist"\ + unless File.dir?("#{@ardir}/packages/#{name}") + end + @packages = list + end + + def init_installers + @installers = {} + @packages.each do |pack| + @installers[pack] = Installer.new(@config, + "#{@ardir}/packages/#{pack}", + "packages/#{pack}") + end + with = extract_selection(config('with')) + without = extract_selection(config('without')) + @selected = @installers.keys.select {|name| + (with.empty? or with.include?(name)) \ + and not without.include?(name) + } + end + + def extract_selection(list) + a = list.split(/,/) + a.each do |name| + setup_rb_error "no such package: #{name}" unless @installers.key?(name) + end + a + end + + def print_usage(f) + super + f.puts 'Inluded packages:' + f.puts ' ' + @packages.sort.join(' ') + f.puts + end + + # + # Task Handlers + # + + def exec_config + run_hook 'pre-config' + each_selected_installers {|inst| inst.exec_config } + run_hook 'post-config' + @config.save # must be final + end + + def exec_setup + run_hook 'pre-setup' + each_selected_installers {|inst| inst.exec_setup } + run_hook 'post-setup' + end + + def exec_install + run_hook 'pre-install' + each_selected_installers {|inst| inst.exec_install } + run_hook 'post-install' + end + + def exec_test + run_hook 'pre-test' + each_selected_installers {|inst| inst.exec_test } + run_hook 'post-test' + end + + def exec_clean + rm_f @config.savefile + run_hook 'pre-clean' + each_selected_installers {|inst| inst.exec_clean } + run_hook 'post-clean' + end + + def exec_distclean + rm_f @config.savefile + run_hook 'pre-distclean' + each_selected_installers {|inst| inst.exec_distclean } + run_hook 'post-distclean' + end + + # + # lib + # + + def each_selected_installers + Dir.mkdir 'packages' unless File.dir?('packages') + @selected.each do |pack| + $stderr.puts "Processing the package `#{pack}' ..." if verbose? + Dir.mkdir "packages/#{pack}" unless File.dir?("packages/#{pack}") + Dir.chdir "packages/#{pack}" + yield @installers[pack] + Dir.chdir '../..' + end + end + + def run_hook(id) + @root_installer.run_hook id + end + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + +end # class ToplevelInstallerMulti + + +class Installer + + FILETYPES = %w( bin lib ext data conf man ) + + include FileOperations + include HookScriptAPI + + def initialize(config, srcroot, objroot) + @config = config + @srcdir = File.expand_path(srcroot) + @objdir = File.expand_path(objroot) + @currdir = '.' + end + + def inspect + "#<#{self.class} #{File.basename(@srcdir)}>" + end + + def noop(rel) + end + + # + # Hook Script API base methods + # + + def srcdir_root + @srcdir + end + + def objdir_root + @objdir + end + + def relpath + @currdir + end + + # + # Config Access + # + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + + def verbose_off + begin + save, @config.verbose = @config.verbose?, false + yield + ensure + @config.verbose = save + end + end + + # + # TASK config + # + + def exec_config + exec_task_traverse 'config' + end + + alias config_dir_bin noop + alias config_dir_lib noop + + def config_dir_ext(rel) + extconf if extdir?(curr_srcdir()) + end + + alias config_dir_data noop + alias config_dir_conf noop + alias config_dir_man noop + + def extconf + ruby "#{curr_srcdir()}/extconf.rb", *@config.config_opt + end + + # + # TASK setup + # + + def exec_setup + exec_task_traverse 'setup' + end + + def setup_dir_bin(rel) + files_of(curr_srcdir()).each do |fname| + update_shebang_line "#{curr_srcdir()}/#{fname}" + end + end + + alias setup_dir_lib noop + + def setup_dir_ext(rel) + make if extdir?(curr_srcdir()) + end + + alias setup_dir_data noop + alias setup_dir_conf noop + alias setup_dir_man noop + + def update_shebang_line(path) + return if no_harm? + return if config('shebang') == 'never' + old = Shebang.load(path) + if old + $stderr.puts "warning: #{path}: Shebang line includes too many args. It is not portable and your program may not work." if old.args.size > 1 + new = new_shebang(old) + return if new.to_s == old.to_s + else + return unless config('shebang') == 'all' + new = Shebang.new(config('rubypath')) + end + $stderr.puts "updating shebang: #{File.basename(path)}" if verbose? + open_atomic_writer(path) {|output| + File.open(path, 'rb') {|f| + f.gets if old # discard + output.puts new.to_s + output.print f.read + } + } + end + + def new_shebang(old) + if /\Aruby/ =~ File.basename(old.cmd) + Shebang.new(config('rubypath'), old.args) + elsif File.basename(old.cmd) == 'env' and old.args.first == 'ruby' + Shebang.new(config('rubypath'), old.args[1..-1]) + else + return old unless config('shebang') == 'all' + Shebang.new(config('rubypath')) + end + end + + def open_atomic_writer(path, &block) + tmpfile = File.basename(path) + '.tmp' + begin + File.open(tmpfile, 'wb', &block) + File.rename tmpfile, File.basename(path) + ensure + File.unlink tmpfile if File.exist?(tmpfile) + end + end + + class Shebang + def Shebang.load(path) + line = nil + File.open(path) {|f| + line = f.gets + } + return nil unless /\A#!/ =~ line + parse(line) + end + + def Shebang.parse(line) + cmd, *args = *line.strip.sub(/\A\#!/, '').split(' ') + new(cmd, args) + end + + def initialize(cmd, args = []) + @cmd = cmd + @args = args + end + + attr_reader :cmd + attr_reader :args + + def to_s + "#! #{@cmd}" + (@args.empty? ? '' : " #{@args.join(' ')}") + end + end + + # + # TASK install + # + + def exec_install + rm_f 'InstalledFiles' + exec_task_traverse 'install' + end + + def install_dir_bin(rel) + install_files targetfiles(), "#{config('bindir')}/#{rel}", 0755 + end + + def install_dir_lib(rel) + install_files libfiles(), "#{config('rbdir')}/#{rel}", 0644 + end + + def install_dir_ext(rel) + return unless extdir?(curr_srcdir()) + install_files rubyextentions('.'), + "#{config('sodir')}/#{File.dirname(rel)}", + 0555 + end + + def install_dir_data(rel) + install_files targetfiles(), "#{config('datadir')}/#{rel}", 0644 + end + + def install_dir_conf(rel) + # FIXME: should not remove current config files + # (rename previous file to .old/.org) + install_files targetfiles(), "#{config('sysconfdir')}/#{rel}", 0644 + end + + def install_dir_man(rel) + install_files targetfiles(), "#{config('mandir')}/#{rel}", 0644 + end + + def install_files(list, dest, mode) + mkdir_p dest, @config.install_prefix + list.each do |fname| + install fname, dest, mode, @config.install_prefix + end + end + + def libfiles + glob_reject(%w(*.y *.output), targetfiles()) + end + + def rubyextentions(dir) + ents = glob_select("*.#{@config.dllext}", targetfiles()) + if ents.empty? + setup_rb_error "no ruby extention exists: 'ruby #{$0} setup' first" + end + ents + end + + def targetfiles + mapdir(existfiles() - hookfiles()) + end + + def mapdir(ents) + ents.map {|ent| + if File.exist?(ent) + then ent # objdir + else "#{curr_srcdir()}/#{ent}" # srcdir + end + } + end + + # picked up many entries from cvs-1.11.1/src/ignore.c + JUNK_FILES = %w( + core RCSLOG tags TAGS .make.state + .nse_depinfo #* .#* cvslog.* ,* .del-* *.olb + *~ *.old *.bak *.BAK *.orig *.rej _$* *$ + + *.org *.in .* + ) + + def existfiles + glob_reject(JUNK_FILES, (files_of(curr_srcdir()) | files_of('.'))) + end + + def hookfiles + %w( pre-%s post-%s pre-%s.rb post-%s.rb ).map {|fmt| + %w( config setup install clean ).map {|t| sprintf(fmt, t) } + }.flatten + end + + def glob_select(pat, ents) + re = globs2re([pat]) + ents.select {|ent| re =~ ent } + end + + def glob_reject(pats, ents) + re = globs2re(pats) + ents.reject {|ent| re =~ ent } + end + + GLOB2REGEX = { + '.' => '\.', + '$' => '\$', + '#' => '\#', + '*' => '.*' + } + + def globs2re(pats) + /\A(?:#{ + pats.map {|pat| pat.gsub(/[\.\$\#\*]/) {|ch| GLOB2REGEX[ch] } }.join('|') + })\z/ + end + + # + # TASK test + # + + TESTDIR = 'test' + + def exec_test + unless File.directory?('test') + $stderr.puts 'no test in this package' if verbose? + return + end + $stderr.puts 'Running tests...' if verbose? + begin + require 'test/unit' + rescue LoadError + setup_rb_error 'test/unit cannot loaded. You need Ruby 1.8 or later to invoke this task.' + end + runner = Test::Unit::AutoRunner.new(true) + runner.to_run << TESTDIR + runner.run + end + + # + # TASK clean + # + + def exec_clean + exec_task_traverse 'clean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias clean_dir_bin noop + alias clean_dir_lib noop + alias clean_dir_data noop + alias clean_dir_conf noop + alias clean_dir_man noop + + def clean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'clean' if File.file?('Makefile') + end + + # + # TASK distclean + # + + def exec_distclean + exec_task_traverse 'distclean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias distclean_dir_bin noop + alias distclean_dir_lib noop + + def distclean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'distclean' if File.file?('Makefile') + end + + alias distclean_dir_data noop + alias distclean_dir_conf noop + alias distclean_dir_man noop + + # + # Traversing + # + + def exec_task_traverse(task) + run_hook "pre-#{task}" + FILETYPES.each do |type| + if type == 'ext' and config('without-ext') == 'yes' + $stderr.puts 'skipping ext/* by user option' if verbose? + next + end + traverse task, type, "#{task}_dir_#{type}" + end + run_hook "post-#{task}" + end + + def traverse(task, rel, mid) + dive_into(rel) { + run_hook "pre-#{task}" + __send__ mid, rel.sub(%r[\A.*?(?:/|\z)], '') + directories_of(curr_srcdir()).each do |d| + traverse task, "#{rel}/#{d}", mid + end + run_hook "post-#{task}" + } + end + + def dive_into(rel) + return unless File.dir?("#{@srcdir}/#{rel}") + + dir = File.basename(rel) + Dir.mkdir dir unless File.dir?(dir) + prevdir = Dir.pwd + Dir.chdir dir + $stderr.puts '---> ' + rel if verbose? + @currdir = rel + yield + Dir.chdir prevdir + $stderr.puts '<--- ' + rel if verbose? + @currdir = File.dirname(rel) + end + + def run_hook(id) + path = [ "#{curr_srcdir()}/#{id}", + "#{curr_srcdir()}/#{id}.rb" ].detect {|cand| File.file?(cand) } + return unless path + begin + instance_eval File.read(path), path, 1 + rescue + raise if $DEBUG + setup_rb_error "hook #{path} failed:\n" + $!.message + end + end + +end # class Installer + + +class SetupError < StandardError; end + +def setup_rb_error(msg) + raise SetupError, msg +end + +if $0 == __FILE__ + begin + ToplevelInstaller.invoke + rescue SetupError + raise if $DEBUG + $stderr.puts $!.message + $stderr.puts "Try 'ruby #{$0} --help' for detailed usage." + exit 1 + end +end diff --git a/test/test_tokyocabinet_hdb.rb b/test/test_tokyocabinet_hdb.rb new file mode 100644 index 0000000..31a77d7 --- /dev/null +++ b/test/test_tokyocabinet_hdb.rb @@ -0,0 +1,200 @@ +# -*- encoding: binary -*- +require 'test/unit' +require 'tempfile' +require 'stringio' +require 'tokyocabinet' # FIXME: emits warning with 1.29 gem +$-w = true +require 'metropolis' + +class TestTokyocabinetHDB < Test::Unit::TestCase + attr_reader :tmp, :o + + def setup + tmp = Tempfile.new('tchdb') + @path_pattern = tmp.path + ".%01x.tch" + tmp.close! + end + + def teardown + Dir[@path_pattern.sub!(/%\d*x/, '*')].each { |x| File.unlink(x) } + end + + def osetup + o = Object.new + o.extend Metropolis::TokyoCabinet::HDB + assert_nothing_raised do + o.setup :path_pattern => @path_pattern + end + o + end + + def test_create_put_get_delete + o = osetup + r = o.put('hello', { 'rack.input' => StringIO.new('world') }) + assert_equal 201, r[0].to_i + assert_equal 'text/plain', r[1]['Content-Type'] + assert_equal '8', r[1]['Content-Length'] + assert_equal "Created\n", r[2].join('') + + r = o.put('hellox', { 'rack.input' => StringIO.new('worldx') }) + assert_equal 201, r[0].to_i + assert_equal 'text/plain', r[1]['Content-Type'] + assert_equal '8', r[1]['Content-Length'] + assert_equal "Created\n", r[2].join('') + + r = o.get('hello') + assert_equal 200, r[0].to_i + assert_equal 'application/octet-stream', r[1]['Content-Type'] + assert_equal '5', r[1]['Content-Length'] + assert_equal %w(world), r[2] + + r = o.head('hello') + assert_equal 200, r[0].to_i + assert_equal 'application/octet-stream', r[1]['Content-Type'] + assert_equal '5', r[1]['Content-Length'] + assert_equal [], r[2] + + r = o.get('hellox') + assert_equal 200, r[0].to_i + assert_equal 'application/octet-stream', r[1]['Content-Type'] + assert_equal '6', r[1]['Content-Length'] + assert_equal %w(worldx), r[2] + + r = o.delete('hellox') + assert_equal 200, r[0].to_i + assert_equal 'text/plain', r[1]['Content-Type'] + assert_equal '3', r[1]['Content-Length'] + assert_equal "OK\n", r[2].join('') + + r = o.delete('hellox') + assert_equal 404, r[0].to_i + assert_equal 'text/plain', r[1]['Content-Type'] + assert_equal '10', r[1]['Content-Length'] + assert_equal "Not Found\n", r[2].join('') + + r = o.get('hellox') + assert_equal 404, r[0].to_i + assert_equal 'text/plain', r[1]['Content-Type'] + assert_equal '10', r[1]['Content-Length'] + assert_equal "Not Found\n", r[2].join('') + + r = o.head('hellox') + assert_equal 404, r[0].to_i + assert_equal 'text/plain', r[1]['Content-Type'] + assert_equal '10', r[1]['Content-Length'] + assert_equal "Not Found\n", r[2].join('') + end + + def test_putkeep + o = osetup + env = { + "rack.input" => StringIO.new("hello"), + "HTTP_X_TT_PDMODE" => "1" + } + assert_equal 201, o.put("x", env)[0] + env["rack.input"] = StringIO.new("wrong") + assert_equal 409, o.put("x", env)[0] + assert_equal "hello", o.get("x")[2].join('') + end + + def test_putcat + o = osetup + env = { + "rack.input" => StringIO.new("hello"), + "HTTP_X_TT_PDMODE" => "2" + } + assert_equal 201, o.put("x", env)[0] + env["rack.input"] = StringIO.new("MOAR") + assert_equal 201, o.put("x", env)[0] + assert_equal "helloMOAR", o.get("x")[2].join('') + end + + def test_multiproc + nr = 2 + key = "k" + str = "." * (1024 * 1024) + nr.times { + fork { + o = osetup + sio = StringIO.new(str) + env = { "rack.input" => sio } + 100.times { + o.put(key, env) + sio.rewind + o.get(key) + } + } + } + res = Process.waitall + assert_equal nr, res.size + res.each { |(pid, status)| assert status.success? } + end + + def test_readonly + key = "x" + wr = osetup + wr.put(key, { "rack.input" => StringIO.new("OK") }) + o = Object.new + o.extend Metropolis::TokyoCabinet::HDB + assert_nothing_raised do + o.setup :path_pattern => @path_pattern, :read_only => true + end + %w(PUT DELETE).each do |rm| + env = { + "rack.input" => StringIO.new("FAIL"), + "REQUEST_METHOD" => rm, + "PATH_INFO" => "/#{key}" + } + assert_equal 405, o.call(env)[0] + end + env = { + "REQUEST_METHOD" => "GET", + "PATH_INFO" => "/#{key}", + } + assert_equal 200, o.call(env)[0] + assert_equal '2', o.call(env)[1]["Content-Length"] + assert_equal 'application/octet-stream', o.call(env)[1]["Content-Type"] + assert_equal "OK", o.call(env)[2].join('') + + env["REQUEST_METHOD"] = "HEAD" + assert_equal 200, o.call(env)[0] + assert_equal '2', o.call(env)[1]["Content-Length"] + assert_equal 'application/octet-stream', o.call(env)[1]["Content-Type"] + assert_equal "", o.call(env)[2].join('') + end + + def test_create_toplevel + k = "x" + nr_bytes = 1024 * 1024 * 20 + data = "0" * nr_bytes + uri = "tc://#{@path_pattern}" + obj = nil + assert_nothing_raised { obj = Metropolis.new(:uri => uri) } + + query = "large=true&apow=3&bnum=65536&compress=deflate" + assert_nothing_raised { + obj = Metropolis.new(:uri => "#{uri}?#{query}") + } + optimize_args = obj.instance_variable_get(:@optimize) + flags = ::TokyoCabinet::HDB::TLARGE | ::TokyoCabinet::HDB::TDEFLATE + assert_equal flags, optimize_args[3] + assert_equal 65536, optimize_args[0] + assert_nil optimize_args[2] + assert_equal 3, optimize_args[1] + assert_nothing_raised { obj.get(k) } + assert_nothing_raised { obj.put(k,{'rack.input' => StringIO.new(data)}) } + + obj = Metropolis.new(:uri => "#{uri}?#{query}", :read_only => true) + assert_equal data, obj.get(k)[2].join('') + obj.close! + + obj = Metropolis.new(:uri => uri, :read_only => true) + assert_equal data, obj.get(k)[2].join('') + obj.close! + sum = obj.instance_eval { + @hdbv.inject(0) { |size, (hdb,path)| size += File.stat(path).size } + } + assert sum <= nr_bytes, "#{sum} > #{nr_bytes}" + obj.close! + end +end |