diff --git a/cc-by-sa.txt b/cc-by-sa.txt new file mode 100644 index 00000000..a5c50375 --- /dev/null +++ b/cc-by-sa.txt @@ -0,0 +1,542 @@ + + + + + + + Creative Commons Legal Code + + + + + + + + + + + +
+
+ + +

Creative Commons Legal Code

+ +
+

Attribution-ShareAlike 3.0 Unported

+
+
+ +
+
+ + +
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES + NOT PROVIDE LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE + DOES NOT CREATE AN ATTORNEY-CLIENT RELATIONSHIP. CREATIVE + COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS. + CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE + INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR DAMAGES + RESULTING FROM ITS USE. +
+ +

License

+ +

THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS + OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR + "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER + APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS + AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS + PROHIBITED.

+ +

BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU + ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. + TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A + CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE + IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND + CONDITIONS.

+ +

1. Definitions

+ +
    +
  1. "Adaptation" means a work based upon + the Work, or upon the Work and other pre-existing works, + such as a translation, adaptation, derivative work, + arrangement of music or other alterations of a literary + or artistic work, or phonogram or performance and + includes cinematographic adaptations or any other form in + which the Work may be recast, transformed, or adapted + including in any form recognizably derived from the + original, except that a work that constitutes a + Collection will not be considered an Adaptation for the + purpose of this License. For the avoidance of doubt, + where the Work is a musical work, performance or + phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be + considered an Adaptation for the purpose of this + License.
    2. + +
  2. "Collection" means a collection of + literary or artistic works, such as encyclopedias and + anthologies, or performances, phonograms or broadcasts, + or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection + and arrangement of their contents, constitute + intellectual creations, in which the Work is included in + its entirety in unmodified form along with one or more + other contributions, each constituting separate and + independent works in themselves, which together are + assembled into a collective whole. A work that + constitutes a Collection will not be considered an + Adaptation (as defined below) for the purposes of this + License.
    3. + +
  3. "Creative Commons Compatible + License" means a license that is listed at + http://creativecommons.org/compatiblelicenses that has + been approved by Creative Commons as being essentially + equivalent to this License, including, at a minimum, + because that license: (i) contains terms that have the + same purpose, meaning and effect as the License Elements + of this License; and, (ii) explicitly permits the + relicensing of adaptations of works made available under + that license under this License or a Creative Commons + jurisdiction license with the same License Elements as + this License.
    4. + +
  4. "Distribute" means to make available + to the public the original and copies of the Work or + Adaptation, as appropriate, through sale or other + transfer of ownership.
    5. + +
  5. "License Elements" means the + following high-level license attributes as selected by + Licensor and indicated in the title of this License: + Attribution, ShareAlike.
    6. + +
  6. "Licensor" means the individual, + individuals, entity or entities that offer(s) the Work + under the terms of this License.
    7. + +
  7. "Original Author" means, in the case + of a literary or artistic work, the individual, + individuals, entity or entities who created the Work or + if no individual or entity can be identified, the + publisher; and in addition (i) in the case of a + performance the actors, singers, musicians, dancers, and + other persons who act, sing, deliver, declaim, play in, + interpret or otherwise perform literary or artistic works + or expressions of folklore; (ii) in the case of a + phonogram the producer being the person or legal entity + who first fixes the sounds of a performance or other + sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast.
    8. + +
  8. "Work" means the literary and/or + artistic work offered under the terms of this License + including without limitation any production in the + literary, scientific and artistic domain, whatever may be + the mode or form of its expression including digital + form, such as a book, pamphlet and other writing; a + lecture, address, sermon or other work of the same + nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a + musical composition with or without words; a + cinematographic work to which are assimilated works + expressed by a process analogous to cinematography; a + work of drawing, painting, architecture, sculpture, + engraving or lithography; a photographic work to which + are assimilated works expressed by a process analogous to + photography; a work of applied art; an illustration, map, + plan, sketch or three-dimensional work relative to + geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of + data to the extent it is protected as a copyrightable + work; or a work performed by a variety or circus + performer to the extent it is not otherwise considered a + literary or artistic work.
    9. + +
  9. "You" means an individual or entity + exercising rights under this License who has not + previously violated the terms of this License with + respect to the Work, or who has received express + permission from the Licensor to exercise rights under + this License despite a previous violation.
    10. + +
  10. "Publicly Perform" means to perform + public recitations of the Work and to communicate to the + public those public recitations, by any means or process, + including by wire or wireless means or public digital + performances; to make available to the public Works in + such a way that members of the public may access these + Works from a place and at a place individually chosen by + them; to perform the Work to the public by any means or + process and the communication to the public of the + performances of the Work, including by public digital + performance; to broadcast and rebroadcast the Work by any + means including signs, sounds or images.
    11. + +
  11. "Reproduce" means to make copies of + the Work by any means including without limitation by + sound or visual recordings and the right of fixation and + reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or + other electronic medium.
    12. +
+ +

2. Fair Dealing Rights. Nothing in this + License is intended to reduce, limit, or restrict any uses + free from copyright or rights arising from limitations or + exceptions that are provided for in connection with the + copyright protection under copyright law or other + applicable laws.

+ +

3. License Grant. Subject to the terms + and conditions of this License, Licensor hereby grants You + a worldwide, royalty-free, non-exclusive, perpetual (for + the duration of the applicable copyright) license to + exercise the rights in the Work as stated below:

+ +
    +
  1. to Reproduce the Work, to incorporate the Work into + one or more Collections, and to Reproduce the Work as + incorporated in the Collections;
    2. + +
  2. to create and Reproduce Adaptations provided that any + such Adaptation, including any translation in any medium, + takes reasonable steps to clearly label, demarcate or + otherwise identify that changes were made to the original + Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or + a modification could indicate "The original work has been + modified.";
    3. + +
  3. to Distribute and Publicly Perform the Work including + as incorporated in Collections; and,
    4. + +
  4. to Distribute and Publicly Perform Adaptations.
    5. + +
  5. +

    For the avoidance of doubt:

    + +
      +
    1. Non-waivable Compulsory License + Schemes. In those jurisdictions in which the + right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the + Licensor reserves the exclusive right to collect such + royalties for any exercise by You of the rights + granted under this License;
      2. + +
    2. Waivable Compulsory License + Schemes. In those jurisdictions in which the + right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the + Licensor waives the exclusive right to collect such + royalties for any exercise by You of the rights + granted under this License; and,
      3. + +
    3. Voluntary License Schemes. The + Licensor waives the right to collect royalties, + whether individually or, in the event that the + Licensor is a member of a collecting society that + administers voluntary licensing schemes, via that + society, from any exercise by You of the rights + granted under this License.
      4. +
    +
    6. +
+ +

The above rights may be exercised in all media and + formats whether now known or hereafter devised. The above + rights include the right to make such modifications as are + technically necessary to exercise the rights in other media + and formats. Subject to Section 8(f), all rights not + expressly granted by Licensor are hereby reserved.

+ +

4. Restrictions. The license granted in + Section 3 above is expressly made subject to and limited by + the following restrictions:

+ +
    +
  1. You may Distribute or Publicly Perform the Work only + under the terms of this License. You must include a copy + of, or the Uniform Resource Identifier (URI) for, this + License with every copy of the Work You Distribute or + Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or + the ability of the recipient of the Work to exercise the + rights granted to that recipient under the terms of the + License. You may not sublicense the Work. You must keep + intact all notices that refer to this License and to the + disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or + Publicly Perform the Work, You may not impose any + effective technological measures on the Work that + restrict the ability of a recipient of the Work from You + to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to + the Work as incorporated in a Collection, but this does + not require the Collection apart from the Work itself to + be made subject to the terms of this License. If You + create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the + Collection any credit as required by Section 4(c), as + requested. If You create an Adaptation, upon notice from + any Licensor You must, to the extent practicable, remove + from the Adaptation any credit as required by Section + 4(c), as requested.
    2. + +
  2. You may Distribute or Publicly Perform an Adaptation + only under the terms of: (i) this License; (ii) a later + version of this License with the same License Elements as + this License; (iii) a Creative Commons jurisdiction + license (either this or a later license version) that + contains the same License Elements as this License (e.g., + Attribution-ShareAlike 3.0 US)); (iv) a Creative Commons + Compatible License. If you license the Adaptation under + one of the licenses mentioned in (iv), you must comply + with the terms of that license. If you license the + Adaptation under the terms of any of the licenses + mentioned in (i), (ii) or (iii) (the "Applicable + License"), you must comply with the terms of the + Applicable License generally and the following + provisions: (I) You must include a copy of, or the URI + for, the Applicable License with every copy of each + Adaptation You Distribute or Publicly Perform; (II) You + may not offer or impose any terms on the Adaptation that + restrict the terms of the Applicable License or the + ability of the recipient of the Adaptation to exercise + the rights granted to that recipient under the terms of + the Applicable License; (III) You must keep intact all + notices that refer to the Applicable License and to the + disclaimer of warranties with every copy of the Work as + included in the Adaptation You Distribute or Publicly + Perform; (IV) when You Distribute or Publicly Perform the + Adaptation, You may not impose any effective + technological measures on the Adaptation that restrict + the ability of a recipient of the Adaptation from You to + exercise the rights granted to that recipient under the + terms of the Applicable License. This Section 4(b) + applies to the Adaptation as incorporated in a + Collection, but this does not require the Collection + apart from the Adaptation itself to be made subject to + the terms of the Applicable License.
    3. + +
  3. If You Distribute, or Publicly Perform the Work or + any Adaptations or Collections, You must, unless a + request has been made pursuant to Section 4(a), keep + intact all copyright notices for the Work and provide, + reasonable to the medium or means You are utilizing: (i) + the name of the Original Author (or pseudonym, if + applicable) if supplied, and/or if the Original Author + and/or Licensor designate another party or parties (e.g., + a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's + copyright notice, terms of service or by other reasonable + means, the name of such party or parties; (ii) the title + of the Work if supplied; (iii) to the extent reasonably + practicable, the URI, if any, that Licensor specifies to + be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information + for the Work; and (iv) , consistent with Ssection 3(b), + in the case of an Adaptation, a credit identifying the + use of the Work in the Adaptation (e.g., "French + translation of the Work by Original Author," or + "Screenplay based on original Work by Original Author"). + The credit required by this Section 4(c) may be + implemented in any reasonable manner; provided, however, + that in the case of a Adaptation or Collection, at a + minimum such credit will appear, if a credit for all + contributing authors of the Adaptation or Collection + appears, then as part of these credits and in a manner at + least as prominent as the credits for the other + contributing authors. For the avoidance of doubt, You may + only use the credit required by this Section for the + purpose of attribution in the manner set out above and, + by exercising Your rights under this License, You may not + implicitly or explicitly assert or imply any connection + with, sponsorship or endorsement by the Original Author, + Licensor and/or Attribution Parties, as appropriate, of + You or Your use of the Work, without the separate, + express prior written permission of the Original Author, + Licensor and/or Attribution Parties.
    4. + +
  4. Except as otherwise agreed in writing by the Licensor + or as may be otherwise permitted by applicable law, if + You Reproduce, Distribute or Publicly Perform the Work + either by itself or as part of any Adaptations or + Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work + which would be prejudicial to the Original Author's honor + or reputation. Licensor agrees that in those + jurisdictions (e.g. Japan), in which any exercise of the + right granted in Section 3(b) of this License (the right + to make Adaptations) would be deemed to be a distortion, + mutilation, modification or other derogatory action + prejudicial to the Original Author's honor and + reputation, the Licensor will waive or not assert, as + appropriate, this Section, to the fullest extent + permitted by the applicable national law, to enable You + to reasonably exercise Your right under Section 3(b) of + this License (right to make Adaptations) but not + otherwise.
    5. +
+ +

5. Representations, Warranties and + Disclaimer

+ +

UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN + WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO + REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE + WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, + WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, + FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE + ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE + PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. + SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED + WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.

+ +

6. Limitation on Liability. EXCEPT TO + THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL + LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY + SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY + DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, + EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF + SUCH DAMAGES.

+ +

7. Termination

+ +
    +
  1. This License and the rights granted hereunder will + terminate automatically upon any breach by You of the + terms of this License. Individuals or entities who have + received Adaptations or Collections from You under this + License, however, will not have their licenses terminated + provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, + and 8 will survive any termination of this License.
    2. + +
  2. Subject to the above terms and conditions, the + license granted here is perpetual (for the duration of + the applicable copyright in the Work). Notwithstanding + the above, Licensor reserves the right to release the + Work under different license terms or to stop + distributing the Work at any time; provided, however that + any such election will not serve to withdraw this License + (or any other license that has been, or is required to + be, granted under the terms of this License), and this + License will continue in full force and effect unless + terminated as stated above.
    3. +
+ +

8. Miscellaneous

+ +
    +
  1. Each time You Distribute or Publicly Perform the Work + or a Collection, the Licensor offers to the recipient a + license to the Work on the same terms and conditions as + the license granted to You under this License.
    2. + +
  2. Each time You Distribute or Publicly Perform an + Adaptation, Licensor offers to the recipient a license to + the original Work on the same terms and conditions as the + license granted to You under this License.
    3. + +
  3. If any provision of this License is invalid or + unenforceable under applicable law, it shall not affect + the validity or enforceability of the remainder of the + terms of this License, and without further action by the + parties to this agreement, such provision shall be + reformed to the minimum extent necessary to make such + provision valid and enforceable.
    4. + +
  4. No term or provision of this License shall be deemed + waived and no breach consented to unless such waiver or + consent shall be in writing and signed by the party to be + charged with such waiver or consent.
    5. + +
  5. This License constitutes the entire agreement between + the parties with respect to the Work licensed here. There + are no understandings, agreements or representations with + respect to the Work not specified here. Licensor shall + not be bound by any additional provisions that may appear + in any communication from You. This License may not be + modified without the mutual written agreement of the + Licensor and You.
    6. + +
  6. The rights granted under, and the subject matter + referenced, in this License were drafted utilizing the + terminology of the Berne Convention for the Protection of + Literary and Artistic Works (as amended on September 28, + 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms + Treaty of 1996 and the Universal Copyright Convention (as + revised on July 24, 1971). These rights and subject + matter take effect in the relevant jurisdiction in which + the License terms are sought to be enforced according to + the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. + If the standard suite of rights granted under applicable + copyright law includes additional rights not granted + under this License, such additional rights are deemed to + be included in the License; this License is not intended + to restrict the license of any rights under applicable + law.
    7. +
+ + +
+

Creative Commons Notice

+ +

Creative Commons is not a party to this License, and + makes no warranty whatsoever in connection with the Work. + Creative Commons will not be liable to You or any party + on any legal theory for any damages whatsoever, including + without limitation any general, special, incidental or + consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, + if Creative Commons has expressly identified itself as + the Licensor hereunder, it shall have all rights and + obligations of Licensor.

+ +

Except for the limited purpose of indicating to the + public that the Work is licensed under the CCPL, Creative + Commons does not authorize the use by either party of the + trademark "Creative Commons" or any related trademark or + logo of Creative Commons without the prior written + consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark + usage guidelines, as may be published on its website or + otherwise made available upon request from time to time. + For the avoidance of doubt, this trademark restriction + does not form part of the License.

+ +

Creative Commons may be contacted at http://creativecommons.org/.

+
+
+
+ +
+ +
+
+ + diff --git a/fdl.txt b/fdl.txt new file mode 100644 index 00000000..2f7e03ca --- /dev/null +++ b/fdl.txt @@ -0,0 +1,451 @@ + + GNU Free Documentation License + Version 1.3, 3 November 2008 + + + Copyright (C) 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc. + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +0. PREAMBLE + +The purpose of this License is to make a manual, textbook, or other +functional and useful document "free" in the sense of freedom: to +assure everyone the effective freedom to copy and redistribute it, +with or without modifying it, either commercially or noncommercially. +Secondarily, this License preserves for the author and publisher a way +to get credit for their work, while not being considered responsible +for modifications made by others. + +This License is a kind of "copyleft", which means that derivative +works of the document must themselves be free in the same sense. It +complements the GNU General Public License, which is a copyleft +license designed for free software. + +We have designed this License in order to use it for manuals for free +software, because free software needs free documentation: a free +program should come with manuals providing the same freedoms that the +software does. But this License is not limited to software manuals; +it can be used for any textual work, regardless of subject matter or +whether it is published as a printed book. We recommend this License +principally for works whose purpose is instruction or reference. + + +1. APPLICABILITY AND DEFINITIONS + +This License applies to any manual or other work, in any medium, that +contains a notice placed by the copyright holder saying it can be +distributed under the terms of this License. Such a notice grants a +world-wide, royalty-free license, unlimited in duration, to use that +work under the conditions stated herein. The "Document", below, +refers to any such manual or work. Any member of the public is a +licensee, and is addressed as "you". You accept the license if you +copy, modify or distribute the work in a way requiring permission +under copyright law. + +A "Modified Version" of the Document means any work containing the +Document or a portion of it, either copied verbatim, or with +modifications and/or translated into another language. + +A "Secondary Section" is a named appendix or a front-matter section of +the Document that deals exclusively with the relationship of the +publishers or authors of the Document to the Document's overall +subject (or to related matters) and contains nothing that could fall +directly within that overall subject. (Thus, if the Document is in +part a textbook of mathematics, a Secondary Section may not explain +any mathematics.) The relationship could be a matter of historical +connection with the subject or with related matters, or of legal, +commercial, philosophical, ethical or political position regarding +them. + +The "Invariant Sections" are certain Secondary Sections whose titles +are designated, as being those of Invariant Sections, in the notice +that says that the Document is released under this License. If a +section does not fit the above definition of Secondary then it is not +allowed to be designated as Invariant. The Document may contain zero +Invariant Sections. If the Document does not identify any Invariant +Sections then there are none. + +The "Cover Texts" are certain short passages of text that are listed, +as Front-Cover Texts or Back-Cover Texts, in the notice that says that +the Document is released under this License. A Front-Cover Text may +be at most 5 words, and a Back-Cover Text may be at most 25 words. + +A "Transparent" copy of the Document means a machine-readable copy, +represented in a format whose specification is available to the +general public, that is suitable for revising the document +straightforwardly with generic text editors or (for images composed of +pixels) generic paint programs or (for drawings) some widely available +drawing editor, and that is suitable for input to text formatters or +for automatic translation to a variety of formats suitable for input +to text formatters. A copy made in an otherwise Transparent file +format whose markup, or absence of markup, has been arranged to thwart +or discourage subsequent modification by readers is not Transparent. +An image format is not Transparent if used for any substantial amount +of text. A copy that is not "Transparent" is called "Opaque". + +Examples of suitable formats for Transparent copies include plain +ASCII without markup, Texinfo input format, LaTeX input format, SGML +or XML using a publicly available DTD, and standard-conforming simple +HTML, PostScript or PDF designed for human modification. Examples of +transparent image formats include PNG, XCF and JPG. Opaque formats +include proprietary formats that can be read and edited only by +proprietary word processors, SGML or XML for which the DTD and/or +processing tools are not generally available, and the +machine-generated HTML, PostScript or PDF produced by some word +processors for output purposes only. + +The "Title Page" means, for a printed book, the title page itself, +plus such following pages as are needed to hold, legibly, the material +this License requires to appear in the title page. For works in +formats which do not have any title page as such, "Title Page" means +the text near the most prominent appearance of the work's title, +preceding the beginning of the body of the text. + +The "publisher" means any person or entity that distributes copies of +the Document to the public. + +A section "Entitled XYZ" means a named subunit of the Document whose +title either is precisely XYZ or contains XYZ in parentheses following +text that translates XYZ in another language. (Here XYZ stands for a +specific section name mentioned below, such as "Acknowledgements", +"Dedications", "Endorsements", or "History".) To "Preserve the Title" +of such a section when you modify the Document means that it remains a +section "Entitled XYZ" according to this definition. + +The Document may include Warranty Disclaimers next to the notice which +states that this License applies to the Document. These Warranty +Disclaimers are considered to be included by reference in this +License, but only as regards disclaiming warranties: any other +implication that these Warranty Disclaimers may have is void and has +no effect on the meaning of this License. + +2. VERBATIM COPYING + +You may copy and distribute the Document in any medium, either +commercially or noncommercially, provided that this License, the +copyright notices, and the license notice saying this License applies +to the Document are reproduced in all copies, and that you add no +other conditions whatsoever to those of this License. You may not use +technical measures to obstruct or control the reading or further +copying of the copies you make or distribute. However, you may accept +compensation in exchange for copies. If you distribute a large enough +number of copies you must also follow the conditions in section 3. + +You may also lend copies, under the same conditions stated above, and +you may publicly display copies. + + +3. COPYING IN QUANTITY + +If you publish printed copies (or copies in media that commonly have +printed covers) of the Document, numbering more than 100, and the +Document's license notice requires Cover Texts, you must enclose the +copies in covers that carry, clearly and legibly, all these Cover +Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on +the back cover. Both covers must also clearly and legibly identify +you as the publisher of these copies. The front cover must present +the full title with all words of the title equally prominent and +visible. You may add other material on the covers in addition. +Copying with changes limited to the covers, as long as they preserve +the title of the Document and satisfy these conditions, can be treated +as verbatim copying in other respects. + +If the required texts for either cover are too voluminous to fit +legibly, you should put the first ones listed (as many as fit +reasonably) on the actual cover, and continue the rest onto adjacent +pages. + +If you publish or distribute Opaque copies of the Document numbering +more than 100, you must either include a machine-readable Transparent +copy along with each Opaque copy, or state in or with each Opaque copy +a computer-network location from which the general network-using +public has access to download using public-standard network protocols +a complete Transparent copy of the Document, free of added material. +If you use the latter option, you must take reasonably prudent steps, +when you begin distribution of Opaque copies in quantity, to ensure +that this Transparent copy will remain thus accessible at the stated +location until at least one year after the last time you distribute an +Opaque copy (directly or through your agents or retailers) of that +edition to the public. + +It is requested, but not required, that you contact the authors of the +Document well before redistributing any large number of copies, to +give them a chance to provide you with an updated version of the +Document. + + +4. MODIFICATIONS + +You may copy and distribute a Modified Version of the Document under +the conditions of sections 2 and 3 above, provided that you release +the Modified Version under precisely this License, with the Modified +Version filling the role of the Document, thus licensing distribution +and modification of the Modified Version to whoever possesses a copy +of it. In addition, you must do these things in the Modified Version: + +A. Use in the Title Page (and on the covers, if any) a title distinct + from that of the Document, and from those of previous versions + (which should, if there were any, be listed in the History section + of the Document). You may use the same title as a previous version + if the original publisher of that version gives permission. +B. List on the Title Page, as authors, one or more persons or entities + responsible for authorship of the modifications in the Modified + Version, together with at least five of the principal authors of the + Document (all of its principal authors, if it has fewer than five), + unless they release you from this requirement. +C. State on the Title page the name of the publisher of the + Modified Version, as the publisher. +D. Preserve all the copyright notices of the Document. +E. Add an appropriate copyright notice for your modifications + adjacent to the other copyright notices. +F. Include, immediately after the copyright notices, a license notice + giving the public permission to use the Modified Version under the + terms of this License, in the form shown in the Addendum below. +G. Preserve in that license notice the full lists of Invariant Sections + and required Cover Texts given in the Document's license notice. +H. Include an unaltered copy of this License. +I. Preserve the section Entitled "History", Preserve its Title, and add + to it an item stating at least the title, year, new authors, and + publisher of the Modified Version as given on the Title Page. If + there is no section Entitled "History" in the Document, create one + stating the title, year, authors, and publisher of the Document as + given on its Title Page, then add an item describing the Modified + Version as stated in the previous sentence. +J. Preserve the network location, if any, given in the Document for + public access to a Transparent copy of the Document, and likewise + the network locations given in the Document for previous versions + it was based on. These may be placed in the "History" section. + You may omit a network location for a work that was published at + least four years before the Document itself, or if the original + publisher of the version it refers to gives permission. +K. For any section Entitled "Acknowledgements" or "Dedications", + Preserve the Title of the section, and preserve in the section all + the substance and tone of each of the contributor acknowledgements + and/or dedications given therein. +L. Preserve all the Invariant Sections of the Document, + unaltered in their text and in their titles. Section numbers + or the equivalent are not considered part of the section titles. +M. Delete any section Entitled "Endorsements". Such a section + may not be included in the Modified Version. +N. Do not retitle any existing section to be Entitled "Endorsements" + or to conflict in title with any Invariant Section. +O. Preserve any Warranty Disclaimers. + +If the Modified Version includes new front-matter sections or +appendices that qualify as Secondary Sections and contain no material +copied from the Document, you may at your option designate some or all +of these sections as invariant. To do this, add their titles to the +list of Invariant Sections in the Modified Version's license notice. +These titles must be distinct from any other section titles. + +You may add a section Entitled "Endorsements", provided it contains +nothing but endorsements of your Modified Version by various +parties--for example, statements of peer review or that the text has +been approved by an organization as the authoritative definition of a +standard. + +You may add a passage of up to five words as a Front-Cover Text, and a +passage of up to 25 words as a Back-Cover Text, to the end of the list +of Cover Texts in the Modified Version. Only one passage of +Front-Cover Text and one of Back-Cover Text may be added by (or +through arrangements made by) any one entity. If the Document already +includes a cover text for the same cover, previously added by you or +by arrangement made by the same entity you are acting on behalf of, +you may not add another; but you may replace the old one, on explicit +permission from the previous publisher that added the old one. + +The author(s) and publisher(s) of the Document do not by this License +give permission to use their names for publicity for or to assert or +imply endorsement of any Modified Version. + + +5. COMBINING DOCUMENTS + +You may combine the Document with other documents released under this +License, under the terms defined in section 4 above for modified +versions, provided that you include in the combination all of the +Invariant Sections of all of the original documents, unmodified, and +list them all as Invariant Sections of your combined work in its +license notice, and that you preserve all their Warranty Disclaimers. + +The combined work need only contain one copy of this License, and +multiple identical Invariant Sections may be replaced with a single +copy. If there are multiple Invariant Sections with the same name but +different contents, make the title of each such section unique by +adding at the end of it, in parentheses, the name of the original +author or publisher of that section if known, or else a unique number. +Make the same adjustment to the section titles in the list of +Invariant Sections in the license notice of the combined work. + +In the combination, you must combine any sections Entitled "History" +in the various original documents, forming one section Entitled +"History"; likewise combine any sections Entitled "Acknowledgements", +and any sections Entitled "Dedications". You must delete all sections +Entitled "Endorsements". + + +6. COLLECTIONS OF DOCUMENTS + +You may make a collection consisting of the Document and other +documents released under this License, and replace the individual +copies of this License in the various documents with a single copy +that is included in the collection, provided that you follow the rules +of this License for verbatim copying of each of the documents in all +other respects. + +You may extract a single document from such a collection, and +distribute it individually under this License, provided you insert a +copy of this License into the extracted document, and follow this +License in all other respects regarding verbatim copying of that +document. + + +7. AGGREGATION WITH INDEPENDENT WORKS + +A compilation of the Document or its derivatives with other separate +and independent documents or works, in or on a volume of a storage or +distribution medium, is called an "aggregate" if the copyright +resulting from the compilation is not used to limit the legal rights +of the compilation's users beyond what the individual works permit. +When the Document is included in an aggregate, this License does not +apply to the other works in the aggregate which are not themselves +derivative works of the Document. + +If the Cover Text requirement of section 3 is applicable to these +copies of the Document, then if the Document is less than one half of +the entire aggregate, the Document's Cover Texts may be placed on +covers that bracket the Document within the aggregate, or the +electronic equivalent of covers if the Document is in electronic form. +Otherwise they must appear on printed covers that bracket the whole +aggregate. + + +8. TRANSLATION + +Translation is considered a kind of modification, so you may +distribute translations of the Document under the terms of section 4. +Replacing Invariant Sections with translations requires special +permission from their copyright holders, but you may include +translations of some or all Invariant Sections in addition to the +original versions of these Invariant Sections. You may include a +translation of this License, and all the license notices in the +Document, and any Warranty Disclaimers, provided that you also include +the original English version of this License and the original versions +of those notices and disclaimers. In case of a disagreement between +the translation and the original version of this License or a notice +or disclaimer, the original version will prevail. + +If a section in the Document is Entitled "Acknowledgements", +"Dedications", or "History", the requirement (section 4) to Preserve +its Title (section 1) will typically require changing the actual +title. + + +9. TERMINATION + +You may not copy, modify, sublicense, or distribute the Document +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense, or distribute it is void, and +will automatically terminate your rights under this License. + +However, if you cease all violation of this License, then your license +from a particular copyright holder is reinstated (a) provisionally, +unless and until the copyright holder explicitly and finally +terminates your license, and (b) permanently, if the copyright holder +fails to notify you of the violation by some reasonable means prior to +60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, receipt of a copy of some or all of the same material does +not give you any rights to use it. + + +10. FUTURE REVISIONS OF THIS LICENSE + +The Free Software Foundation may publish new, revised versions of the +GNU Free Documentation License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in +detail to address new problems or concerns. See +http://www.gnu.org/copyleft/. + +Each version of the License is given a distinguishing version number. +If the Document specifies that a particular numbered version of this +License "or any later version" applies to it, you have the option of +following the terms and conditions either of that specified version or +of any later version that has been published (not as a draft) by the +Free Software Foundation. If the Document does not specify a version +number of this License, you may choose any version ever published (not +as a draft) by the Free Software Foundation. If the Document +specifies that a proxy can decide which future versions of this +License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the +Document. + +11. RELICENSING + +"Massive Multiauthor Collaboration Site" (or "MMC Site") means any +World Wide Web server that publishes copyrightable works and also +provides prominent facilities for anybody to edit those works. A +public wiki that anybody can edit is an example of such a server. A +"Massive Multiauthor Collaboration" (or "MMC") contained in the site +means any set of copyrightable works thus published on the MMC site. + +"CC-BY-SA" means the Creative Commons Attribution-Share Alike 3.0 +license published by Creative Commons Corporation, a not-for-profit +corporation with a principal place of business in San Francisco, +California, as well as future copyleft versions of that license +published by that same organization. + +"Incorporate" means to publish or republish a Document, in whole or in +part, as part of another Document. + +An MMC is "eligible for relicensing" if it is licensed under this +License, and if all works that were first published under this License +somewhere other than this MMC, and subsequently incorporated in whole or +in part into the MMC, (1) had no cover texts or invariant sections, and +(2) were thus incorporated prior to November 1, 2008. + +The operator of an MMC Site may republish an MMC contained in the site +under CC-BY-SA on the same site at any time before August 1, 2009, +provided the MMC is eligible for relicensing. + + +ADDENDUM: How to use this License for your documents + +To use this License in a document you have written, include a copy of +the License in the document and put the following copyright and +license notices just after the title page: + + Copyright (c) YEAR YOUR NAME. + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.3 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. + A copy of the license is included in the section entitled "GNU + Free Documentation License". + +If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts, +replace the "with...Texts." line with this: + + with the Invariant Sections being LIST THEIR TITLES, with the + Front-Cover Texts being LIST, and with the Back-Cover Texts being LIST. + +If you have Invariant Sections without Cover Texts, or some other +combination of the three, merge those two alternatives to suit the +situation. + +If your document contains nontrivial examples of program code, we +recommend releasing these examples in parallel under your choice of +free software license, such as the GNU General Public License, +to permit their use in free software. diff --git a/src/parallel b/src/parallel index 1ab86a96..aeb9efc3 100755 --- a/src/parallel +++ b/src/parallel @@ -817,6 +817,92 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . +=head2 Documentation license I + +Permission is granted to copy, distribute and/or modify this documentation +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, with no Front-Cover Texts, and with no Back-Cover +Texts. A copy of the license is included in the file fdl.txt. + +=head2 Documentation license II + +You are free: + +=over 9 + +=item B + +to copy, distribute and transmit the work + +=item B + +to adapt the work + +=back + +Under the following conditions: + +=over 9 + +=item B + +You must attribute the work in the manner specified by the author or +licensor (but not in any way that suggests that they endorse you or +your use of the work). + +=item B + +If you alter, transform, or build upon this work, you may distribute +the resulting work only under the same, similar or a compatible +license. + +=back + +With the understanding that: + +=over 9 + +=item B + +Any of the above conditions can be waived if you get permission from +the copyright holder. + +=item B + +Where the work or any of its elements is in the public domain under +applicable law, that status is in no way affected by the license. + +=item B + +In no way are any of the following rights affected by the license: + +=over 9 + +=item * + +Your fair dealing or fair use rights, or other applicable +copyright exceptions and limitations; + +=item * + +The author's moral rights; + +=item * + +Rights other persons may have either in the work itself or in +how the work is used, such as publicity or privacy rights. + +=back + +=item B + +For any reuse or distribution, you must make clear to others the +license terms of this work. + +=back + +A copy of the full license is included in the file as cc-by-sa.txt. =head1 DEPENDENCIES diff --git a/src/parallel~ b/src/parallel~ deleted file mode 100755 index cfc979b8..00000000 --- a/src/parallel~ +++ /dev/null @@ -1,1806 +0,0 @@ -#!/usr/bin/perl -w - -=head1 NAME - -parallel - build and execute shell command lines from standard input in parallel - -=head1 SYNOPSIS - -B [-0cdEfghiIkmnpqrtuUvX] [-I str] [-j num] [--silent] [command [arguments]] [< list_of_arguments] - -=head1 DESCRIPTION - -For each line of input B will execute B with the -line as arguments. If no B is given the line of input is -executed. B can often be used as a substitute for B -or B. - -Several lines will be run in parallel. - -=over 9 - -=item I - -Command to execute. If B or the following arguments contain -{} every instance will be substituted with the input line. Setting a -command also invokes B<-f>. - -If B is given, B will behave similar to B. If -B is not given B will behave similar to B. - - -=item B<{}> - -Input line. This is the default replacement string and will normally -be used for putting the argument in the command line. It can be -changed with B<-I>. - - -=item B<{.}> - -Input line without extension. This is a specialized replacement string -with the extension removed. It will remove from the last B<.> till the -end of line of each input line and replace {.} with the -remaining. E.g. I becomes I. If the input line does -not contain B<.> it will remain unchanged. - -{.} can be used the same places as {}. - - -=item B<--null> - -=item B<-0> - -Use NUL as delimiter. Normally input lines will end in \n -(newline). If they end in \0 (NUL), then use this option. It is useful -for processing filenames that may contain \n (newline). - - -=item B<--command> - -=item B<-c> - -Line is a command. The input line contains more than one argument or -the input line needs to be evaluated by the shell. This is the default -if B is not set. Can be reversed with B<-f>. - - -=item B<--delimiter> I - -=item B<-d> I - -Input items are terminated by the specified character. Quotes and -backslash are not special; every character in the input is taken -literally. Disables the end-of-file string, which is treated like any -other argument. This can be used when the input consists of simply -newline-separated items, although it is almost always better to design -your program to use --null where this is possible. The specified -delimiter may be a single character, a C-style character escape such -as \n, or an octal or hexadecimal escape code. Octal and -hexadecimal escape codes are understood as for the printf command. -Multibyte characters are not supported. - - -=item B<--file> - -=item B<-f> - -Line is a filename. The input line contains a filename that will be -quoted so it is not evaluated by the shell. This is the default if -B is set. Can be reversed with B<-c>. - - -=item B<--group> - -=item B<-g> - -Group output. Output from each jobs is grouped together and is only -printed when the command is finished. STDERR first followed by STDOUT. -B<-g> is the default. Can be reversed with B<-u>. - - -=item B<-I> I - -Use the replacement string I instead of {}. - - -=item B<-U> I - -=item B<--extensionreplace> I - -Use the replacement string I instead of {.} for input line without extension. - - -=item B<--jobs> I - -=item B<-j> I - -=item B<--max-procs> I - -=item B<-P> I - -Run up to N jobs in parallel. 0 means as many as possible. Default is 10. - - -=item B<--jobs> I<+N> - -=item B<-j> I<+N> - -=item B<--max-procs> I<+N> - -=item B<-P> I<+N> - -Add N to the number of CPU cores. Run this many jobs in parallel. For -compute intensive jobs I<-j +0> is useful as it will run -number-of-cpu-cores jobs in parallel. See also ---use-cpus-instead-of-cores. - - -=item B<--jobs> I<-N> - -=item B<-j> I<-N> - -=item B<--max-procs> I<-N> - -=item B<-P> I<-N> - -Subtract N from the number of CPU cores. Run this many jobs in parallel. -If the evaluated number is less than 1 then 1 will be used. See also ---use-cpus-instead-of-cores. - - -=item B<--jobs> I% - -=item B<-j> I% - -=item B<--max-procs> I% - -=item B<-P> I% - -Multiply N% with the number of CPU cores. Run this many jobs in parallel. -If the evaluated number is less than 1 then 1 will be used. See also ---use-cpus-instead-of-cores. - - -=item B<--keeporder> - -=item B<-k> - -Keep sequence of output same as the order of input. If jobs 1 2 3 4 -end in the sequence 3 1 4 2 the output will still be 1 2 3 4. - - -=item B<--number-of-cpus> - -Print the number of CPUs and exit (used by B itself to -determine the number of CPUs on remote machines). - - -=item B<--number-of-cores> - -Print the number of cores and exit (used by B itself to determine the -number of cores on remote machines). - - -=item B<--quote> - -=item B<-q> - -Quote B. This will quote the command line so special -characters are not interpreted by the shell. See the section -QUOTING. Most people will never need this. Quoting is disabled by -default. - - -=item B<-S> I<[ncpu/]sshlogin[,[ncpu/]sshlogin]> (not implemented) - -=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin]> (not implemented) - -Distribute jobs to remote servers. The jobs will be run on a list of -remote servers. B will determine the number of CPU cores on -the remote servers and run the number of jobs as specified by -j. If -the number I is given B will use this number for -number of CPUs on the host. Normally I will not be needed. - -An I is the string you would normally pass to SSH to login, -e.g. I, I, or I<"-l foo -p -2222 server.example.com">. The sshlogin must not require a password. - -The sshlogin ':' is special, it means 'no ssh' and will therefore run -on the local machine. - -To specify more sshlogins separate the sshlogins by comma or repeat -the options multiple times. - -For examples: see B<--sshloginfile>. - -The remote host must have B installed. - - -=item B<--sshloginfile> I (not implemented) - -File with sshlogins. The file consists of sshlogins on separate -lines. Empty lines and lines starting with '#' are ignored. Example: - - server.example.com - username@server2.example.com - 8/my-8-core-server.example.com - 2/myusername@my-dualcore.example.net - # This server has SSH running on port 2222 - -p 2222 server.example.net - 4/-p 2222 quadserver.example.net - # Assume 16 cores on the local machine - 16/: - - -=item B<--silent> - -Silent. The job to be run will not be printed. This is the default. -Can be reversed with B<-v>. - - -=item B<--transfer> (not implemented) - -Transfer files to remote servers. B<--transfer> is used with -B<--sshlogin> when the arguments are files and should be transfered to -the remote servers. The files will be transfered using B and -will be put relative to the default login dir. E.g. - - echo foo/bar.txt | parallel \ - --sshlogin server.example.com --transfer wc - -This will transfer the file I to the server -I to the file $HOME/foo/bar.txt before running -B on I. - - echo /tmp/foo/bar.txt | parallel \ - --sshlogin server.example.com --transfer wc - -This will transfer the file I to the server -I to the file /tmp/foo/bar.txt before running -B on I. - -B<--transfer> is often used with B<--return> and B<--cleanup>. - -B<--transfer> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. - - -=item B<--trc> I (not implemented) - -Transfer, Return, Cleanup. Short hand for: - - --transfer --return I --cleanup - - -=item B<--return> I (not implemented) - -Transfer files from remote servers. B<--return> is used with -B<--sshlogin> when the arguments are files on the remote servers. When -processing is done the file I will be transfered -from the remote server using B and will be put relative to -the default login dir. E.g. - - echo foo/bar.txt | parallel \ - --sshlogin server.example.com --return {}.out touch {}.out - -This will transfer the file I<$HOME/foo/bar.txt.out> from the server -I to the file I after running -B on I. - - echo /tmp/foo/bar.txt | parallel \ - --sshlogin server.example.com --return {}.out touch {}.out - -This will transfer the file I from the server -I to the file I after running -B on I. - -Multiple files can be transfered by repeating the options multiple -times: - - echo /tmp/foo/bar.txt | \ - parallel --sshlogin server.example.com \ - --return {}.out --return {}.out2 touch {}.out {}.out2 - -B<--return> is often used with B<--transfer> and B<--cleanup>. - -B<--return> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. - - -=item B<--cleanup> (not implemented) - -Remove transfered files. B<--cleanup> will remove the transfered files -on the remote server after processing is done. - - find log -name '*gz' | parallel \ - --sshlogin server.example.com --transfer --return {.}.bz2 \ - --cleanup "zcat {} | bzip -9 >{.}.bz2" - -With B<--transfer> the file transfered to the remote server will be -removed on the remote server. Directories created will not be removed -- even if they are empty. - -With B<--return> the file transfered from the remote server will be -removed on the remote server. Directories created will not be removed -- even if they are empty. - -B<--cleanup> is ignored when not used with B<--transfer> or B<--return>. - - -=item B<--ungroup> - -=item B<-u> - -Ungroup output. Output is printed as soon as possible. This may cause -output from different commands to be mixed. Can be reversed with B<-g>. - - -=item B<--use-cpus-instead-of-cores> (not implemented) - -Count the number of CPUs instead of cores. When computing how many -jobs to run in parallel relative to the number of cores you can ask -parallel to instead look at the number of CPUs. This will make sense -for computers that have hyperthreading as two jobs running on one CPU -with hyperthreading will run slower than two jobs running on two CPUs. -Normal users will not need this option. - - -=item B<-v> - -Verbose. Print the job to be run on STDOUT. Can be reversed with -B<--silent>. - - -=item B<--xargs> - -=item B<-m> - -Multiple. Insert as many arguments as the command line length permits. If -{} is not used the arguments will be appended to the line. If {} is -used multiple times each {} will be replaced with all the arguments. - - -=item B<-X> - -xargs with context replace. This works like B<-m> except if {} is part -of a word (like I) then the whole word will be repeated. - -=back - -=head1 EXAMPLE 1: Working as cat | sh. Ressource inexpensive jobs and evaluation - -B can work similar to B. - -A ressource inexpensive job is a job that takes very little CPU, disk -I/O and network I/O. Ping is an example of a ressource inexpensive -job. wget is too - if the webpages are small. - -The content of the file jobs_to_run: - - ping -c 1 10.0.0.1 - wget http://status-server/status.cgi?ip=10.0.0.1 - ping -c 1 10.0.0.2 - wget http://status-server/status.cgi?ip=10.0.0.2 - ... - ping -c 1 10.0.0.255 - wget http://status-server/status.cgi?ip=10.0.0.255 - -To run 100 processes simultaneously do: - -B - -As there is not a B the option B<-c> is default because the -jobs needs to be evaluated by the shell. - -=head1 EXAMPLE 2: Working as xargs -n1. Argument appending - -B can work similar to B. - -To output all html files run: - -B - -As there is a B the option B<-f> is default because the -filenames needs to be protected from the shell in case a filename -contains special characters. - -=head1 EXAMPLE 3: Compute intensive jobs and substitution - -If ImageMagick is installed this will generate a thumbnail of a jpg -file: - -B - -If the system has more than 1 CPU core it can be run with -number-of-cpu-cores jobs in parallel (-j +0). This will do that for -all jpg files in a directory: - -B - -To do it recursively use B: - -B - -Notice how the argument has to start with {} as {} will include path -(e.g. running B would clearly be wrong). It will result in files -like ./foo/bar.jpg_thumb.jpg. If that is not wanted this can fix it: - - find . -name '*.jpg' | \ - perl -pe 'chomp; $a=$_; s:/([^/]+)$:/thumb_$1:; $_="convert -geometry 120 $a $_\n"' | \ - parallel -c -j +0 - -Unfortunately this will not work if the filenames contain special -characters (such as space or quotes). If you have B installed this -is a better solution: - - find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {}_thumb.jpg - find . -name '*_thumb.jpg' | ren 's:/([^/]+)_thumb.jpg$:/thumb_$1:' - -This will make files like ./foo/bar_thumb.jpg: - -B - -=head1 EXAMPLE 4: Substitution and redirection - -This will compare all files in the dir to the file foo and save the -diffs in corresponding .diff files: - -B>B<"{}.diff> - -Quoting of > is necessary to postpone the redirection. Another -solution is to quote the whole command: - -B>B<{}.diff"> - - -=head1 EXAMPLE 5: Composed commands - -A job can consist of several commands. This will print the number of -files in each directory: - -B - -To put the output in a file called .dir: - -B> B<{}.dir'> - - -=head1 EXAMPLE 6: Context replace - -To remove the files I .. I you could do: - -B - -You could also do: - -B - -The first will run B 10000 times, while the last will only run -B as many times needed to keep the command line length short -enough (typically 1-2 times). - -You could also run: - -B - -This will also only run B as many times needed to keep the command -line length short enough. - -=head1 EXAMPLE 7: Group output lines - -When runnning jobs that output data, you often do not want the output -of multiple jobs to run together. B defaults to grouping the -output of each job, so the output is printed when the job finishes. If -you want the output to be printed while the job is running you can use -B<-u>. - -Compare the output of: - -B<(echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel traceroute> - -to the output of: - -B<(echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel -u traceroute> - - -=head1 EXAMPLE 8: Keep order of output same as order of input - -Normally the output of a job will be printed as soon as it -completes. Sometimes you want the order of the output to remain the -same as the order of the input. B<-k> will make sure the order of -output will be in the same order as input even if later jobs end -before earlier jobs. - -B<(echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel traceroute> - -will give traceroute of foss.org.my, debian.org and -freenetproject.org, but it will be sorted according to which job -completed first. - -To keep the order the same as input run: - -B<(echo foss.org.my; echo debian.org; echo freenetproject.org) | parallel -k traceroute> - -This will make sure the traceroute to foss.org.my will be printed -first. - -=head1 EXAMPLE 9: Using remote computers (not implemented) - -To run commands on a remote computer SSH needs to be set up and you -must be able to login without entering a password (B may be -handy). - -To run B on B: - - seq 1 10 | parallel --sshlogin server.example.com echo - -To run commands on more than one remote computer run: - - seq 1 10 | parallel --sshlogin server.example.com,server2.example.net echo - -Or: - - seq 1 10 | parallel --sshlogin server.example.com \ - --sshlogin server2.example.net echo - -If the login username is I on I use: - - seq 1 10 | parallel --sshlogin server.example.com \ - --sshlogin foo@server2.example.net echo - -To distribute the commands to a list of machines, make a file -I with all the machines: - - server.example.com - foo@server2.example.com - server3.example.com - -Then run: - - seq 1 10 | parallel --sshloginfile mymachines echo - -To include the local machine add the special sshlogin ':' to the list: - - server.example.com - foo@server2.example.com - server3.example.com - : - -If the number of CPU cores on the remote servers is not identified -correctly the number of CPU cores can be added in front. Here the -server has 8 CPU cores. - - seq 1 10 | parallel --sshlogin 8/server.example.com echo - - -=head1 EXAMPLE 10: Transferring of files (not implemented) - -To recompress gzipped files with bzip2 using a remote server run: - - find logs/ -name '*.gz' | \ - parallel --sshlogin server.example.com \ - --transfer "zcat {} | bzip2 -9 >{.}.bz2" - -This will list the .gz-files in the I directory and all -directories below. Then it will transfer the files to -I to the corresponding directory in -I<$HOME/logs>. On I the file will be recompressed -using B and B resulting in the corresponding file with -I<.gz> replaced with I<.bz2>. - -If you want the file to be transfered back to the local machine add -I<--return {.}.bz2>: - - find logs/ -name '*.gz' | \ - parallel --sshlogin server.example.com \ - --transfer --return {.}.bz2 "zcat {} | bzip2 -9 >{.}.bz2" - -After the recompressing is done the I<.bz2>-file is transfered back to -the local machine and put next to the original I<.gz>-file. - -If you want to delete the transfered files on the remote machine add -I<--cleanup>. This will remove both the file transfered to the remote -machine and the files transfered from the remote machine: - - find logs/ -name '*.gz' | \ - parallel --sshlogin server.example.com \ - --transfer --return {.}.bz2 --cleanup "zcat {} | bzip2 -9 >{.}.bz2" - -If you want run one several servers add the servers to I<--sshlogin> -either using ',' or separate I<--sshlogin>: - - find logs/ -name '*.gz' | \ - parallel --sshlogin server.example.com,server2.example.com \ - --sshlogin server3.example.com \ - --transfer --return {.}.bz2 --cleanup "zcat {} | bzip2 -9 >{.}.bz2" - -You can add the local machine using I<--sshlogin :>. This will disable the -removing and transferring for the local machine only: - - find logs/ -name '*.gz' | \ - parallel --sshlogin server.example.com,server2.example.com \ - --sshlogin server3.example.com \ - --sshlogin : \ - --transfer --return {.}.bz2 --cleanup "zcat {} | bzip2 -9 >{.}.bz2" - -Often I<--transfer>, I<--return> and I<--cleanup> are used together. They can be -shortened to I<--trc>: - - find logs/ -name '*.gz' | \ - parallel --sshlogin server.example.com,server2.example.com \ - --sshlogin server3.example.com \ - --sshlogin : \ - --trc {.}.bz2 "zcat {} | bzip2 -9 >{.}.bz2" - -With the file I containing the compute machines it becomes: - - find logs/ -name '*.gz' | parallel --sshloginfile mymachines \ - --trc {.}.bz2 "zcat {} | bzip2 -9 >{.}.bz2" - -=head1 QUOTING - -For more advanced use quoting may be an issue. The following will -print the filename for each line that has exactly 2 columns: - -B - -This can be done by B using: - -B - -Notice how \'s, "'s, and $'s needs to be quoted. B can do -the quoting by using option B<-q>: - -B - -However, this means you cannot make the shell interpret special -characters. For example this B: - -B>B<{}.diff"> - -B - -because > and | need to be interpreted by the shell. - -If you get errors like: - -B - -then you might try using B<-q>. - -If you are using B process substitution like B<<(cat foo)> then -you may try B<-q> and prepending B with B: - -B - -Or for substituting output: - -B>B<(gzip >>B<{}.tar.gz) | bzip2 >>B<{}.tar.bz2'> - -B: To avoid dealing with the quoting problems it may be -easier just to write a small script and have B call that -script. - - -=head1 LIST RUNNING JOBS - -If you want a list of the jobs currently running you can run: - -B - -B will then print the currently running jobs on STDERR. - - -=head1 COMPLETE RUNNING JOBS BUT DO NOT START NEW JOBS - -If you regret starting a lot of jobs you can simply break B, -but if you want to make sure you do not have halfcompleted jobs you -should send the signal B to B: - -B - -This will tell B to not start any new jobs, but wait until -the currently running jobs are finished. - - -=head1 DIFFERENCES BETWEEN xargs/find -exec AND parallel - -B and B offer some of the same possibilites as -B. - -B only works on files. So processing other input (such as -hosts or URLs) will require creating these inputs as files. B has no support for running commands in parallel. - -B deals badly with special characters (such as space, ' and -"). To see the problem try this: - - touch important_file - touch 'not important_file' - ls not* | xargs rm - mkdir -p '12" records' - ls | xargs rmdir - -You can specify B<-0> or B<-d "\n">, but many input generators are not -optimized for using B as separator but are optimized for -B as separator. E.g B, B, B, B, B, -B, B, B (-0 and \0 instead of \n), B -(requires using -0), B (requires using -print0), B -(requires user to use -z or -Z). - -So B's newline separation can be emulated with: - -B> - -B can run a given number of jobs in parallel, but has no -support for running number-of-cpu-cores jobs in parallel. - -B has no support for grouping the output, therefore output may -run together, e.g. the first half of a line is from one process and -the last half of the line is from another process. - -B has no support for keeping the order of the output, therefore -if running jobs in parallel using B the output of the second -job cannot be postponed till the first job is done. - -B has no support for context replace, so you will have to create the -arguments. - -If you use a replace string in B (B<-I>) you can not force -B to use more than one argument. - -Quoting in B works like B<-q> in B. This means -composed commands and redirection requires using B. - -B> B<{}.wc"> - -becomes - -B>B< {}.wc"> - -and - -B - -becomes - -B - - -=head1 DIFFERENCES BETWEEN mdm/middleman AND parallel - -middleman(mdm) is also a tool for running jobs in parallel. - -Here are the shellscripts of http://mdm.berlios.de/usage.html ported -to parallel use: - -B>B< result> - -B - - -=head1 BUGS - -Filenames beginning with '-' can cause some commands to give -unexpected results, as it will often be interpreted as an option. - -=head1 REPORTING BUGS - -Report bugs to . - -=head1 IDEAS - -One char options not used: F G J K M P Q Y - -Test if -0 works on filenames ending in '\n' - -xargs dropin-replacement. -Implement the missing --features - -monitor to see which jobs are currently running -http://code.google.com/p/ppss/ - -Accept signal INT instead of TERM to complete current running jobs but -do not start new jobs. Print out the number of jobs waiting to -complete on STDERR. Accept sig INT again to kill now. This seems to be -hard, as all foreground processes get the INT from the shell. - -If there are nomore jobs (STDIN is closed) then make sure to -distribute the arguments evenly if running -X. - - -Distribute jobs to computers with different speeds/number-of-cpu-cores using ssh -ask the computers how many cpus they have and spawn appropriately -according to -j setting. Reuse ssh connection (-M and -S) - -SEED=$RANDOM -ssh -MS /tmp/ssh-%r@%h:%p-$SEED elvis -rsync --rsh="ssh -S /tmp/ssh-%r@%h:%p-$SEED" gitup elvis:/tmp/ -ssh -S /tmp/ssh-%r@%h:%p-$SEED elvis hostname - -FILE=gpl-3.0.txt -BASE=gpl-3.0 -$ rsync -z $FILE e:$FILE -$ ssh e "cat $FILE | bzip2 > $BASE.bz2" -$ rsync -z e:$BASE.bz2 $BASE.bz2 -$ ssh e "rm $FILE $BASE" - -http://www.semicomplete.com/blog/geekery/distributed-xargs.html?source=rss20 -http://code.google.com/p/ppss/wiki/Manual2 - -http://www.gnu.org/software/pexec/ - -Where will '>' be run? Local or remote? Remote. - - -Parallelize so this can be done: -mdm.screen find dir -execdir mdm-run cmd {} \; -Maybe: -find dir -execdir parallel --communication-file /tmp/comfile cmd {} \; - -=head2 Comfile - -This will put a lock on /tmp/comfile. The number of locks is the number of running commands. -If the number is smaller than -j then it will start a process in the background ( cmd & ), -otherwise wait. - -parallel --wait /tmp/comfile will wait until no more locks on the file - -=head1 AUTHOR - -Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk - -Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk - - -=head1 LICENSE - -Copyright (C) 2007-2010 Free Software Foundation, Inc. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 3 of the License, or -at your option any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - - -=head1 DEPENDENCIES - -B uses Perl, and the Perl modules Getopt::Long, IPC::Open3, -Symbol, IO::File, POSIX, and File::Temp. - - -=head1 SEE ALSO - -B(1), B(1) - -=cut - - -use IPC::Open3; -use Symbol qw(gensym); -use IO::File; -use POSIX ":sys_wait_h"; -use File::Temp qw/ tempfile tempdir /; -use Getopt::Long; -use strict; - -my ($processes,$command); - -Getopt::Long::Configure ("bundling","require_order"); -GetOptions("debug|D" => \$::opt_D, - "xargs|m" => \$::opt_m, - "X" => \$::opt_X, - "v" => \$::opt_v, - "silent" => \$::opt_silent, - "keeporder|k" => \$::opt_k, - "group|g" => \$::opt_g, - "ungroup|u" => \$::opt_u, - "command|c" => \$::opt_c, - "file|f" => \$::opt_f, - "null|0" => \$::opt_0, - "quote|q" => \$::opt_q, - "I=s" => \$::opt_I, - "extensionreplace|U=s" => \$::opt_U, - "jobs|j=s" => \$::opt_P, - "number-of-cpus" => \$::opt_number_of_cpus, - "number-of-cores" => \$::opt_number_of_cores, - # xargs-compatibility - implemented, man, unittest - "max-procs|P=s" => \$::opt_P, - "delimiter|d=s" => \$::opt_d, - # xargs-compatibility - implemented, unittest - man missing - "max-chars|s=i" => \$::opt_s, - "arg-file|a=s" => \$::opt_a, - "no-run-if-empty|r" => \$::opt_r, - "replace|i:s" => \$::opt_i, - "E=s" => \$::opt_E, - "eof|e:s" => \$::opt_E, - "max-args|n=i" => \$::opt_n, - "verbose|t" => \$::opt_t, - "help|h" => \$::opt_h, - "version" => \$::opt_version, - ## xargs-compatibility - implemented - unittest missing - man missing - "interactive|p" => \$::opt_p, - ## How to unittest? tty skal emuleres - - # xargs-compatability - unimplemented - "L=i" => \$::opt_L, - "max-lines|l:i" => \$::opt_l, - ## (echo a b;echo c) | xargs -l1 echo - ## (echo a b' ';echo c) | xargs -l1 echo - "show-limits" => \$::opt_show_limits, - "exit|x" => \$::opt_x, - ) || die_usage(); - -# Defaults: -$Global::version = 20100416; -$Global::progname = "parallel"; -$Global::debug = 0; -$Global::processes_to_run = 10; -$command = undef; -$Global::verbose = 0; -$Global::grouped = 1; -$Global::keeporder = 0; -$Global::quoting = 0; -$Global::replacestring = '{}'; -$Global::replace_no_ext = '{.}'; -$Global::input_is_filename = (@ARGV); -$/="\n"; -$Global::ignore_empty = 0; -$Global::argfile = *STDIN; -$Global::interactive = 0; -$Global::stderr_verbose = 0; - -$Global::debug = (defined $::opt_D); -if(defined $::opt_m) { $Global::xargs = 1; } -if(defined $::opt_X) { $Global::Xargs = 1; } -if(defined $::opt_v) { $Global::verbose = 1; } -if(defined $::opt_silent) { $Global::verbose = 0; } -if(defined $::opt_k) { $Global::keeporder = 1; } -if(defined $::opt_g) { $Global::grouped = 1; } -if(defined $::opt_u) { $Global::grouped = 0; } -if(defined $::opt_c) { $Global::input_is_filename = 0; } -if(defined $::opt_f) { $Global::input_is_filename = 1; } -if(defined $::opt_0) { $/ = "\0"; } -if(defined $::opt_d) { my $e="sprintf \"$::opt_d\""; $/ = eval $e; } -if(defined $::opt_p) { $Global::interactive = $::opt_p; } -if(defined $::opt_q) { $Global::quoting = 1; } -if(defined $::opt_r) { $Global::ignore_empty = 1; } -if(defined $::opt_t) { $Global::stderr_verbose = 1; } -if(defined $::opt_I) { $Global::replacestring = $::opt_I; } -if(defined $::opt_U) { $Global::replace_no_ext = $::opt_U; } -if(defined $::opt_i and $::opt_i) { $Global::replacestring = $::opt_i; } -if(defined $::opt_E and $::opt_E) { $Global::end_of_file_string = $::opt_E; } -if(defined $::opt_n and $::opt_n) { $Global::max_number_of_args = $::opt_n; } -if(defined $::opt_h) { die_usage(); } -if(defined $::opt_number_of_cpus) { print no_of_cpus(),"\n"; exit(0); } -if(defined $::opt_number_of_cores) { print no_of_cores(),"\n"; exit(0); } -if(defined $::opt_version) { version(); exit(0); } - -if(defined $::opt_a) { - if(not open(ARGFILE,"<".$::opt_a)) { - print STDERR "$Global::progname: Cannot open input file `$::opt_a': No such file or directory\n"; - exit(-1); - } - $Global::argfile = *ARGFILE; -} - -if(@ARGV) { - if($Global::quoting) { - $Global::command = join(" ", shell_quote(@ARGV)); - } else { - $Global::command = join(" ", @ARGV); - } -} -# Needs to be done after setting $Global::command and $Global::command_line_max_len -# as '-m' influences the number of commands that needs to be run -if(defined $::opt_P) { $Global::processes_to_run = compute_number_of_processes($::opt_P); } - -$Global::job_end_sequence=1; - -init_run_jobs(); -DoNotReap(); -start_more_jobs(); -ReapIfNeeded(); -drain_job_queue(); - -# -# Generating the command line -# - -sub generate_command_line { - my $command = shift; - my ($job_line,$last_good); - my ($next_arg,@quoted_args,@quoted_args_no_ext,$arg_length); - my ($number_of_substitution) = 1; - my ($number_of_substitution_no_ext) = 0; - my ($length_of_context) = 0; - my ($length_of_command_no_args); - if($Global::xargs or $Global::Xargs) { - # Count number of {}'s on the command line - $number_of_substitution = ($command =~ s/\Q$Global::replacestring\E/$Global::replacestring/go); - $number_of_substitution ||= 1; - } - if($Global::xargs or $Global::Xargs) { - # Count number of {.}'s on the command line - $number_of_substitution_no_ext = - ($command =~ s/\Q$Global::replace_no_ext\E/$Global::replace_no_ext/go); - $number_of_substitution_no_ext ||= 0; - } - my $spaces=0; - if($Global::xargs or $Global::Xargs) { - my $c = $command; - # count number of replacements - my $no_of_replace = 0; - $no_of_replace++ while ($c =~ m/\Q$Global::replacestring\E/g); - my $no_of_no_ext = 0; - $no_of_no_ext++ while ($c =~ m/\Q$Global::replace_no_ext\E/g); - - if($Global::xargs) { - # remove all {}s - $c =~ s/\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E//og; - $length_of_command_no_args = length($c) - $no_of_replace - $no_of_no_ext; - $length_of_context = 0; - $spaces = 1; - } - if($Global::Xargs) { - $c =~ s/\S*\Q$Global::replacestring\E\S*//go; - $c =~ s/\S*\Q$Global::replace_no_ext\E\S*//go; - - $length_of_command_no_args = length($c) - 1; - $length_of_context = length($command) - $length_of_command_no_args - - $no_of_replace * length($Global::replacestring) - - $no_of_no_ext * length($Global::replace_no_ext); - } - } - - my $number_of_args = 0; - # max number of lines (-L) = - # number_of_read_lines = 0 - while (defined($next_arg = get_next_arg())) { - my $next_arg_no_ext = $next_arg; - $next_arg_no_ext =~ s/\.[^\.]*$//; # Remove .ext from argument - # if defined max_number_of_lines - # number_of_read_lines++ - # if $next_arg =~ /\w$/ then number_of_read_lines-- - # Trailing blanks cause an - # input line to be logically continued on the next input line. - # if number_of_read_lines > max_number_of_lines - # last - push (@quoted_args, $next_arg); - push (@quoted_args_no_ext, $next_arg_no_ext); - $number_of_args++; - if(not $Global::xargs and not $Global::Xargs) { - last; - } else { - # Emulate xargs if there is a command and -x or -X is set - my $next_arg_len = $number_of_substitution * (length ($next_arg) + $spaces) + - + $number_of_substitution_no_ext * (length ($next_arg_no_ext) + $spaces) - + $length_of_context; - - $arg_length += $next_arg_len; - # debug("arglen $arg_length = $number_of_substitution * (1 + length ($next_arg)) + $length_of_context\n"); - my $job_line_length = $length_of_command_no_args + $arg_length; - # debug("linelen $job_line_length = $length_of_command_no_args + 1 + $arg_length\n"); -# print STDERR "1234567890123456789012345678901234567890\n"; - #print STDERR "LENcalc $number_of_args CON$length_of_context $length_of_command_no_args ".length ($next_arg)." LL$job_line_length NAL$next_arg_len ",$job_line_length-$next_arg_len-1,"\n"; - if($job_line_length >= max_length_of_command_line()) { - unget_arg(pop @quoted_args); - if(defined $quoted_args[0]) { - last; - } else { - die ("Command line too long ($job_line_length >= " - . max_length_of_command_line() . ") at number $number_of_args: $next_arg"); - } - } - if($Global::max_number_of_args and $number_of_args >= $Global::max_number_of_args) { - last; - } - } - } - if(@quoted_args) { - $job_line = $command; - if(defined $job_line and - ($job_line =~/\Q$Global::replacestring\E/o or $job_line =~/\Q$Global::replace_no_ext\E/o)) { - # substitute {} and {.} with args - if($Global::Xargs) { - # Context sensitive replace (foo{}bar with fooargsbar) - $job_line = context_replace($job_line, \@quoted_args, \@quoted_args_no_ext); - } else { - # Normal replace {} with args and {.} with args without extension - my $arg=join(" ",@quoted_args); - my $arg_no_ext=join(" ",@quoted_args_no_ext); - $job_line =~ s/\Q$Global::replacestring\E/$arg/go; - $job_line =~ s/\Q$Global::replace_no_ext\E/$arg_no_ext/go; - } - } else { - # append args - my $arg=join(" ",@quoted_args); - $job_line .= " $arg"; - } - debug("Return jobline: $job_line\n"); - } - return $job_line; -} - -sub shell_quote { - # Quote the string so shell will not expand any special chars - my (@strings) = (@_); - my $arg; - for $arg (@strings) { - # what is the right thing to do about '-' at start of line? - # maybe substitute with './' - # so it is not regarded as -option. - $arg =~ s/\\/\\\\/g; - - $arg =~ s/([\#\?\`\(\)\*\>\<\~\|\; \"\!\$\&\'])/\\$1/g; - $arg =~ s/([\002-\011\013-\032])/\\$1/g; - $arg =~ s/([\n])/'\n'/g; # filenames with '\n' is quoted using \' - } - return (@strings); -} - -# Replace foo{}bar or foo{.}bar -sub context_replace { - my ($job_line,$quoted,$no_ext) = (@_); - while($job_line =~/\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E/o) { - $job_line =~ /(\S*(\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E)\S*)/o - or die ("This should never happen"); - my $wordarg = $1; # This is the context that needs to be substituted - my @all_word_arg; - for my $n (0 .. $#$quoted) { - my $arg = $quoted->[$n]; - my $arg_no_ext = $no_ext->[$n]; - my $substituted = $wordarg; - $substituted=~s/\Q$Global::replacestring\E/$arg/go; - $substituted=~s/\Q$Global::replace_no_ext\E/$arg_no_ext/go; - push @all_word_arg, $substituted; - } - my $all_word_arg = join(" ",@all_word_arg); - $job_line =~ s/\Q$wordarg\E/$all_word_arg/; - } - return $job_line; -} - -# -# Number of processes, filehandles, max length of command line -# - -# Maximal command line length (for -m and -X) -sub max_length_of_command_line { - # Find the max_length of a command line - # First find an upper bound - if(not $Global::command_line_max_len) { - my $len = 10; - do { - $len *= 10; - } while (is_acceptable_command_line_length($len)); - # Then search for the actual max length between 0 and upper bound - $Global::command_line_max_len = binary_find_max_length(int(($len)/10),$len); - if($::opt_s) { - if($::opt_s <= $Global::command_line_max_len) { - $Global::command_line_max_len = $::opt_s; - } else { - print STDERR "$Global::progname: ", - "value for -s option should be < $Global::command_line_max_len\n"; - } - } - } - return $Global::command_line_max_len; -} - -sub binary_find_max_length { - # Given a lower and upper bound find the max_length of a command line - my ($lower, $upper) = (@_); - if($lower == $upper or $lower == $upper-1) { return $lower; } - my $middle = int (($upper-$lower)/2 + $lower); - debug("$lower,$upper,$middle\n"); - if (is_acceptable_command_line_length($middle)) { - return binary_find_max_length($middle,$upper); - } else { - return binary_find_max_length($lower,$middle); - } -} - -sub is_acceptable_command_line_length { - # Test if a command line of this length can run - my $len = shift; - $Global::is_acceptable_command_line_length++; - debug("$Global::is_acceptable_command_line_length $len\n"); - local *STDERR; - open (STDERR,">/dev/null"); - system "true "."x"x$len; - close STDERR; - return not $?; -} - -# Number of parallel processes to run - -sub compute_number_of_processes { - # Number of processes wanted and limited by system ressources - my $opt_P = shift; - my $wanted_processes = user_requested_processes($opt_P); - debug("Wanted procs: $wanted_processes\n"); - my $system_limit = processes_available_by_system_limit($wanted_processes); - debug("Limited to procs: $system_limit\n"); - return $system_limit; -} - -sub processes_available_by_system_limit { - # If the wanted number of processes is bigger than the system limits: - # Limit them to the system limits - # Limits are: File handles, number of input lines, processes, - # and taking > 1 second to spawn 10 extra processes - my $wanted_processes = shift; - my $system_limit=0; - my @command_lines=(); - my $next_command_line; - my $more_filehandles; - my $max_system_proc_reached=0; - my $spawning_too_slow=0; - my $time = time; - my %fh; - my @children; - DoNotReap(); - - # Reserve filehandles - # perl uses 7 filehandles for something? - # parallel uses 1 for memory_usage - for my $i (1..8) { - open($fh{"init-$i"}," 2) { - # It took more than 2 second to fork ten processes. We should stop forking. - # Let us give the system a little slack - debug("\nLimiting processes to: $system_limit-10%=". - (int ($system_limit * 0.9)+1)."\n"); - $system_limit = int ($system_limit * 0.9)+1; - $spawning_too_slow = 1; - } - } while($system_limit < $wanted_processes - and defined $next_command_line - and $more_filehandles - and not $max_system_proc_reached - and not $spawning_too_slow); - if($system_limit < $wanted_processes and not $more_filehandles) { - print STDERR ("Warning: Only enough filehandles to run ", - $system_limit, " jobs in parallel. ", - "Raising ulimit -n may help\n"); - } - if($system_limit < $wanted_processes and $max_system_proc_reached) { - print STDERR ("Warning: Only enough available processes to run ", - $system_limit, " jobs in parallel.\n"); - } - if($system_limit < $wanted_processes and $spawning_too_slow) { - print STDERR ("Warning: Starting 10 extra processes takes > 2 sec.\n", - "Limiting to ", $system_limit, " jobs in parallel.\n"); - } - # Cleanup: Close the files - for (values %fh) { close $_ } - # Cleanup: Kill the children - for my $pid (@children) { - kill 15, $pid; - waitpid($pid,0); - } - wait(); - # Cleanup: Unget the command_lines - unget_command_line(@command_lines); - return $system_limit; -} - -sub enough_file_handles { - # check that we have enough filehandles available for starting - # another job - if($Global::grouped) { - my %fh; - my $enough_filehandles = 1; - # We need a filehandle for STDOUT and STDERR - # open3 uses 2 extra filehandles temporarily - for my $i (1..4) { - $enough_filehandles &&= open($fh{$i},") { - /^processor.*[:]/ and $no_of_cpus++; - } - close IN; - } - return $no_of_cpus; -} - -sub no_of_cpus_darwin { - my $no_of_cpus = `sysctl -n hw.ncpu 2>/dev/null`; - return $no_of_cpus; -} - -sub no_of_cpus_solaris { - if(-x "/usr/sbin/psrinfo") { - my @psrinfo = `/usr/sbin/psrinfo`; - if($#psrinfo >= 0) { - return $#psrinfo +1; - } - } - if(-x "/usr/sbin/prtconf") { - my @prtconf = `/usr/sbin/prtconf | grep cpu..instance`; - if($#prtconf >= 0) { - return $#prtconf +1; - } - } - return undef; -} - - -# -# Running and printing the jobs -# - -sub init_run_jobs { - # Remember the original STDOUT and STDERR - open $Global::original_stdout, ">&STDOUT" or die "Can't dup STDOUT: $!"; - open $Global::original_stderr, ">&STDERR" or die "Can't dup STDERR: $!"; - $Global::running_jobs=0; - $SIG{USR1} = \&ListRunningJobs; - $Global::original_sigterm = $SIG{TERM}; - $SIG{TERM} = \&StartNoNewJobs; -} - -sub next_command_line { - my $cmd_line; - if(@Global::unget_next_command_line) { - $cmd_line = shift @Global::unget_next_command_line; - } else { - do { - $cmd_line = generate_command_line($Global::command); - } while (defined $cmd_line and $cmd_line =~ /^\s*$/); # Skip empty lines - } - return $cmd_line; -} - -sub unget_command_line { - push @Global::unget_next_command_line, @_; -} - -sub get_next_arg { - my $arg; - if(@Global::unget_arg) { - $arg = shift @Global::unget_arg; - } else { - if(eof $Global::argfile) { - return undef; - } - $arg = <$Global::argfile>; - chomp $arg; - if($Global::end_of_file_string and $arg eq $Global::end_of_file_string) { - # Ignore the rest of STDIN - while (<$Global::argfile>) {} - return undef; - } - if($Global::ignore_empty) { - if($arg =~ /^\s*$/) { - return get_next_arg(); - } - } - if($Global::input_is_filename) { - ($arg) = shell_quote($arg); - } - } - debug("Next arg: ".$arg."\n"); - return $arg; -} - -sub unget_arg { - push @Global::unget_arg, @_; -} - -sub drain_job_queue { - while($Global::running_jobs > 0) { - debug("jobs running: $Global::running_jobs Memory usage:".my_memory_usage()."\n"); - sleep 1; - } -} - -sub start_more_jobs { - my $jobs_started = 0; - if(not $Global::StartNoNewJobs) { - while($Global::running_jobs < $Global::processes_to_run - and - start_another_job()) { - $jobs_started++; - } - } - return $jobs_started; -} - -sub start_another_job { - # Grab a job from @Global::command, start it - # and remember the pid, the STDOUT and the STDERR handles - # If no more jobs: do nothing - # Do we have enough file handles to start another job? - if(enough_file_handles()) { - my $command = next_command_line(); - if(defined $command) { - my %jobinfo = start_job($command); - if(%jobinfo) { - $Global::running{$jobinfo{"pid"}} = \%jobinfo; - } - return 1; - } else { - return 0; - } - } else { - return 0; - } -} - -sub start_job { - # Setup STDOUT and STDERR for a job and start it. - my $command = shift; - my ($pid,$out,$err,%out,%err,$outname,$errname,$name); - if($Global::grouped) { - # To group we create temporary files for STDOUT and STDERR - # Filehandles are global, so to not overwrite the filehandles use a hash with new keys - # To avoid the cleanup unlink the files immediately (but keep them open) - $outname = ++$Global::TmpFilename; - ($out{$outname},$name) = tempfile(SUFFIX => ".par"); - unlink $name; - $errname = ++$Global::TmpFilename; - ($err{$errname},$name) = tempfile(SUFFIX => ".par"); - unlink $name; - - open STDOUT, '>&', $out{$outname} or die "Can't redirect STDOUT: $!"; - open STDERR, '>&', $err{$errname} or die "Can't dup STDOUT: $!"; - } - - if($Global::interactive or $Global::stderr_verbose) { - if($Global::interactive) { - print $Global::original_stderr "$command ?..."; - open(TTY,"/dev/tty") || die; - my $answer = ; - close TTY; - my $run_yes = ($answer =~ /^\s*y/i); - if (not $run_yes) { - open STDOUT, ">&", $Global::original_stdout or die "Can't dup \$oldout: $!"; - open STDERR, ">&", $Global::original_stderr or die "Can't dup \$oldout: $!"; - return; - } - } else { - print $Global::original_stderr "$command\n"; - } - } - if($Global::verbose and not $Global::grouped) { - print STDOUT $command,"\n"; - } - $Global::running_jobs++; - debug("$Global::running_jobs processes. Starting: $command\n"); - #print STDERR "LEN".length($command)."\n"; - $pid = open3(gensym, ">&STDOUT", ">&STDERR", $command) || - die("open3 failed. Report a bug to \n"); - debug("started: $command\n"); - open STDOUT, ">&", $Global::original_stdout or die "Can't dup \$oldout: $!"; - open STDERR, ">&", $Global::original_stderr or die "Can't dup \$oldout: $!"; - - $Global::job_start_sequence++; - if($Global::grouped) { - return ("seq" => $Global::job_start_sequence, - "pid" => $pid, - "out" => $out{$outname}, - "err" => $err{$errname}, - "command" => $command); - } else { - return ("seq" => $Global::job_start_sequence, - "pid" => $pid, - "command" => $command); - } -} - -sub print_job { - # Print the output of the jobs - # Only relevant for grouping - $Global::grouped or return; - my $fhs = shift; - if(not defined $fhs) { - return; - } - my $out = $fhs->{out}; - my $err = $fhs->{err}; - my $command = $fhs->{command}; - - debug(">>joboutput $command\n"); - if($Global::verbose and $Global::grouped) { - print STDOUT $command,"\n"; - # If STDOUT and STDERR is merged, we want the command to be printed first - # so flush to avoid STDOUT being buffered - flush STDOUT; - } - seek $_, 0, 0 for $out, $err; - if($Global::debug) { - print STDERR "ERR:\n"; - } - my $buf; - while(sysread($err,$buf,1000_000)) { - print STDERR $buf; - } - if($Global::debug) { - print STDOUT "OUT:\n"; - } - while(sysread($out,$buf,1000_000)) { - print STDOUT $buf; - } - debug("<{'command'},"\n"; - } -} - -sub StartNoNewJobs { - print STDERR ("$Global::progname: SIGTERM received. No new jobs will be started.\n", - "$Global::progname: Waiting for these ", scalar(keys %Global::running), - " jobs to finish. Send SIGTERM again to stop now.\n"); - ListRunningJobs(); - $Global::StartNoNewJobs++; - $SIG{TERM} = $Global::original_sigterm; -} - -sub CountSigChild { - $Global::SigChildCaught++; -} - -sub DoNotReap { - # This will postpone SIGCHILD for sections that cannot be distracted by a dying child - # (Racecondition) - $SIG{CHLD} = \&CountSigChild; -} - -sub ReapIfNeeded { - # Do the postponed SIGCHILDs if any and re-install normal reaper for SIGCHILD - # (Racecondition) - if($Global::SigChildCaught) { - $Global::SigChildCaught = 0; - Reaper(); - } - $SIG{CHLD} = \&Reaper; -} - -sub Reaper { - # A job finished. - # Print the output. - # Start another job - DoNotReap(); - $Global::reaperlevel++; - my $stiff; - debug("Reaper called $Global::reaperlevel\n"); - while (($stiff = waitpid(-1, &WNOHANG)) > 0) { - if($Global::keeporder) { - $Global::print_later{$Global::running{$stiff}{"seq"}} = $Global::running{$stiff}; - debug("died: $Global::running{$stiff}{'seq'}"); - while($Global::print_later{$Global::job_end_sequence}) { - debug("Found job end $Global::job_end_sequence"); - print_job($Global::print_later{$Global::job_end_sequence}); - delete $Global::print_later{$Global::job_end_sequence}; - $Global::job_end_sequence++; - } - delete $Global::running{$stiff}; - $Global::running_jobs--; - start_more_jobs(); - } else { - print_job($Global::running{$stiff}); - delete $Global::running{$stiff}; - $Global::running_jobs--; - start_more_jobs(); - } - } - ReapIfNeeded(); - debug("Reaper exit $Global::reaperlevel\n"); - $Global::reaperlevel--; -} - -# -# Usage -# - -sub die_usage { - usage(); - exit(1); -} - -sub usage { - print "Usage:\n"; - print "$Global::progname [options] [command [arguments]] < list_of_arguments\n"; - print "\n"; - print "See 'man $Global::progname' for the options\n"; -} - -sub version { - print join("\n", - "$Global::progname $Global::version", -# "Copyright (C) 2007,2008,2009,2010 Ole Tange and Free Software Foundation, Inc.", - "Copyright (C) 2007,2008,2009,2010 Ole Tange.", - "License GPLv3+: GNU GPL version 3 or later ", - "This is free software: you are free to change and redistribute it.", - "$Global::progname comes with no warranty.", - "", - "Web site: http://${Global::progname}.nongnu.org\n" - ); -} - - -# -# Debugging -# - -sub debug { - $Global::debug or return; - if($Global::original_stdout) { - print $Global::original_stdout @_; - } else { - print @_; - } -} - -sub my_memory_usage { - use strict; - use FileHandle; - - - my $pid = $$; - if(-e "/proc/$pid/stat") { - my $fh = FileHandle->new("; - chomp $data; - $fh->close; - - my @procinfo = split(/\s+/,$data); - - return $procinfo[22]; - } else { - return 0; - } -} - -sub my_size { - my @size_this = (@_); - eval "use Devel::Size qw(size total_size)"; - if ($@) { - return -1; - } else { - return total_size(@_); - } -} - - -sub my_dump { - my @dump_this = (@_); - eval "use Data::Dump qw(dump);"; - if ($@) { - # Data::Dump not installed - eval "use Data::Dumper;"; - if ($@) { - my $err = "Neither Data::Dump nor Data::Dumper is installed\n". - "Not dumping output\n"; - print STDERR $err; - return $err; - } else { - return Dumper(@dump_this); - } - } else { - eval "use Data::Dump qw(dump);"; - return (Data::Dump::dump(@dump_this)); - } -} - -# Keep perl -w happy -$main::opt_u = $main::opt_e = $main::opt_c = $main::opt_f = -$main::opt_q = $main::opt_0 = $main::opt_s = $main::opt_v = -$main::opt_g = $main::opt_P = $main::opt_D = $main::opt_m = -$main::opt_X = $main::opt_x = $main::opt_k = $main::opt_d = -$main::opt_P = $main::opt_i = $main::opt_p = $main::opt_a = -$main::opt_version = $main::opt_L = $main::opt_l = -$main::opt_show_limits = $main::opt_n = $main::opt_e = $main::opt_t = -$main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0;