diff options
| -rw-r--r-- | COPYING | 339 | ||||
| -rw-r--r-- | COPYING.lib | 504 | ||||
| -rw-r--r-- | ChangeLog | 1790 | ||||
| -rw-r--r-- | Makefile | 187 | ||||
| -rw-r--r-- | README | 14 | ||||
| -rw-r--r-- | bin/Makefile | 98 | ||||
| -rwxr-xr-x | bin/createrepo | 2 | ||||
| -rwxr-xr-x | bin/mergerepo | 2 | ||||
| -rwxr-xr-x | bin/modifyrepo | 2 | ||||
| -rw-r--r-- | createrepo.bash | 115 | ||||
| -rw-r--r-- | createrepo.spec | 80 | ||||
| -rw-r--r-- | createrepo/Makefile | 64 | ||||
| -rw-r--r-- | createrepo/__init__.py | 1344 | ||||
| -rw-r--r-- | createrepo/deltarpms.py | 124 | ||||
| -rw-r--r-- | createrepo/merge.py | 139 | ||||
| -rw-r--r-- | createrepo/readMetadata.py | 217 | ||||
| -rw-r--r-- | createrepo/utils.py | 143 | ||||
| -rw-r--r-- | createrepo/yumbased.py | 235 | ||||
| -rwxr-xr-x | dmd.py | 156 | ||||
| -rw-r--r-- | docs/Makefile | 99 | ||||
| -rw-r--r-- | docs/createrepo.8 | 139 | ||||
| -rw-r--r-- | docs/mergerepo.1 | 56 | ||||
| -rw-r--r-- | docs/modifyrepo.1 | 41 | ||||
| -rwxr-xr-x | genpkgmetadata.py | 277 | ||||
| -rwxr-xr-x | mergerepo.py | 85 | ||||
| -rwxr-xr-x | modifyrepo.py | 148 | ||||
| -rwxr-xr-x | worker.py | 99 |
27 files changed, 6499 insertions, 0 deletions
@@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 675 Mass Ave, Cambridge, MA 02139, USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) 19yy <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) 19yy name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/COPYING.lib b/COPYING.lib new file mode 100644 index 0000000..8add30a --- /dev/null +++ b/COPYING.lib @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + <one line to give the library's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + <signature of Ty Coon>, 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..fb537ec --- /dev/null +++ b/ChangeLog @@ -0,0 +1,1790 @@ +2011-01-26 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: mark as 0.9.9 + +2011-01-26 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Override timestamp check on repos. for mergerepo (like repodiff). + Add createrepo --workers (non)completion. Add modifyrepo option + completion. + +2011-01-21 James Antill <james@and.org> + + * createrepo/merge.py: Override timestamp check on repos. for + mergerepo (like repodiff). + +2011-01-03 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: make sure when we want to look for rpms we + say .rpm not rpm b/c with the latter we catch .drpm files, too. :( + +2010-11-02 Ville Skyttä <ville.skytta@iki.fi> + + * createrepo.bash: Add createrepo --workers (non)completion. + +2010-11-02 Ville Skyttä <ville.skytta@iki.fi> + + * createrepo.bash: Add modifyrepo option completion. + +2010-10-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: - add yum 3.2.29 requirement b/c of the small change I needed to + repoMDObject.py - set it to use /usr/share/createrepo/worker.py + +2010-10-07 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: remove libxml2 import from __init__.py :) + + +2010-10-07 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: make createrepo use the repomd/repodata + mechanism from yum for making a repomd.xml which simplifies the code + dramatically since we don't have to mess with xml in here. + +2010-10-07 Seth Vidal <skvidal@fedoraproject.org> + + * modifyrepo.py: fix up the usage output for modifyrepo + +2010-09-10 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, worker.py: - make sure we handle remote_url pkgs correctly until we get the + worker hooked up to handle them - if there are no pkgs to handle, + don't launch workers with nothing to do. - give better output from + the workers and have them obey -v/-q - everyone loves callbacks! + +2010-09-09 Seth Vidal <skvidal@fedoraproject.org> + + * Makefile, createrepo/__init__.py, createrepo/utils.py, + createrepo/yumbased.py, genpkgmetadata.py, worker.py: create a + worker script for createrepo so createrepo can fork off N processes + to handle the md gathering from pkgs. This should speed up results + on systems which have been cpubound on the createrepo process. If + you're io bound it won't help you at all, and MAY make it worse. + many misc issues to iron out here - not the least of which is the + callback output and gathering stdout/stderr from the workers + +2010-08-20 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: handle broken locking on nfs target dirs + better if database is true. - sqlite dbs don't like being made on + locations without locking available. - if we know we're going to be + creating dbs then we should attempt to lock before doing anything + else and bail out nicely if we can't get an exclusive lock + +2010-08-19 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec: require yum 3.2.28 due to the imports in + modifyrepo + +2010-08-19 Seth Vidal <skvidal@fedoraproject.org> + + * docs/modifyrepo.1: document --mdtype option + +2010-08-19 Seth Vidal <skvidal@fedoraproject.org> + + * modifyrepo.py: - add option parsing for --mdtype - use the yum repomd objects to + read/write the repomd.xml - add mdtype option to RepoMetadata.add() + method + +2010-06-11 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Don't use /usr/bin/env ... it's evil --database is now the default + for mergerepo too, have --no-database in completions instead. + +2010-06-11 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: - add option to createrepo config to collapse libc.so.6 requires - + this will only work with yum 3.2.28 and beyond + +2010-06-03 James Antill <james@and.org> + + * modifyrepo.py: Don't use /usr/bin/env ... it's evil + +2010-06-02 Ville Skyttä <ville.skytta@iki.fi> + + * createrepo.bash: --database is now the default for mergerepo too, + have --no-database in completions instead. + +2010-06-01 Seth Vidal <skvidal@fedoraproject.org> + + * mergerepo.py: whoops - no-database shouldn't default to true! + +2010-06-01 Seth Vidal <skvidal@fedoraproject.org> + + * mergerepo.py: add --no-database to mergrepo, too + +2010-05-31 Ville Skyttä <ville.skytta@iki.fi> + + * createrepo.bash: --database is now the default, have --no-database + in completions instead. + +2010-05-28 Seth Vidal <skvidal@fedoraproject.org> + + * docs/createrepo.8: update the docs for --no-database + +2010-05-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: make -d/--database the + default add --no-database in case someone somewhere needs to do + that + +2010-04-26 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: fixme comments about try/excepting the + database creation calls due to a weird issue with locks not working + on a nfs mount and createreepo tracing back with a TypeError of all + things + +2010-04-21 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/readMetadata.py: if we cannot + find one of the old repodata files make the warning look more dire + make sure we look for the 'repodata' subdir inside update_md_path + +2010-04-21 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: when the update_md_path doesn't exist - + emit a warning of some kind - rather than a somewhat quieter message + from MetadataIndex() this is mostly to help jesse b/c he asked + nicely + +2010-04-16 Colin Walters <walters@fedoraproject.org> + + * genpkgmetadata.py: if we're not a tty, don't use the progress + output + +2010-04-15 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Tell git to ignore tarballs. Document --repo in man page. Add + bash completion. + +2010-04-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: - catch errors when moving the olddir out/back - if we get a + yumLocalPackage object in our pkglist we should record we read it. + +2010-03-05 Ville Skyttä <ville.skytta@iki.fi> + + * .gitignore: Tell git to ignore tarballs. + +2010-03-05 Ville Skyttä <ville.skytta@iki.fi> + + * docs/createrepo.8: Document --repo in man page. + +2010-02-19 Ville Skyttä <ville.skytta@iki.fi> + + * Makefile, createrepo.bash, createrepo.spec: Add bash completion. + +2010-03-05 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Trim trailing whitespace. + +2010-03-05 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: add repo tags and + --repo option to describe the repo itself. request from suse. + +2010-02-12 Ville Skyttä <ville.skytta@iki.fi> + + * createrepo/__init__.py, createrepo/deltarpms.py, + createrepo/merge.py, createrepo/readMetadata.py, + createrepo/utils.py, createrepo/yumbased.py, dmd.py, + genpkgmetadata.py, mergerepo.py, modifyrepo.py: Trim trailing + whitespace. + +2010-02-11 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: add option for setting max-delta-rpm-size for + pkgs which are too large to be delta'd. + +2010-02-10 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Make *Emacs unsuspicious about trailing whitespace. Fix --exclude + -> --excludes typo. Add missing spaces in various help strings. + +2010-02-10 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: add --read-pkgs-list + option to output list of pkgs actually read. completely optional + and only really useful with --update or a --cachedir for what pkgs + DID get read/parsed. + +2010-02-09 Ville Skyttä <ville.skytta@iki.fi> + + * Makefile: Make *Emacs unsuspicious about trailing whitespace. + +2010-02-09 Ville Skyttä <ville.skytta@iki.fi> + + * docs/createrepo.8: Fix --exclude -> --excludes typo. + +2010-02-09 Ville Skyttä <ville.skytta@iki.fi> + + * genpkgmetadata.py: Add missing spaces in various help strings. + +2010-02-08 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Add missing space in --checkts help string. Ignore *.py[co]. + Remove outdated comment about --baseurl. + +2010-02-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, docs/createrepo.8, genpkgmetadata.py: - make --unique-md-filenames the default - add simple-md-filenames + to be able to disable the above if desired + +2010-01-18 Ville Skyttä <ville.skytta@iki.fi> + + * genpkgmetadata.py: Add missing space in --checkts help string. + +2009-09-25 Ville Skyttä <ville.skytta@iki.fi> + + * .gitignore: Ignore *.py[co]. + +2009-01-26 Ville Skyttä <ville.skytta@iki.fi> + + * docs/createrepo.8: Remove outdated comment about --baseurl. At + least yum uses it nowadays. + +2010-01-07 Dennis Gregorovic <dgregor@redhat.com> + + * createrepo/readMetadata.py: Fixed, convert stat mtime to int so + comparison can work, --update, BZ 553030 + +2010-01-07 Dennis Gregorovic <dgregor@redhat.com> + + * createrepo/readMetadata.py: Convert stat mtime to int so + comparison can work, --update, BZ 553030 + +2010-01-06 Dennis Gregorovic <dgregor@redhat.com> + + * createrepo/__init__.py: Change baseurl of "old" packages on + update, when baseurl specified + +2009-10-05 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: apply fix for + https://bugzilla.redhat.com/show_bug.cgi?id=527259 make sure we're + not only testing the first pkg. Test all of them until we find one + that is newer. + +2009-09-14 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec: add requirement on python-deltarpm for new patch + from Bill. + +2009-09-14 Bill Nottingham <notting@redhat.com> + + * createrepo/deltarpms.py: createrepo patch to use the new deltarpm + python bindings + +2009-08-28 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: changelog merge + +2009-08-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: mark as 0.9.8 + +2009-08-28 Seth Vidal <skvidal@fedoraproject.org> + + * docs/createrepo.8: add man page entry for -n/--includepkg + +2009-08-25 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: add -n, --includepkg option to allow users to + specify urls to pkgs to include in the repo on the cli - like a + fully cli version of --pkglist/-i + +2009-08-25 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: capture ioerror/oserrors on handling + metadata files and emit a proper MDError fixes rh bug: + https://bugzilla.redhat.com/show_bug.cgi?id=514995 + +2009-08-18 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: commit obviously broken pragma setting fix + from mikeb@redhat.com + +2009-07-21 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: fix for + https://bugzilla.redhat.com/show_bug.cgi?id=512610 take timestamp of + repomd.xml - not of repodata dir - just in case repodata dir is + empty, for some bizarre reason + +2009-06-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: remove extra 0 from max_delta_rpm_size + +2009-06-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: more/better output about makedeltarpm + timing + +2009-06-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: output how long it took to make the drpm + file + +2009-06-16 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: - prestodelta.xml file generation is now roughly 60X faster than it + was before - python unicode string concatenation sucks, a lot. - + add a delta xml generation profile output - get rid of some + incorrect output about db files and delta metadata - get rid of some + old not-useful comments in the code + +2009-05-14 James Antill <james@and.org> + + * genpkgmetadata.py: Make the UI for --checksum a bit nicer + +2009-05-14 James Antill <james@and.org> + + * docs/createrepo.8: Fix -profile in man page, to be --profile + +2009-05-14 James Antill <james@and.org> + + * docs/createrepo.8: Add some more documentation about --checksum + +2009-05-13 James Antill <james@and.org> + + * createrepo/__init__.py: Add open-size and size fo *_db MD. Fix + file to stat for *.xml.gz size + +2009-05-13 James Antill <james@and.org> + + Merge branch 'master' of + ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo * + 'master' of + ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo: if + our deltarpm dir doesn't exist, don't go looking for it - and + definitely + +2009-05-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: if our deltarpm dir doesn't exist, don't + go looking for it - and definitely don't traceback + +2009-05-05 James Antill <james@and.org> + + Merge branch 'size-in-repomd.xml' * size-in-repomd.xml: Add + size to the repomd.xml output + +2009-04-24 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: pylint fixes for __init__ - lots of line + cleanups and a couple of potential bugs. + +2009-04-24 Seth Vidal <skvidal@fedoraproject.org> + + * modifyrepo.py: pylint clean up on modifyrepo + +2009-04-24 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: genpkgmetadata.py pylint cleanup. + +2009-04-22 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: if we've not enabled the deltas, don't try + to do them. silly, but harmless in this case + +2009-04-21 Tim Lauridsen <timlau@fedoraproject.org> + + * createrepo/__init__.py: pylint: fixed Uses of a deprecated module + 'string' + +2009-04-21 Tim Lauridsen <timlau@fedoraproject.org> + + * createrepo/__init__.py, createrepo/utils.py: pylint: fixed + Redefining built-in + +2009-04-21 Tim Lauridsen <timlau@fedoraproject.org> + + * createrepo/deltarpms.py, createrepo/yumbased.py: pylint: fixed + unused imports + +2009-04-21 Tim Lauridsen <timlau@fedoraproject.org> + + * createrepo/readMetadata.py: pylint: fixed Bad indentation + +2009-04-21 Tim Lauridsen <timlau@fedoraproject.org> + + * Makefile: Added the pylint basic and disabled the warning we dont + care about + +2009-04-18 James Antill <james@and.org> + + * createrepo/__init__.py: Fix copy and paste error on message + +2009-04-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py: make sure our + sumtype specified propagates down to the pkg checksums, too + +2009-04-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: set a max size option so we don't kill + systems with < memory than deltarpm likes to use. + +2009-04-17 James Antill <james@and.org> + + * createrepo/yumbased.py: Use the same checksum type for the key, as + for the data in the key + +2009-04-16 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: remove the deprecation notice since: 1. it + works fine 2. there is a legit use for it + +2009-04-16 Seth Vidal <skvidal@fedoraproject.org> + + * docs/createrepo.8: document the deltarpm options + +2009-04-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: it helps to have the right order of items + in the pkgtup :( + +2009-04-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: quiet down output + +2009-04-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: make sure we don't try to sqlite the + prestodelta xml, yet. + +2009-04-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: add missing '>' + +2009-04-13 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/utils.py, modifyrepo.py: make + sure the checksum type we use is being used everywhere. closes + rhbug: https://bugzilla.redhat.com/show_bug.cgi?id=494951 + +2009-03-24 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: changelog merge + +2009-03-24 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: 0.9.7 require yum 3.2.22 + + +2009-02-09 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/deltarpms.py: when we process + the rpms only do the drpm creation. after we're done take the drpms + and generate the metadata from there + +2009-02-03 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/merge.py: some fixes and to make it work on + rhel5/python2.4 + +2009-02-03 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: and one more mistake + +2009-02-03 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: correct tabbing so createrepo works when + you're NOT using deltas + +2009-01-29 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: add --num-deltas option + +2009-01-29 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Add missing documentation on --checksum and --profile + +2009-01-29 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/deltarpms.py, + genpkgmetadata.py: --deltas, enable the creation and + metadata-creation for presto/deltarpms + +2009-01-27 James Antill <james@and.org> + + Merge branch 'master' of + ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo * + 'master' of + ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo: make + modifyrepo behave with sha256 as the default checksum + +2009-01-27 James Antill <james@and.org> + + * docs/createrepo.8: Add missing documentation on --checksum and + --profile + +2009-01-27 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/utils.py, modifyrepo.py: make modifyrepo behave with + sha256 as the default checksum + +2009-01-26 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: + make sha256 the default checksum type everywhere + +2009-01-23 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/merge.py: add init options to specify your own yumbase + object, mdconf object md generator class + +2009-01-22 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/utils.py: make sure we keep working on python 2.4 :( + +2009-01-22 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/utils.py: use gzip.name not gzip.filename to avoid + python 2.6 deprecation warnings + +2009-01-22 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: get rid of the md5 badness - use yum's + Checksum class so we don't have to deal with python 2.4 vs 2.6isms + +2009-01-22 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: add --profile option to + the cli interface so profile info is outputted only when it is used. + + +2009-01-19 James Antill <james@and.org> + + * createrepo/yumbased.py: Use correct cachedir after rename + +2008-12-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: allow alternative path + for --update via --update-md-path, So your old repodata does not + have to be in the path you want to look through. + +2008-10-28 Seth Vidal <skvidal@fedoraproject.org> + + * modifyrepo.py: try/excepts on modifyrepo so we don't smack the + user with a traceback + +2008-10-28 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: remerge changelog + +2008-10-28 Seth Vidal <skvidal@fedoraproject.org> + + * bin/Makefile: minor changes to the make file so that it will make + a proper archive :) + +2008-10-27 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: merge changelog + +2008-10-23 Seth Vidal <skvidal@fedoraproject.org> + + * modifyrepo.py: allow already-compressed metadata files to work and + not be double-compressed + +2008-10-21 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, docs/Makefile, docs/mergerepo.1, mergerepo.py: + mergerepo man page todos added to mergerepo + +2008-10-21 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo + * 'master' of + ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: + Change the NamedTemporaryFile() usage to mkstemp(), stupid API Fix + parallel updates to the cachedir, thx to Michael Schwendt for + spotting it + +2008-10-21 Seth Vidal <skvidal@fedoraproject.org> + + * AUTHORS, README, createrepo.spec, createrepo/__init__.py, + docs/createrepo.8, genpkgmetadata.py: - document --content, --distro and --revision - update urls in spec + and docs - add Authors file + +2008-10-17 Seth Vidal <skvidal@fedoraproject.org> + + * Makefile: add merge repo here, too + +2008-10-17 Seth Vidal <skvidal@fedoraproject.org> + + * mergerepo.py: pylintian cleanups + +2008-10-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: bump version to 0.9.6 - + change the dep to yum 3.2.20 - since it is what WILL be needed + +2008-10-17 Seth Vidal <skvidal@fedoraproject.org> + + * bin/Makefile, bin/mergerepo, createrepo/merge.py, mergerepo.py: + add mergerepo + +2008-10-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: add arbitrary metadata to config options + for api callers + +2008-10-09 James Antill <james@and.org> + + * createrepo/yumbased.py: Change the NamedTemporaryFile() usage to + mkstemp(), stupid API + +2008-10-08 James Antill <james@and.org> + + * createrepo/yumbased.py: Fix parallel updates to the cachedir, thx + to Michael Schwendt for spotting it + +2008-09-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec: bump the yum requirement to 3.2.19.. + +2008-09-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/utils.py: remove unused utf8String function from utils + - move most of it into yum.misc + +2008-09-12 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: make the profile option work again + +2008-08-13 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: + remove most of the yumbased code, disable database-only for now + +2008-08-08 James Antill <james@and.org> + + * createrepo/__init__.py: Add size to the repomd.xml output + +2008-08-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/utils.py, + createrepo/yumbased.py: minor changes for handling a packagesack + and/or list of package objects as the pkglist to create the repo + from also rename the xml_dump functions - eventually going to remove + them. + +2008-06-17 James Antill <james@and.org> + + * docs/createrepo.8: Add missing doc. for --skip-stat option + +2008-06-05 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: + some fixmes and starts - and recommit a working --database-only + +2008-05-12 James Antill <james@and.org> + + * createrepo/__init__.py, genpkgmetadata.py: Remove -n option, it's + a noop atm. anyway + +2008-05-12 James Antill <james@and.org> + + * createrepo/__init__.py: Pass just dir. to getFileList(), makes -C + work. Fixes bug#446040 + +2008-04-16 James Antill <james@and.org> + + * createrepo/utils.py: Talk to libxml maintainer ... tweak + +2008-04-16 James Antill <james@and.org> + + * createrepo/utils.py: Just remove bad small bytes, like 0x01 atm. + +2008-04-02 James Antill <james@and.org> + + * docs/createrepo.8: Add some missing options to man page + +2008-03-11 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/utils.py: a few tweaks to speed + up the database creation + +2008-03-11 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/utils.py, + createrepo/yumbased.py, docs/createrepo.8: more or less complete + createrepo --database-only + +2008-03-11 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: - partial patch to enable --database-only output from createrepo - + still to implement filelists direct to sqlite, changelogs direct to + sqlite - this check in is just a hedge against loss from my laptop, + do not use the feature in this commit + +2008-03-03 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: better name for node + +2008-03-03 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py: exclude rpmlib + requires from metadata b/c they are silly store them in the + repomd.xml per-repo so we have them if we ever actually need them + +2008-02-29 James Antill <james@and.org> + + * genpkgmetadata.py: Fix line overflow, minor IO optimisation + +2008-02-20 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: + allow --pkglist or self.conf.pkglist to be a list of arbitrary urls + to packages. createrepo will download the remote urls to a tempdir, + read them in and add them to the metadata. + +2008-02-18 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: merge changelog + +2008-02-18 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: bump version numbers + +2008-02-18 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py: - clean up some garbage spaces and an extra 'return' - write some + notes on something interesting to do for completely arbitrary + repositories - make sure that under no circumstances will a package + that we cannot get a pkgid/checksum from will be in the metadata. + And it will output an error message + +2008-02-13 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: raise, don't print + +2008-02-12 Luke Macken <lmacken@redhat.com> + + * genpkgmetadata.py: Clean up some more unused modules + +2008-02-12 Luke Macken <lmacken@redhat.com> + + * createrepo/__init__.py, createrepo/utils.py: If we want to use + MDError in utils.py, we need to define it outside of __init__ to + avoid circular deps + +2008-02-12 Luke Macken <lmacken@redhat.com> + + * createrepo/__init__.py: Pull in createrepo.utils.errorprint in our + __init__ module. + +2008-02-12 Luke Macken <lmacken@redhat.com> + + * createrepo/__init__.py: Import shutil since we use it in + createrepo.__init__ + +2008-02-12 Luke Macken <lmacken@redhat.com> + + * createrepo/__init__.py: s/conf.checkts/self.conf.checkts/ + +2008-02-12 Luke Macken <lmacken@redhat.com> + + * createrepo/__init__.py, createrepo/readMetadata.py, + createrepo/yumbased.py: Remove a bunch of module imports that we + aren't using. One of which being 'hashlib', which prevents + createrepo from running on anything by Python 2.5. + +2008-01-31 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/readMetadata.py, + createrepo/utils.py, modifyrepo.py: - make sure group files are compressed/sha-named - add group_gz + section for compressed group file - add addArbitraryMetadata() + method to MetaDataGenerator class - fix up modifyrepo to generate + sha-named files - make modifyrepo act a bit more like createrepo for + its operations + +2008-01-30 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: changelog merge + +2008-01-29 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: make sure things work + out as the right default + +2008-01-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: bump ver to 0.9.4 in spec + and module + +2008-01-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: make sure non-unique-md-filenanmes-repos + cleanup sqlite files if we switch to unique-md-filenames + +2008-01-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: clean up old versions of primary, + filelists and other that are lingering in the repodata dir due to + sha1-addition + +2008-01-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: swap around the filename creation order so + it doesn't make leaves files around + +2008-01-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/readMetadata.py: make + readMetadata.py take its metadata file locations from repomd.xml + using the yum parser for it. complete --unique-md-filenames + implementation + +2008-01-28 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: part of + --unique-md-filenames is complete. This lets createrepo generate + metadata files with the file's checksum included in the filename. + This helps it work more nicely with proxies. --update support will + not work with --unique-md-filenames at the moment. Need to read in + repomd.xml to make that work. + +2008-01-22 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: changelog merge + +2008-01-22 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: 0.9.3 + +2008-01-22 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: + add changelog-limit option to restrict the number of changelogs we + add, by default + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: make sure empty directories still work + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog: update changelog + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: comment out a debug print + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py: re-enable + --cachedir in code fix logic issues in cachedir from 0.4.X + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: - fix more problems with relative paths and --split - revert + cachedir disabling - cachedir isn't working yet, but it's no longer + deprecated due to a use case I hadn't considered. + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: make --update and --split mostly work + again + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: - move a bunch of tests into the base class - deprecate --cachedir - + make it enable --update instead b/c it is MUCH faster + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: bump version to 0.9.2 + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: add some more correct outputs about the + sqlite db generation + +2008-01-17 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: --split works again, + fix relative paths with ../../ curse in the direction of --split and + --basedir :( + +2008-01-16 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/utils.py, + createrepo/yumbased.py, genpkgmetadata.py: clean up api to simplify + all of the code calling it probably severely break --split for the + moment, though + +2008-01-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: make sure we have empty stubs for items in + the rpm-format section, older versions of yum will go bonkers if + not. + +2008-01-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: - add debug output for database time generation + +2008-01-15 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/readMetadata.py, + genpkgmetadata.py: - add --skip-stat to skip the stat() call on updates - this is + mostly b/c the fedora createrepo call is done over nfs and that is + VERY VERY slow for 20K stat() calls :( - fix --pkglist - patch + from jesse keating - document options in --help output + +2008-01-14 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/yumbased.py: - add copyright statements and licenses to top of files - start + function to remove all the directory mauling in genpkgmetadata - + fixmes + +2008-01-11 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: return src for arch when it's a srpm + +2008-01-10 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: make sure that files are run through the + xml escaping, too. + +2008-01-09 Seth Vidal <skvidal@fedoraproject.org> + + * ChangeLog, Makefile: update changelog, add changelog target to + makefile + +2008-01-09 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: clean up old comments and cruft + +2008-01-09 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/readMetadata.py, + genpkgmetadata.py: - clean up interface a lot - add callback interface for progress + output - more proper catching of exceptions - remove improper + sys.exit() calls from the base class + +2008-01-09 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: make --update work correctly - by getting + the right item from os.stat() for mtime + +2008-01-09 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: free up memory in the changelog output + used by generating the xml node + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: clean up a debug output + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/readMetadata.py, + createrepo/yumbased.py: fix up a lot of xml creation errors and make + --update work again + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, createrepo/__init__.py: bump to 0.9.1 + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: and a little more utf8'ing - just for + completeness and insanity + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: utf8 files, too :( + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: all of the xml tools suck, this is + evidence of them fall back to the generating the xml via libxml2 for + the changelog hopefully this won't make memory explode + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: politely bounce over damaged rpms - output + an error report new errorlog() method - can be easily replaced in + subclass + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: remove debug prints :) + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py: make sure that we check for nonexistent + items in the hdr + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: make it a more proper ts + +2008-01-08 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py: try except on package opening + +2008-01-07 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/yumbased.py, genpkgmetadata.py: - clean out old classes from yumbased.py - clean out debug prints + from genpkgmetadata.py + +2008-01-07 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: make the version stuff make sense + +2008-01-07 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo.spec, genpkgmetadata.py: - make rpmbuild work - mark a fixme + +2008-01-03 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, createrepo/utils.py, + createrepo/yumbased.py, genpkgmetadata.py: - port to optionparser from getopt - redo config class to make use + outside of cli more do-able - handle repomd.xml creation in class, + too - still have a lot of changes to complete + +2007-12-20 Seth Vidal <skvidal@fedoraproject.org> + + * createrepo/__init__.py, genpkgmetadata.py: a little more + class-full + +2007-12-20 Seth Vidal <skvidal@fedoraproject.org> + + * Makefile, bin/Makefile, createrepo.spec, createrepo/Makefile, + createrepo/__init__.py, createrepo/readMetadata.py, + createrepo/utils.py, createrepo/yumbased.py, docs/Makefile, + dumpMetadata.py, genpkgmetadata.py, readMetadata.py: Whew: this is + the beginning of a big conversion of createrepo to use the yum + modules, behave more like a modular program and have a proper class + structure. It's not done, but it's a start. + +2007-12-18 Seth Vidal <skvidal@fedoraproject.org> + + Merge branch 'master' of + ssh://login.linux.duke.edu/home/groups/createrepo/git/createrepo * + 'master' of + ssh://login.linux.duke.edu/home/groups/createrepo/git/createrepo: + Update ChangeLog Remove some unnecessary imports Better unicode + handling in modifyrepo Add a man page for modifyrepo + +2007-12-18 Seth Vidal <skvidal@fedoraproject.org> + + * docs/createrepo.8, dumpMetadata.py, genpkgmetadata.py, + modifyrepo.py: merge maintenance changes up to head before we nuke + it from orbit + +2007-12-06 Luke Macken <lmacken@redhat.com> + + * ChangeLog: Update ChangeLog + +2007-12-06 Luke Macken <lmacken@redhat.com> + + * dumpMetadata.py, readMetadata.py: Remove some unnecessary imports + + +2007-12-06 Luke Macken <lmacken@redhat.com> + + * modifyrepo.py: Better unicode handling in modifyrepo + +2007-12-03 Luke Macken <lmacken@redhat.com> + + * ChangeLog, createrepo.spec, docs/Makefile, docs/modifyrepo.1, + modifyrepo.py: Add a man page for modifyrepo + +2007-11-14 Seth Vidal <skvidal@fedoraproject.org> + + * genpkgmetadata.py: merge pkglist option to HEAD + +2007-08-08 Seth Vidal <skvidal@fedoraproject.org> + + * README: update readme, point to better url, clean up explanation + +2007-07-01 James Bowes <jbowes@redhat.com> + + * dmd.py: Add delta metadata diff and patch script + +2007-06-07 Paul Nasrat <pnasrat@redhat.com> + + * createrepo.spec: Bump version + +2007-06-07 Paul Nasrat <pnasrat@redhat.com> + + * ChangeLog, Makefile: Prepare for release + +2007-06-07 Paul Nasrat <pnasrat@redhat.com> + + * Makefile, docs/createrepo.8, genpkgmetadata.py, readMetadata.py: + This patch adds a --update option to createrepo. + + https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html Patch from Mike Bonnet <mikeb@redhat.com> + +2007-05-18 Paul Nasrat <pnasrat@redhat.com> + + * dumpMetadata.py: Fix for older rpm versions Christoph Thiel + <cthiel@suse.de> + +2007-05-16 Paul Nasrat <pnasrat@redhat.com> + + * ChangeLog, Makefile, createrepo.spec, genpkgmetadata.py: Update + ChangeLog Bump version to 0.4.9 + +2007-05-16 Paul Nasrat <pnasrat@redhat.com> + + * dumpMetadata.py: Figure out appropriate dbversion Jeremy Katz + <katzj@redhat.com> + +2007-05-16 Paul Nasrat <pnasrat@redhat.com> + + * dumpMetadata.py: createrepo-0.4.8-cachefix.patch * changes the way the hashkey + for the cache is generated. (The original version just used + rpm.RPMTAG_SIGMD5, which was the same for the same signed and + unsigned package. However, this lead to a wrong checksum + ending up in the metadata.) Christoph Thiel <cthiel@suse.de> + +2007-05-16 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: createrepo-0.4.8-skip-symlinks.patch * adds an option to skip + symlinks (-S, --skip-symlinks) Christoph Thiel <cthiel@suse.de> + +2007-02-13 Seth Vidal <skvidal@linux.duke.edu> + + * ChangeLog: update changelog, again + +2007-02-13 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile: add copying and copying.lib to makefile for 'make + archive' + +2007-02-13 Seth Vidal <skvidal@linux.duke.edu> + + * ChangeLog: check in changelog + +2007-02-13 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile: and makefile ver + +2007-02-13 Seth Vidal <skvidal@linux.duke.edu> + + * createrepo.spec, genpkgmetadata.py: mark as 0.4.8 + +2007-02-08 Paul Nasrat <pnasrat@redhat.com> + + * COPYING.lib, createrepo.spec: Add LGPL file + +2007-02-08 Paul Nasrat <pnasrat@redhat.com> + + * COPYING, createrepo.spec: Add COPYING + +2007-02-07 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: merge in Christoph Thiel's patch to fix string + conversion for odd EVR's + +2007-02-07 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: merge Jesse Keatings' patch to find groups file + properly + +2007-02-07 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile: ver number in Makefile + +2007-02-06 Seth Vidal <skvidal@linux.duke.edu> + + * createrepo.spec: yum-metadata-parser dep and new version number + +2007-02-06 Seth Vidal <skvidal@linux.duke.edu> + + * docs/createrepo.8: update docs for -d + +2007-02-06 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: 0.4.7 version number + +2007-02-04 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: make database version listed in repomd + +2007-02-04 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: add dbversion to sqlite metadata in repomd. + +2007-02-03 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: default to max compression + +2007-02-03 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: make the sqlite file names not look stupid + +2007-02-03 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: allow optionally creating + compressed sqlite databases + +2006-10-22 Luke Macken <lmacken@redhat.com> + + * ChangeLog, modifyrepo.py: use the mdname for the 'href' element, + so it doesn't explode when dealing with xml.dom.minidom.Document + objects. + +2006-10-14 Luke Macken <lmacken@redhat.com> + + * ChangeLog, Makefile, bin/Makefile, createrepo.spec: 2006-10-14 + 01:30 lmacken * Makefile, bin/Makefile, createrepo.spec: + Makefile changes for modifyrepo, and added it to the spec as + well. + +2006-08-23 Luke Macken <lmacken@redhat.com> + + * ChangeLog, bin/modifyrepo, modifyrepo.py: 2006-08-23 15:40 + lmacken * modifyrepo.py, bin/modifyrepo: Initial import + +2006-08-11 Paul Nasrat <pnasrat@redhat.com> + + * ChangeLog: Update changelog with cvs2cl + +2006-08-11 Paul Nasrat <pnasrat@redhat.com> + + * createrepo.spec: update date + +2006-08-11 Paul Nasrat <pnasrat@redhat.com> + + * docs/createrepo.8, genpkgmetadata.py: Patch from Hans-Peter Jansen + <hpj@urpla.net> -C, --checkts option added to avoid metadata + generation, if ctime filestamps are up to date. It's currently + mutually exclusive with the --split option. + +2006-07-28 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Fix cache output dir to 0.4.5 behaviour + +2006-07-28 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Fix filtering out path from file list and + passing correct path to writeMetaData + +2006-07-28 Paul Nasrat <pnasrat@redhat.com> + + * test/testMetaDataGenerator.py, test/testSplitMetaDataGenerator.py: + nuke tests for now + +2006-07-21 Paul Nasrat <pnasrat@redhat.com> + + * Makefile: Bump + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Make splitmetadata handler do it' own + getFileList to correctly manipulate paths. + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * test/testSplitMetaDataGenerator.py: Improve tests for split cases + + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * test/testSplitMetaDataGenerator.py: duplicate for split tests + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * test/testMetaDataGenerator.py: More consistent naming Relative and + parallel dir testing + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * test/testMetaDataGenerator.py: Refactor tests, add additional + tests + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * test/testMetaDataGenerator.py: Start unit testing so we don't + regress behaviour + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Set outputdir correctly + +2006-07-20 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Move to split basedir and directory everywhere + to preserve command line paths. Use os.path.walk rather than our own + implementation Improve error messages + +2006-07-19 Paul Nasrat <pnasrat@redhat.com> + + * createrepo.spec: genpkgmetadata.py + +2006-07-19 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Consistent directory handling and errors + +2006-07-19 Paul Nasrat <pnasrat@redhat.com> + + * dumpMetadata.py: Patch from hpj@urpla.net to use a more robust rpm + header signature retrieval method for cache files, as recommended by + Jeff Johnson. + +2006-07-19 Luke Macken <lmacken@redhat.com> + + * ChangeLog, createrepo.spec: 2006-07-19 14:23 lmacken * + createrepo.spec: remove python-urlgrabber dependency + +2006-07-19 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Tolerate unknown files in repodata dirs - Ville + Skyttä + +2006-07-19 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: fix up relative paths (#199228) + +2006-06-30 Paul Nasrat <pnasrat@redhat.com> + + * dumpMetadata.py: Fix srpm detection for rpm-4.4.6 and later + +2006-06-26 Seth Vidal <skvidal@linux.duke.edu> + + * ChangeLog: overwrite changelog + +2006-06-15 Luke Macken <lmacken@redhat.com> + + * ChangeLog, docs/createrepo.8, genpkgmetadata.py: 2006-06-15 11:40 + lmacken * genpkgmetadata.py, docs/createrepo.8: Revert + --update-info-location patch, since yum now supports arbitrary + metadata via YumRepository::retrieveMD() + +2006-06-09 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: include Panu's patch to + support --noepoch for use with old versions of rpm + +2006-06-09 Seth Vidal <skvidal@linux.duke.edu> + + * createrepo.spec: fix the dep + +2006-06-09 Seth Vidal <skvidal@linux.duke.edu> + + * createrepo.spec, genpkgmetadata.py: fix versions and bump by one. + Thanks to Gareth Armstrong for noticing this. + +2006-03-04 Paul Nasrat <pnasrat@redhat.com> + + * ChangeLog: add changelog + +2006-03-04 Paul Nasrat <pnasrat@redhat.com> + + * createrepo.spec: release + +2006-02-21 Paul Nasrat <pnasrat@redhat.com> + + * Makefile, createrepo.spec, docs/createrepo.8, genpkgmetadata.py: + Documentation and version updates + +2006-02-21 Paul Nasrat <pnasrat@redhat.com> + + * dumpMetadata.py, genpkgmetadata.py: Enable seperate outputdir + (dgregor) + +2006-02-18 Luke Macken <lmacken@redhat.com> + + * docs/createrepo.8, genpkgmetadata.py: Add support for -U + (--update-info-location) flag to query a specified server for + package update metadata. The metadata will currently be stored in + 'repodata/update-info' and each package in the primary.xml will have + an <update id="FEDORA-XXXX-XX" + location="update-info/pkg-ver-rel.xml"/> tag which points to it's + corresponding update information. + +2006-01-13 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: apply two patches from + dgregor@redhat.com - verifies that the checksum cache file is more + recent than the corresponding rpm - move around cmds dict + initialization to make it more consistent. + +2005-12-08 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Fix cachedir/groupfile handling with --basedir + and using paths not relative to cwd when run without --basedir. + +2005-12-08 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Support --split option to label media with urls + across directories. + +2005-12-08 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Split out processing into smaller methods. + Make ts internal. Files and base/file/other data attributes. + +2005-12-08 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Cleanup of generator class to use cmds + internally as an attribute. + +2005-12-08 Paul Nasrat <pnasrat@redhat.com> + + * genpkgmetadata.py: Initial work to form metadata generator class. + + +2005-11-27 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: speed up by caching file mode lookup by Dennis + Gregorovic + +2005-11-11 Paul Nasrat <pnasrat@redhat.com> + + * dumpMetadata.py, genpkgmetadata.py: Enable basedir to be used - + dgregor + +2005-11-02 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: patch from Christoph Thiel to + make it work on suse 9.3 and to allow for non absolute-path cache + dirs. + +2005-08-11 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: turn off all signature checking when reading in + headers + +2005-07-24 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: document that the -g option is for a file + relative to the directory you are creating the repository for. + +2005-07-14 Seth Vidal <skvidal@linux.duke.edu> + + * docs/createrepo.8: man page for cachedir + +2005-07-14 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: 0.4.3 + +2005-07-11 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: - disable the checksum flag - default and only use sha1sum's - add + in -c,--cachedir option to setup a cachedir for the cache files of + the checksums of the packages. Uses name-hdrid from the package hdr + as filenames. Contents of the file is a single line of the package's + checksum. This dramatically speeds up rebuilding a repository's + metadata b/c the checksum of the package file was the item taking + the most time. + +2005-05-29 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: Apply Arun Bhanu's patch to + add in --quiet and --verbose options instead of just -q and -v + +2005-03-30 Seth Vidal <skvidal@linux.duke.edu> + + * docs/Makefile: fix mandir path for docs + +2005-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, docs/Makefile: fix the Makefiles, f'real + +2005-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * docs/Makefile, docs/createrepo.8: real commit + +2005-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec: adding man page and upating the + Makefiles and specfile accordingly. Thanks Bob Kashani for the man + page. + +2005-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: need to seek to the beginning before doing a new + read operation. + +2005-01-17 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec: spec and Makefile to 0.4.2 + +2005-01-17 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: move around gzipOpen for use + in another program relabel 0.4.2 + +2005-01-07 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: single open for all file + operations. about a 30% time savings. + +2004-11-02 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: small fix for --exclude to work. -x works, but + --exclude didn't, now it is fixed + +2004-10-21 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: update version + numbers + +2004-10-21 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: problem with ghost entries not showing up in + primary.xml even if they matched the regex strings. + +2004-10-18 Seth Vidal <skvidal@linux.duke.edu> + + * bin/createrepo: whoops! need to quote that var string + +2004-10-11 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: correct problem with + handling dirs with a space in the filename update version number + +2004-10-04 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: clean up argument parsing to handle --version + and --help more correctly. Not quite the patch Ville Skyttä + submitted. + +2004-09-30 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: one more place to tag + +2004-09-30 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec: update to 0.3.9 + +2004-09-30 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: checksum of group file will be wrong if specified + - didn't seek(0) after copying it. + +2004-09-20 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: made 'cannot remove old metadata dir' a + non-fatal error. It just warns. + +2004-09-20 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: updated to default to sha-1 checksums + +2004-09-11 Seth Vidal <skvidal@linux.duke.edu> + + * createrepo.spec, genpkgmetadata.py: update spec file as 0.3.8 fix + for bug in command handling of groups location + +2004-09-11 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile: fix for group file path being wrong - Bill Nottingham + mark as 0.3.8 + +2004-09-11 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: fix for error when string is + None for utf8 conversion + +2004-09-03 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile: Makefile update to fix a bug reported by Anvil + +2004-09-01 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: 0.3.7 + +2004-08-27 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: patch from Ville Skytta (this + a will be wrong, sorry) to correct decoding/encoding problems. + +2004-07-23 Seth Vidal <skvidal@linux.duke.edu> + + * README: updated readme with anoncvs location + +2004-07-23 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: ver to 0.3.6 + +2004-07-23 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: fix filelists to be complete + +2004-07-23 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: remove a debug print call + +2004-07-23 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec: mark as 0.3.5 + +2004-07-23 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: fix up for broken filelists in packages + +2004-07-23 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: silly string fix + +2004-07-20 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: bump number to 0.3.4 + + +2004-07-20 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: re-enabled group files + documented it + +2004-06-30 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: add pre=1 to requires entries for prereq marking + + +2004-06-30 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: - include xmlns:rpm in main metadata tag rather than per-format node + - fix output for sorta-list, sorta-string rpm header tags + +2004-06-28 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: fix for namespace for license, vendor, group, + buildhost and sourcerpm was None, should have been formatns (rpm + namespace) + +2004-06-09 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: mark as 0.3.3 + +2004-06-06 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: included a not-that-terribly accurate package + count + +2004-06-05 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: generate uncompressed + checksums a much easier way. + +2004-06-05 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: revert some changes + +2004-06-03 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: fix stupid version thing + +2004-06-03 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: keep checksum of uncompressed + metadata files in repomd.xml + +2004-06-03 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: move versioned prco from + separate string to properties of the entry + +2004-04-16 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile: fix makefile + +2004-04-16 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, genpkgmetadata.py: update to 0.3.2 + added -p or --pretty flag to make it pretty-print the xml output not + pretty printing the output makes the import a lot faster and the + data smaller + +2004-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec, dumpMetadata.py, genpkgmetadata.py: 1. make it actually work :) 2. bump to 0.3.1 + +2004-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, createrepo.spec: add README for real *boggle* + +2004-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, README, createrepo.spec: tagged Makefile and createrepo + as 0.3 Add README to both of the above + +2004-01-18 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: make metadata files be written to repodata/ + +2004-01-17 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: fix bug where not all files + were getting included make the directory detection more reliable + +2004-01-14 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, bin/Makefile, createrepo.spec, dumpMetadata.py, + genpkgmetadata.py: fixed memory use problem updated spec for 0.2 + fixed makefile dumbness fixed problems with broken symlinks + +2004-01-13 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: catch some errors on broken symlinks + +2004-01-11 Seth Vidal <skvidal@linux.duke.edu> + + * Makefile, bin/Makefile, bin/createrepo, createrepo.spec, + dumpMetadata.py, genpkgmetadata.py: - translation stubs - makefiles - spec file - bin wrapper + +2004-01-10 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: silly updates in comments + +2004-01-10 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: CVS Id Tags + +2004-01-10 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py: [no log message] + +2004-01-10 Seth Vidal <skvidal@linux.duke.edu> + + * genpkgmetadata.py: added --version and __version__ string + +2004-01-10 Seth Vidal <skvidal@linux.duke.edu> + + * dumpMetadata.py, genpkgmetadata.py: move two functions around to + more logically arrange the repomd.xml generating function + +2004-01-09 Seth Vidal <skvidal@linux.duke.edu> + + * Initial revision + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..60bb9db --- /dev/null +++ b/Makefile @@ -0,0 +1,187 @@ +PKGNAME = createrepo +VERSION=$(shell awk '/Version:/ { print $$2 }' ${PKGNAME}.spec) +RELEASE=$(shell awk '/Release:/ { print $$2 }' ${PKGNAME}.spec) +CVSTAG=createrepo-$(subst .,_,$(VERSION)-$(RELEASE)) +PYTHON=python +SUBDIRS = $(PKGNAME) bin docs +PYFILES = $(wildcard *.py) + + +SHELL = /bin/sh +top_srcdir = . +srcdir = . +prefix = /usr +exec_prefix = ${prefix} + +bindir = ${exec_prefix}/bin +sbindir = ${exec_prefix}/sbin +libexecdir = ${exec_prefix}/libexec +datadir = ${prefix}/share +sysconfdir = ${prefix}/etc +sharedstatedir = ${prefix}/com +localstatedir = ${prefix}/var +libdir = ${exec_prefix}/lib +infodir = ${prefix}/info +docdir = +includedir = ${prefix}/include +oldincludedir = /usr/include +mandir = ${prefix}/share/man + +pkgdatadir = $(datadir)/$(PKGNAME) +pkglibdir = $(libdir)/$(PKGNAME) +pkgincludedir = $(includedir)/$(PKGNAME) +top_builddir = + +# all dirs +DIRS = $(DESTDIR)$(bindir) $(DESTDIR)$(sysconfdir)/bash_completion.d \ + $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(mandir) + + +# INSTALL scripts +INSTALL = install -p --verbose +INSTALL_BIN = $(INSTALL) -m 755 +INSTALL_DIR = $(INSTALL) -m 755 -d +INSTALL_DATA = $(INSTALL) -m 644 +INSTALL_MODULES = $(INSTALL) -m 755 -D +RM = rm -f + +MODULES = $(srcdir)/genpkgmetadata.py \ + $(srcdir)/modifyrepo.py \ + $(srcdir)/mergerepo.py \ + $(srcdir)/worker.py + +.SUFFIXES: .py .pyc +.py.pyc: + python -c "import py_compile; py_compile.compile($*.py)" + + +all: $(MODULES) + for subdir in $(SUBDIRS) ; do \ + $(MAKE) -C $$subdir VERSION=$(VERSION) PKGNAME=$(PKGNAME) DESTDIR=$(DESTDIR); \ + done + +check: + pychecker $(MODULES) || exit 0 + +install: all installdirs + $(INSTALL_MODULES) $(srcdir)/$(MODULES) $(DESTDIR)$(pkgdatadir) + $(INSTALL_DATA) $(PKGNAME).bash $(DESTDIR)$(sysconfdir)/bash_completion.d + for subdir in $(SUBDIRS) ; do \ + $(MAKE) -C $$subdir install VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ + done + +installdirs: + for dir in $(DIRS) ; do \ + $(INSTALL_DIR) $$dir ; \ + done + + +uninstall: + for module in $(MODULES) ; do \ + $(RM) $(pkgdatadir)/$$module ; \ + done + for subdir in $(SUBDIRS) ; do \ + $(MAKE) -C $$subdir uninstall VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ + done + +clean: + $(RM) *.pyc *.pyo + for subdir in $(SUBDIRS) ; do \ + $(MAKE) -C $$subdir clean VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ + done + +distclean: clean + $(RM) -r .libs + $(RM) core + $(RM) *~ + for subdir in $(SUBDIRS) ; do \ + $(MAKE) -C $$subdir distclean VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ + done + +pylint: + @pylint --rcfile=test/createrepo-pylintrc *.py createrepo + +pylint-short: + @pylint -r n --rcfile=test/createrepo-pylintrc *.py createrepo + +mostlyclean: + $(MAKE) clean + + +maintainer-clean: + $(MAKE) distclean + $(RM) $(srcdir)/configure + +changelog: + git log --pretty --numstat --summary | git2cl > ChangeLog + +dist: + olddir=`pwd`; \ + distdir=$(PKGNAME)-$(VERSION); \ + $(RM) -r .disttmp; \ + $(INSTALL_DIR) .disttmp; \ + $(INSTALL_DIR) .disttmp/$$distdir; \ + $(MAKE) distfiles + distdir=$(PKGNAME)-$(VERSION); \ + cd .disttmp; \ + tar -cvz > ../$$distdir.tar.gz $$distdir; \ + cd $$olddir + $(RM) -r .disttmp + +daily: + olddir=`pwd`; \ + distdir=$(PKGNAME); \ + $(RM) -r .disttmp; \ + $(INSTALL_DIR) .disttmp; \ + $(INSTALL_DIR) .disttmp/$$distdir; \ + $(MAKE) dailyfiles + day=`/bin/date +%Y%m%d`; \ + distdir=$(PKGNAME); \ + tarname=$$distdir-$$day ;\ + cd .disttmp; \ + perl -pi -e "s/\#DATE\#/$$day/g" $$distdir/$(PKGNAME)-daily.spec; \ + echo $$day; \ + tar -cvz > ../$$tarname.tar.gz $$distdir; \ + cd $$olddir + $(RM) -rf .disttmp + +dailyfiles: + distdir=$(PKGNAME); \ + cp \ + $(srcdir)/*.py \ + $(srcdir)/Makefile \ + $(srcdir)/ChangeLog \ + $(srcdir)/COPYING \ + $(srcdir)/COPYING.lib \ + $(srcdir)/README \ + $(srcdir)/$(PKGNAME).spec \ + $(srcdir)/$(PKGNAME).bash \ + $(top_srcdir)/.disttmp/$$distdir + for subdir in $(SUBDIRS) ; do \ + $(MAKE) -C $$subdir dailyfiles VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ + done + +distfiles: + distdir=$(PKGNAME)-$(VERSION); \ + cp \ + $(srcdir)/*.py \ + $(srcdir)/Makefile \ + $(srcdir)/ChangeLog \ + $(srcdir)/COPYING \ + $(srcdir)/COPYING.lib \ + $(srcdir)/README \ + $(srcdir)/$(PKGNAME).spec \ + $(srcdir)/$(PKGNAME).bash \ + $(top_srcdir)/.disttmp/$$distdir + for subdir in $(SUBDIRS) ; do \ + $(MAKE) -C $$subdir distfiles VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ + done + +archive: dist + +.PHONY: todo +todo: + @echo ---------------=========================================== + @grep -n TODO\\\|FIXME `find . -type f` | grep -v grep + @echo ---------------=========================================== +.PHONY: all install install-strip uninstall clean distclean mostlyclean maintainer-clean info dvi dist distfiles check installcheck installdirs daily dailyfiles @@ -0,0 +1,14 @@ +This program generates a repodata dir and xml files for a repository of +rpm packages. This repository is compatible with apt/yum/smart/yast and many +other package-repository-related tools. + +run createrepo -h for usage syntax + +http://createrepo.baseurl.org/ + + + + + + + diff --git a/bin/Makefile b/bin/Makefile new file mode 100644 index 0000000..8e0a7e2 --- /dev/null +++ b/bin/Makefile @@ -0,0 +1,98 @@ +SHELL = /bin/sh +top_srcdir = .. +srcdir = ../bin +prefix = /usr +exec_prefix = ${prefix} + +bindir = ${exec_prefix}/bin +sbindir = ${exec_prefix}/sbin +libexecdir = ${exec_prefix}/libexec +datadir = ${prefix}/share +sysconfdir = ${prefix}/etc +sharedstatedir = ${prefix}/com +localstatedir = ${prefix}/var +libdir = ${exec_prefix}/lib +infodir = ${prefix}/info +docdir = +includedir = ${prefix}/include +oldincludedir = /usr/include +mandir = ${prefix}/man + +pkgdatadir = $(datadir)/$(PKGNAME) +pkglibdir = $(libdir)/$(PKGNAME) +pkgincludedir = $(includedir)/$(PKGNAME) +top_builddir = ../ + +# all dirs +DIRS = $(DESTDIR)$(bindir) $(DESTDIR)/etc $(DESTDIR)$(pkgdatadir) + + +# INSTALL scripts +INSTALL = install -p --verbose +INSTALL_BIN = $(INSTALL) -m 755 +INSTALL_DIR = $(INSTALL) -m 755 -d +INSTALL_DATA = $(INSTALL) -m 644 +INSTALL_MODULES = $(INSTALL) -m 755 -D +RM = rm -f + + +all: + echo "Nothing to do" + + +install: all installdirs + $(INSTALL_BIN) $(srcdir)/$(PKGNAME) $(DESTDIR)$(bindir)/$(PKGNAME) + $(INSTALL_BIN) $(srcdir)/modifyrepo $(DESTDIR)$(bindir)/modifyrepo + $(INSTALL_BIN) $(srcdir)/mergerepo $(DESTDIR)$(bindir)/mergerepo + + +uninstall: + $(RM) $(bindir)/$(PKGNAME) + + + +clean: + + +distclean: + $(RM) -rf .libs + $(RM) -f core + $(RM) -f *~ + + +mostlyclean: + $(MAKE) clean + + +maintainer-clean: + $(MAKE) distclean + + +distfiles: + distdir=$(PKGNAME)-$(VERSION); \ + mkdir $(top_srcdir)/.disttmp/$$distdir/bin;\ + cp \ + $(srcdir)/$(PKGNAME) \ + $(srcdir)/Makefile \ + $(srcdir)/modifyrepo \ + $(srcdir)/mergerepo \ + $(top_srcdir)/.disttmp/$$distdir/bin + +dailyfiles: + distdir=$(PKGNAME); \ + mkdir $(top_srcdir)/.disttmp/$$distdir/bin;\ + cp \ + $(srcdir)/$(PKGNAME) \ + $(srcdir)/Makefile \ + $(srcdir)/modifyrepo \ + $(srcdir)/mergerepo \ + $(top_srcdir)/.disttmp/$$distdir/bin + +installdirs: + $(MAKE) -C $(top_srcdir) installdirs + + +.PHONY: all install install-strip uninstall clean distclean mostlyclean maintainer-clean info dvi dist distfiles check installcheck installdirs dailyfiles + + + diff --git a/bin/createrepo b/bin/createrepo new file mode 100755 index 0000000..b0de515 --- /dev/null +++ b/bin/createrepo @@ -0,0 +1,2 @@ +#!/bin/sh +exec /usr/share/createrepo/genpkgmetadata.py "$@" diff --git a/bin/mergerepo b/bin/mergerepo new file mode 100755 index 0000000..df6e2f1 --- /dev/null +++ b/bin/mergerepo @@ -0,0 +1,2 @@ +#!/bin/sh +exec /usr/share/createrepo/mergerepo.py "$@" diff --git a/bin/modifyrepo b/bin/modifyrepo new file mode 100755 index 0000000..c9732d8 --- /dev/null +++ b/bin/modifyrepo @@ -0,0 +1,2 @@ +#!/bin/sh +exec /usr/share/createrepo/modifyrepo.py "$@" diff --git a/createrepo.bash b/createrepo.bash new file mode 100644 index 0000000..54ac8b2 --- /dev/null +++ b/createrepo.bash @@ -0,0 +1,115 @@ +# bash completion for createrepo and friends + +_cr_createrepo() +{ + COMPREPLY=() + + case $3 in + --version|-h|--help|-u|--baseurl|--distro|--content|--repo|--workers|\ + --revision|-x|--excludes|--changelog-limit|--max-delta-rpm-size) + return 0 + ;; + --basedir|-c|--cachedir|--update-md-path|-o|--outputdir|\ + --oldpackagedirs) + COMPREPLY=( $( compgen -d -- "$2" ) ) + return 0 + ;; + -g|--groupfile) + COMPREPLY=( $( compgen -f -o plusdirs -X '!*.xml' -- "$2" ) ) + return 0 + ;; + -s|--sumtype) + COMPREPLY=( $( compgen -W 'md5 sha1 sha256 sha512' -- "$2" ) ) + return 0 + ;; + -i|--pkglist|--read-pkgs-list) + COMPREPLY=( $( compgen -f -o plusdirs -- "$2" ) ) + return 0 + ;; + -n|--includepkg) + COMPREPLY=( $( compgen -f -o plusdirs -X '!*.rpm' -- "$2" ) ) + return 0 + ;; + --num-deltas) + COMPREPLY=( $( compgen -W '1 2 3 4 5 6 7 8 9' -- "$2" ) ) + return 0 + ;; + esac + + if [[ $2 == -* ]] ; then + COMPREPLY=( $( compgen -W '--version --help --quiet --verbose --profile + --excludes --basedir --baseurl --groupfile --checksum --pretty + --cachedir --checkts --no-database --update --update-md-path + --skip-stat --split --pkglist --includepkg --outputdir + --skip-symlinks --changelog-limit --unique-md-filenames + --simple-md-filenames --distro --content --repo --revision --deltas + --oldpackagedirs --num-deltas --read-pkgs-list + --max-delta-rpm-size --workers' -- "$2" ) ) + else + COMPREPLY=( $( compgen -d -- "$2" ) ) + fi +} && +complete -F _cr_createrepo -o filenames createrepo genpkgmetadata.py + +_cr_mergerepo() +{ + COMPREPLY=() + + case $3 in + --version|-h|--help|-a|--archlist) + return 0 + ;; + -r|--repo|-o|--outputdir) + COMPREPLY=( $( compgen -d -- "$2" ) ) + return 0 + ;; + esac + + COMPREPLY=( $( compgen -W '--version --help --repo --archlist --no-database + --outputdir --nogroups --noupdateinfo' -- "$2" ) ) +} && +complete -F _cr_mergerepo -o filenames mergerepo mergerepo.py + +_cr_modifyrepo() +{ + COMPREPLY=() + + case $3 in + --version|-h|--help|--mdtype) + return 0 + ;; + esac + + if [[ $2 == -* ]] ; then + COMPREPLY=( $( compgen -W '--version --help --mdtype' -- "$2" ) ) + return 0 + fi + + local i argnum=1 + for (( i=1; i < ${#COMP_WORDS[@]}-1; i++ )) ; do + if [[ ${COMP_WORDS[i]} != -* && + ${COMP_WORDS[i-1]} != @(=|--mdtype) ]]; then + argnum=$(( argnum+1 )) + fi + done + + case $argnum in + 1) + COMPREPLY=( $( compgen -f -o plusdirs -- "$2" ) ) + return 0 + ;; + 2) + COMPREPLY=( $( compgen -d -- "$2" ) ) + return 0 + ;; + esac +} && +complete -F _cr_modifyrepo -o filenames modifyrepo modifyrepo.py + +# Local variables: +# mode: shell-script +# sh-basic-offset: 4 +# sh-indent-comment: t +# indent-tabs-mode: nil +# End: +# ex: ts=4 sw=4 et filetype=sh diff --git a/createrepo.spec b/createrepo.spec new file mode 100644 index 0000000..1e491cd --- /dev/null +++ b/createrepo.spec @@ -0,0 +1,80 @@ +%{!?python_sitelib: %define python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} + +Summary: Creates a common metadata repository +Name: createrepo +Version: 0.9.9 +Release: 1 +License: GPL +Group: System Environment/Base +Source: %{name}-%{version}.tar.gz +URL: http://createrepo.baseurl.org/ +BuildRoot: %{_tmppath}/%{name}-%{version}root +BuildArchitectures: noarch +Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python +Requires: yum-metadata-parser, yum >= 3.2.29, python-deltarpm + +%description +This utility will generate a common metadata repository from a directory of +rpm packages + +%prep +%setup -q + +%install +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf $RPM_BUILD_ROOT +make DESTDIR=$RPM_BUILD_ROOT sysconfdir=%{_sysconfdir} install + +%clean +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf $RPM_BUILD_ROOT + + +%files +%defattr(-, root, root) +%dir %{_datadir}/%{name} +%doc ChangeLog README COPYING COPYING.lib +%{_sysconfdir}/bash_completion.d/ +%{_datadir}/%{name}/* +%{_bindir}/%{name} +%{_bindir}/modifyrepo +%{_bindir}/mergerepo +%{_mandir}/man8/createrepo.8* +%{_mandir}/man1/modifyrepo.1* +%{_mandir}/man1/mergerepo.1* +%{python_sitelib}/createrepo + +%changelog +* Wed Jan 26 2011 Seth Vidal <skvidal at fedoraproject.org> +- bump to 0.9.9 +- add worker.py + +* Thu Aug 19 2010 Seth Vidal <skvidal at fedoraproject.org> +- increase yum requirement for the modifyrepo use of RepoMD, RepoData and RepoMDError + +* Fri Aug 28 2009 Seth Vidal <skvidal at fedoraproject.org> +- 0.9.8 + +* Tue Mar 24 2009 Seth Vidal <skvidal at fedoraproject.org> +- 0.9.7 + +* Fri Oct 17 2008 Seth Vidal <skvidal at fedoraproject.org> +- add mergerepo - 0.9.6 + +* Mon Feb 18 2008 Seth Vidal <skvidal at fedoraproject.org> +- 0.9.5 + +* Mon Jan 28 2008 Seth Vidal <skvidal at fedoraproject.org> +- 0.9.4 + +* Tue Jan 22 2008 Seth Vidal <skvidal at fedoraproject.org> +- 0.9.3 + +* Thu Jan 17 2008 Seth Vidal <skvidal at fedoraproject.org> +- significant api changes + +* Tue Jan 8 2008 Seth Vidal <skvidal at fedoraproject.org> +- 0.9.1 - lots of fixes +- cleanup changelog, too + +* Thu Dec 20 2007 Seth Vidal <skvidal at fedoraproject.org> +- beginning of the new version + diff --git a/createrepo/Makefile b/createrepo/Makefile new file mode 100644 index 0000000..d3d3a34 --- /dev/null +++ b/createrepo/Makefile @@ -0,0 +1,64 @@ +PYTHON=python +PACKAGE = $(shell basename `pwd`) +PYFILES = $(wildcard *.py) +PYVER := $(shell $(PYTHON) -c 'import sys; print "%.3s" %(sys.version)') +PYSYSDIR := $(shell $(PYTHON) -c 'import sys; print sys.prefix') +PYLIBDIR = $(PYSYSDIR)/lib/python$(PYVER) +PKGDIR = $(PYLIBDIR)/site-packages/$(PKGNAME) + +SHELL = /bin/sh +top_srcdir = .. +srcdir = ../$(PKGNAME) +prefix = /usr +exec_prefix = ${prefix} + +bindir = ${exec_prefix}/bin +sbindir = ${exec_prefix}/sbin +libexecdir = ${exec_prefix}/libexec +datadir = ${prefix}/share +sysconfdir = ${prefix}/etc +sharedstatedir = ${prefix}/com +localstatedir = ${prefix}/var +libdir = ${exec_prefix}/lib +infodir = ${prefix}/info +docdir = +includedir = ${prefix}/include +oldincludedir = /usr/include +mandir = ${datadir}/man + +pkgdatadir = $(datadir)/$(PKGNAME) +pkglibdir = $(libdir)/$(PKGNAME) +pkgincludedir = $(includedir)/$(PKGNAME) +top_builddir = ../ + + +all: + echo "Nothing to do" + +clean: + rm -f *.pyc *.pyo *~ + +install: + mkdir -p $(DESTDIR)/$(PKGDIR) + for p in $(PYFILES) ; do \ + install -m 644 $$p $(DESTDIR)/$(PKGDIR)/$$p; \ + done + $(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(PKGDIR)', 1, '$(PKGDIR)', 1)" + +distfiles: + distdir=$(PKGNAME)-$(VERSION); \ + mkdir $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME);\ + cp \ + $(srcdir)/$(PYFILES) \ + $(srcdir)/Makefile \ + $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME) + +dailyfiles: + distdir=$(PKGNAME); \ + mkdir $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME);\ + cp \ + $(srcdir)/$(PYFILES) \ + $(srcdir)/__init__.py \ + $(srcdir)/Makefile \ + $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME) + diff --git a/createrepo/__init__.py b/createrepo/__init__.py new file mode 100644 index 0000000..8f2538e --- /dev/null +++ b/createrepo/__init__.py @@ -0,0 +1,1344 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2009 Red Hat, Inc - +# written by seth vidal skvidal at fedoraproject.org + +import os +import sys +import fnmatch +import time +import yumbased +import shutil +from bz2 import BZ2File +from urlgrabber import grabber +import tempfile +import stat +import fcntl +import subprocess + +from yum import misc, Errors, to_unicode +from yum.repoMDObject import RepoMD, RepoMDError, RepoData +from yum.sqlutils import executeSQL +from yum.packageSack import MetaSack +from yum.packages import YumAvailablePackage, YumLocalPackage + +import rpmUtils.transaction +from utils import _, errorprint, MDError +import readMetadata +try: + import sqlite3 as sqlite +except ImportError: + import sqlite + +try: + import sqlitecachec +except ImportError: + pass + +from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \ + checksum_and_rename, split_list_into_equal_chunks +import deltarpms + +__version__ = '0.9.9' + + +class MetaDataConfig(object): + def __init__(self): + self.quiet = False + self.verbose = False + self.profile = False + self.excludes = [] + self.baseurl = None + self.groupfile = None + self.sumtype = 'sha256' + self.pretty = False + self.cachedir = None + self.use_cache = False + self.basedir = os.getcwd() + self.checkts = False + self.split = False + self.update = False + self.deltas = False # do the deltarpm thing + # where to put the .drpms - defaults to 'drpms' inside 'repodata' + self.deltadir = None + self.delta_relative = 'drpms/' + self.oldpackage_paths = [] # where to look for the old packages - + self.deltafile = 'prestodelta.xml.gz' + self.num_deltas = 1 # number of older versions to delta (max) + self.max_delta_rpm_size = 100000000 + self.update_md_path = None + self.skip_stat = False + self.database = True + self.outputdir = None + self.file_patterns = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] + self.dir_patterns = ['.*bin\/.*', '^\/etc\/.*'] + self.skip_symlinks = False + self.pkglist = [] + self.database_only = False + self.primaryfile = 'primary.xml.gz' + self.filelistsfile = 'filelists.xml.gz' + self.otherfile = 'other.xml.gz' + self.repomdfile = 'repomd.xml' + self.tempdir = '.repodata' + self.finaldir = 'repodata' + self.olddir = '.olddata' + self.mdtimestamp = 0 + self.directory = None + self.directories = [] + self.changelog_limit = None # needs to be an int or None + self.unique_md_filenames = True + self.additional_metadata = {} # dict of 'type':'filename' + self.revision = str(int(time.time())) + self.content_tags = [] # flat list of strings (like web 2.0 tags) + self.distro_tags = []# [(cpeid(None allowed), human-readable-string)] + self.repo_tags = []# strings, forwhatever they are worth + self.read_pkgs_list = None # filepath/name to write out list of pkgs + # read in this run of createrepo + self.collapse_glibc_requires = True + self.workers = 1 # number of workers to fork off to grab metadata from the pkgs + self.worker_cmd = '/usr/share/createrepo/worker.py' + + #self.worker_cmd = './worker.py' # helpful when testing + +class SimpleMDCallBack(object): + def errorlog(self, thing): + print >> sys.stderr, thing + + def log(self, thing): + print thing + + def progress(self, item, current, total): + sys.stdout.write('\r' + ' ' * 80) + sys.stdout.write("\r%d/%d - %s" % (current, total, item)) + sys.stdout.flush() + + +class MetaDataGenerator: + def __init__(self, config_obj=None, callback=None): + self.conf = config_obj + if config_obj == None: + self.conf = MetaDataConfig() + if not callback: + self.callback = SimpleMDCallBack() + else: + self.callback = callback + + + self.ts = rpmUtils.transaction.initReadOnlyTransaction() + self.pkgcount = 0 + self.current_pkg = 0 + self.files = [] + self.rpmlib_reqs = {} + self.read_pkgs = [] + + if not self.conf.directory and not self.conf.directories: + raise MDError, "No directory given on which to run." + + if not self.conf.directories: # just makes things easier later + self.conf.directories = [self.conf.directory] + if not self.conf.directory: # ensure we have both in the config object + self.conf.directory = self.conf.directories[0] + + # the cachedir thing: + if self.conf.cachedir: + self.conf.use_cache = True + + # this does the dir setup we need done + self._parse_directory() + self._test_setup_dirs() + + def _parse_directory(self): + """pick up the first directory given to us and make sure we know + where things should go""" + if os.path.isabs(self.conf.directory): + self.conf.basedir = os.path.dirname(self.conf.directory) + self.conf.relative_dir = os.path.basename(self.conf.directory) + else: + self.conf.basedir = os.path.realpath(self.conf.basedir) + self.conf.relative_dir = self.conf.directory + + self.package_dir = os.path.join(self.conf.basedir, + self.conf.relative_dir) + + if not self.conf.outputdir: + self.conf.outputdir = os.path.join(self.conf.basedir, + self.conf.relative_dir) + + def _test_setup_dirs(self): + # start the sanity/stupidity checks + for mydir in self.conf.directories: + if os.path.isabs(mydir): + testdir = mydir + else: + if mydir.startswith('../'): + testdir = os.path.realpath(mydir) + else: + testdir = os.path.join(self.conf.basedir, mydir) + + if not os.path.exists(testdir): + raise MDError, _('Directory %s must exist') % mydir + + if not os.path.isdir(testdir): + raise MDError, _('%s must be a directory') % mydir + + if not os.access(self.conf.outputdir, os.W_OK): + raise MDError, _('Directory %s must be writable.') % self.conf.outputdir + + temp_output = os.path.join(self.conf.outputdir, self.conf.tempdir) + if not checkAndMakeDir(temp_output): + raise MDError, _('Cannot create/verify %s') % temp_output + + temp_final = os.path.join(self.conf.outputdir, self.conf.finaldir) + if not checkAndMakeDir(temp_final): + raise MDError, _('Cannot create/verify %s') % temp_final + + if self.conf.database: + # do flock test on temp_final, temp_output + # if it fails raise MDError + for direc in [temp_final, temp_output]: + f = open(direc + '/locktest', 'w') + try: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + except (OSError, IOError), e: + raise MDError, _("Could not create exclusive lock in %s and sqlite database generation enabled. Is this path on nfs? Is your lockd running?") % direc + else: + os.unlink(direc + '/locktest') + + if self.conf.deltas: + temp_delta = os.path.join(self.conf.outputdir, + self.conf.delta_relative) + if not checkAndMakeDir(temp_delta): + raise MDError, _('Cannot create/verify %s') % temp_delta + self.conf.deltadir = temp_delta + + if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)): + raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir + + # make sure we can write to where we want to write to: + # and pickup the mdtimestamps while we're at it + direcs = ['tempdir' , 'finaldir'] + if self.conf.deltas: + direcs.append('deltadir') + + for direc in direcs: + filepath = os.path.join(self.conf.outputdir, getattr(self.conf, + direc)) + if os.path.exists(filepath): + if not os.access(filepath, os.W_OK): + raise MDError, _('error in must be able to write to metadata dir:\n -> %s') % filepath + + if self.conf.checkts: + # checking for repodata/repomd.xml - not just the data dir + rxml = filepath + '/repomd.xml' + if os.path.exists(rxml): + timestamp = os.path.getctime(rxml) + if timestamp > self.conf.mdtimestamp: + self.conf.mdtimestamp = timestamp + + if self.conf.groupfile: + a = self.conf.groupfile + if self.conf.split: + a = os.path.join(self.package_dir, self.conf.groupfile) + elif not os.path.isabs(a): + a = os.path.join(self.package_dir, self.conf.groupfile) + + if not os.path.exists(a): + raise MDError, _('Error: groupfile %s cannot be found.' % a) + + self.conf.groupfile = a + + if self.conf.cachedir: + a = self.conf.cachedir + if not os.path.isabs(a): + a = os.path.join(self.conf.outputdir, a) + if not checkAndMakeDir(a): + raise MDError, _('Error: cannot open/write to cache dir %s' % a) + + self.conf.cachedir = a + + + def _os_path_walk(self, top, func, arg): + """Directory tree walk with callback function. + copy of os.path.walk, fixes the link/stating problem + """ + + try: + names = os.listdir(top) + except os.error: + return + func(arg, top, names) + for name in names: + name = os.path.join(top, name) + if os.path.isdir(name): + self._os_path_walk(name, func, arg) + def getFileList(self, directory, ext): + """Return all files in path matching ext, store them in filelist, + recurse dirs. Returns a list object""" + + extlen = len(ext) + + def extension_visitor(filelist, dirname, names): + for fn in names: + if os.path.isdir(fn): + continue + if self.conf.skip_symlinks and os.path.islink(fn): + continue + elif fn[-extlen:].lower() == '%s' % (ext): + relativepath = dirname.replace(startdir, "", 1) + relativepath = relativepath.lstrip("/") + filelist.append(os.path.join(relativepath, fn)) + + filelist = [] + startdir = directory + '/' + self._os_path_walk(startdir, extension_visitor, filelist) + return filelist + + def errorlog(self, thing): + """subclass this if you want something different....""" + errorprint(thing) + + def checkTimeStamps(self): + """check the timestamp of our target dir. If it is not newer than + the repodata return False, else True""" + if self.conf.checkts: + dn = os.path.join(self.conf.basedir, self.conf.directory) + files = self.getFileList(dn, '.rpm') + files = self.trimRpms(files) + for f in files: + fn = os.path.join(self.conf.basedir, self.conf.directory, f) + if not os.path.exists(fn): + self.callback.errorlog(_('cannot get to file: %s') % fn) + if os.path.getctime(fn) > self.conf.mdtimestamp: + return False + + return True + + return False + + def trimRpms(self, files): + badrpms = [] + for rpm_file in files: + for glob in self.conf.excludes: + if fnmatch.fnmatch(rpm_file, glob): + if rpm_file not in badrpms: + badrpms.append(rpm_file) + for rpm_file in badrpms: + if rpm_file in files: + files.remove(rpm_file) + return files + + def _setup_old_metadata_lookup(self): + """sets up the .oldData object for handling the --update call. Speeds + up generating updates for new metadata""" + #FIXME - this only actually works for single dirs. It will only + # function for the first dir passed to --split, not all of them + # this needs to be fixed by some magic in readMetadata.py + # using opts.pkgdirs as a list, I think. + if self.conf.update: + #build the paths + opts = { + 'verbose' : self.conf.verbose, + 'pkgdir' : os.path.normpath(self.package_dir) + } + + if self.conf.skip_stat: + opts['do_stat'] = False + + if self.conf.update_md_path: + norm_u_md_path = os.path.normpath(self.conf.update_md_path) + u_md_repodata_path = norm_u_md_path + '/repodata' + if not os.path.exists(u_md_repodata_path): + msg = _('Warning: could not open update_md_path: %s') % u_md_repodata_path + self.callback.errorlog(msg) + old_repo_path = os.path.normpath(norm_u_md_path) + else: + old_repo_path = self.conf.outputdir + + #and scan the old repo + self.oldData = readMetadata.MetadataIndex(old_repo_path, opts) + + def _setup_grabber(self): + if not hasattr(self, '_grabber'): + self._grabber = grabber.URLGrabber() + + return self._grabber + + grabber = property(fget = lambda self: self._setup_grabber()) + + + def doPkgMetadata(self): + """all the heavy lifting for the package metadata""" + if self.conf.update: + self._setup_old_metadata_lookup() + # rpms we're going to be dealing with + if self.conf.pkglist: + packages = self.conf.pkglist + else: + packages = self.getFileList(self.package_dir, '.rpm') + + if not isinstance(packages, MetaSack): + packages = self.trimRpms(packages) + self.pkgcount = len(packages) + try: + self.openMetadataDocs() + self.writeMetadataDocs(packages) + self.closeMetadataDocs() + except (IOError, OSError), e: + raise MDError, _('Cannot access/write repodata files: %s') % e + + + def openMetadataDocs(self): + if self.conf.database_only: + self.setup_sqlite_dbs() + else: + self.primaryfile = self._setupPrimary() + self.flfile = self._setupFilelists() + self.otherfile = self._setupOther() + if self.conf.deltas: + self.deltafile = self._setupDelta() + + def _setupPrimary(self): + # setup the primary metadata file + primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, + self.conf.primaryfile) + fo = _gzipOpen(primaryfilepath, 'w') + fo.write('<?xml version="1.0" encoding="UTF-8"?>\n') + fo.write('<metadata xmlns="http://linux.duke.edu/metadata/common"' \ + ' xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">' % + self.pkgcount) + return fo + + def _setupFilelists(self): + # setup the filelist file + filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, + self.conf.filelistsfile) + fo = _gzipOpen(filelistpath, 'w') + fo.write('<?xml version="1.0" encoding="UTF-8"?>\n') + fo.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists"' \ + ' packages="%s">' % self.pkgcount) + return fo + + def _setupOther(self): + # setup the other file + otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, + self.conf.otherfile) + fo = _gzipOpen(otherfilepath, 'w') + fo.write('<?xml version="1.0" encoding="UTF-8"?>\n') + fo.write('<otherdata xmlns="http://linux.duke.edu/metadata/other"' \ + ' packages="%s">' % + self.pkgcount) + return fo + + def _setupDelta(self): + # setup the other file + deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, + self.conf.deltafile) + fo = _gzipOpen(deltafilepath, 'w') + fo.write('<?xml version="1.0" encoding="UTF-8"?>\n') + fo.write('<prestodelta>\n') + return fo + + + def read_in_package(self, rpmfile, pkgpath=None, reldir=None): + """rpmfile == relative path to file from self.packge_dir""" + baseurl = self.conf.baseurl + + if not pkgpath: + pkgpath = self.package_dir + + if not rpmfile.strip(): + raise MDError, "Blank filename passed in, skipping" + + if rpmfile.find("://") != -1: + + if not hasattr(self, 'tempdir'): + self.tempdir = tempfile.mkdtemp() + + pkgname = os.path.basename(rpmfile) + baseurl = os.path.dirname(rpmfile) + reldir = self.tempdir + dest = os.path.join(self.tempdir, pkgname) + if not self.conf.quiet: + self.callback.log('\nDownloading %s' % rpmfile) + try: + rpmfile = self.grabber.urlgrab(rpmfile, dest) + except grabber.URLGrabError, e: + raise MDError, "Unable to retrieve remote package %s: %s" % ( + rpmfile, e) + + + else: + rpmfile = '%s/%s' % (pkgpath, rpmfile) + + external_data = { '_cachedir': self.conf.cachedir, + '_baseurl': baseurl, + '_reldir': reldir, + '_packagenumber': self.current_pkg, + '_collapse_libc_requires':self.conf.collapse_glibc_requires, + } + + try: + po = yumbased.CreateRepoPackage(self.ts, rpmfile, + sumtype=self.conf.sumtype, + external_data = external_data) + except Errors.MiscError, e: + raise MDError, "Unable to open package: %s" % e + + for r in po.requires_print: + if r.startswith('rpmlib('): + self.rpmlib_reqs[r] = 1 + + if po.checksum in (None, ""): + raise MDError, "No Package ID found for package %s, not going to" \ + " add it" % po + + return po + + def writeMetadataDocs(self, pkglist=[], pkgpath=None): + + if not pkglist: + pkglist = self.conf.pkglist + + if not pkgpath: + directory = self.conf.directory + else: + directory = pkgpath + + # for worker/forked model + # iterate the pkglist - see which ones are handled by --update and let them + # go on their merry way + + newpkgs = [] + if self.conf.update: + # if we're in --update mode then only act on the new/changed pkgs + for pkg in pkglist: + self.current_pkg += 1 + + #see if we can pull the nodes from the old repo + #print self.oldData.basenodes.keys() + old_pkg = pkg + if pkg.find("://") != -1: + old_pkg = os.path.basename(pkg) + nodes = self.oldData.getNodes(old_pkg) + if nodes is not None: # we have a match in the old metadata + if self.conf.verbose: + self.callback.log(_("Using data from old metadata for %s") + % pkg) + (primarynode, filenode, othernode) = nodes + + for node, outfile in ((primarynode, self.primaryfile), + (filenode, self.flfile), + (othernode, self.otherfile)): + if node is None: + break + + if self.conf.baseurl: + anode = node.children + while anode is not None: + if anode.type != "element": + anode = anode.next + continue + if anode.name == "location": + anode.setProp('xml:base', self.conf.baseurl) + anode = anode.next + + output = node.serialize('UTF-8', self.conf.pretty) + if output: + outfile.write(output) + else: + if self.conf.verbose: + self.callback.log(_("empty serialize on write to" \ + "%s in %s") % (outfile, pkg)) + outfile.write('\n') + + self.oldData.freeNodes(pkg) + #FIXME - if we're in update and we have deltas enabled + # check the presto data for this pkg and write its info back out + # to our deltafile + continue + else: + newpkgs.append(pkg) + else: + newpkgs = pkglist + + # setup our reldir + if not pkgpath: + reldir = os.path.join(self.conf.basedir, directory) + else: + reldir = pkgpath + + # filter out those pkgs which are not files - but are pkgobjects + pkgfiles = [] + for pkg in newpkgs: + po = None + if isinstance(pkg, YumAvailablePackage): + po = pkg + self.read_pkgs.append(po.localpath) + + # if we're dealing with remote pkgs - pitch it over to doing + # them one at a time, for now. + elif pkg.find('://') != -1: + po = self.read_in_package(pkgfile, pkgpath=pkgpath, reldir=reldir) + self.read_pkgs.append(pkg) + + if po: + self.primaryfile.write(po.xml_dump_primary_metadata()) + self.flfile.write(po.xml_dump_filelists_metadata()) + self.otherfile.write(po.xml_dump_other_metadata( + clog_limit=self.conf.changelog_limit)) + continue + + pkgfiles.append(pkg) + + + if pkgfiles: + # divide that list by the number of workers and fork off that many + # workers to tmpdirs + # waitfor the workers to finish and as each one comes in + # open the files they created and write them out to our metadata + # add up the total pkg counts and return that value + worker_tmp_path = tempfile.mkdtemp() + worker_chunks = utils.split_list_into_equal_chunks(pkgfiles, self.conf.workers) + worker_cmd_dict = {} + worker_jobs = {} + base_worker_cmdline = [self.conf.worker_cmd, + '--pkgoptions=_reldir=%s' % reldir, + '--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires, + '--pkgoptions=_cachedir=%s' % self.conf.cachedir, + '--pkgoptions=_baseurl=%s' % self.conf.baseurl, + '--globalopts=clog_limit=%s' % self.conf.changelog_limit,] + + if self.conf.quiet: + base_worker_cmdline.append('--quiet') + + if self.conf.verbose: + base_worker_cmdline.append('--verbose') + + for worker_num in range(self.conf.workers): + # make the worker directory + workercmdline = [] + workercmdline.extend(base_worker_cmdline) + thisdir = worker_tmp_path + '/' + str(worker_num) + if checkAndMakeDir(thisdir): + workercmdline.append('--tmpmdpath=%s' % thisdir) + else: + raise MDError, "Unable to create worker path: %s" % thisdir + workercmdline.extend(worker_chunks[worker_num]) + worker_cmd_dict[worker_num] = workercmdline + + + + for (num, cmdline) in worker_cmd_dict.items(): + if not self.conf.quiet: + self.callback.log("Spawning worker %s with %s pkgs" % (num, + len(worker_chunks[num]))) + job = subprocess.Popen(cmdline, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + worker_jobs[num] = job + + gimmebreak = 0 + while gimmebreak != len(worker_jobs.keys()): + gimmebreak = 0 + for (num,job) in worker_jobs.items(): + if job.poll() is not None: + gimmebreak+=1 + line = job.stdout.readline() + if line: + self.callback.log('Worker %s: %s' % (num, line.rstrip())) + line = job.stderr.readline() + if line: + self.callback.errorlog('Worker %s: %s' % (num, line.rstrip())) + + + if not self.conf.quiet: + self.callback.log("Workers Finished") + # finished with workers + # go to their dirs and add the contents + if not self.conf.quiet: + self.callback.log("Gathering worker results") + for num in range(self.conf.workers): + for (fn, fo) in (('primary.xml', self.primaryfile), + ('filelists.xml', self.flfile), + ('other.xml', self.otherfile)): + fnpath = worker_tmp_path + '/' + str(num) + '/' + fn + if os.path.exists(fnpath): + fo.write(open(fnpath, 'r').read()) + + + for pkgfile in pkgfiles: + if self.conf.deltas: + po = self.read_in_package(pkgfile, pkgpath=pkgpath, reldir=reldir) + self._do_delta_rpm_package(po) + self.read_pkgs.append(pkgfile) + + return self.current_pkg + + + def closeMetadataDocs(self): + if not self.conf.quiet: + self.callback.log('') + + + # save them up to the tmp locations: + if not self.conf.quiet: + self.callback.log(_('Saving Primary metadata')) + if self.conf.database_only: + self.md_sqlite.pri_cx.close() + else: + self.primaryfile.write('\n</metadata>') + self.primaryfile.close() + + if not self.conf.quiet: + self.callback.log(_('Saving file lists metadata')) + if self.conf.database_only: + self.md_sqlite.file_cx.close() + else: + self.flfile.write('\n</filelists>') + self.flfile.close() + + if not self.conf.quiet: + self.callback.log(_('Saving other metadata')) + if self.conf.database_only: + self.md_sqlite.other_cx.close() + else: + self.otherfile.write('\n</otherdata>') + self.otherfile.close() + + if self.conf.deltas: + deltam_st = time.time() + if not self.conf.quiet: + self.callback.log(_('Saving delta metadata')) + self.deltafile.write(self.generate_delta_xml()) + self.deltafile.write('\n</prestodelta>') + self.deltafile.close() + if self.conf.profile: + self.callback.log('deltam time: %0.3f' % (time.time() - deltam_st)) + + def _do_delta_rpm_package(self, pkg): + """makes the drpms, if possible, for this package object. + returns the presto/delta xml metadata as a string + """ + drpm_pkg_time = time.time() + # duck and cover if the pkg.size is > whatever + if int(pkg.size) > self.conf.max_delta_rpm_size: + if not self.conf.quiet: + self.callback.log("Skipping %s package " \ + "that is > max_delta_rpm_size" % pkg) + return + + # generate a list of all the potential 'old rpms' + opd = self._get_old_package_dict() + # for each of our old_package_paths - + # make a drpm from the newest of that pkg + # get list of potential candidates which are likely to match + for d in self.conf.oldpackage_paths: + pot_cand = [] + if d not in opd: + continue + for fn in opd[d]: + if os.path.basename(fn).startswith(pkg.name): + pot_cand.append(fn) + + candidates = [] + for fn in pot_cand: + try: + thispo = yumbased.CreateRepoPackage(self.ts, fn, + sumtype=self.conf.sumtype) + except Errors.MiscError, e: + continue + if (thispo.name, thispo.arch) != (pkg.name, pkg.arch): + # not the same, doesn't matter + continue + if thispo == pkg: #exactly the same, doesn't matter + continue + if thispo.EVR >= pkg.EVR: # greater or equal, doesn't matter + continue + candidates.append(thispo) + candidates.sort() + candidates.reverse() + + for delta_p in candidates[0:self.conf.num_deltas]: + #make drpm of pkg and delta_p + dt_st = time.time() + drpmfn = deltarpms.create_drpm(delta_p, pkg, self.conf.deltadir) + if not self.conf.quiet or self.conf.profile: + self.callback.log('created drpm from %s to %s: %s in %0.3f' % ( + delta_p, pkg, drpmfn, (time.time() - dt_st))) + if self.conf.profile: + self.callback.log('total drpm time for %s: %0.3f' % (pkg, + (time.time() - drpm_pkg_time))) + + def _get_old_package_dict(self): + if hasattr(self, '_old_package_dict'): + return self._old_package_dict + + self._old_package_dict = {} + opl = [] + for d in self.conf.oldpackage_paths: + for f in self.getFileList(d, '.rpm'): + fp = d + '/' + f + fpstat = os.stat(fp) + if int(fpstat[stat.ST_SIZE]) > self.conf.max_delta_rpm_size: + self.callback.log("Skipping %s package " \ + "that is > max_delta_rpm_size" % f) + continue + if not self._old_package_dict.has_key(d): + self._old_package_dict[d] = [] + self._old_package_dict[d].append(d + '/' + f) + + return self._old_package_dict + + def generate_delta_xml(self): + """take the delta rpm output dir, process all the drpm files + produce the text output for the presto/delta xml metadata""" + # go through the drpm dir + # for each file -store the drpm info in a dict based on its target. Just + # appending the output. for each of the keys in the dict, return + # the tag for the target + each of the drpm infos + closure for the target + # tag + targets = {} + results = [] + for drpm_fn in self.getFileList(self.conf.deltadir, '.drpm'): + drpm_rel_fn = os.path.normpath(self.conf.delta_relative + + '/' + drpm_fn) # this is annoying + drpm_po = yumbased.CreateRepoPackage(self.ts, + self.conf.deltadir + '/' + drpm_fn, sumtype=self.conf.sumtype) + + drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir, + drpm_rel_fn) + if not targets.has_key(drpm_po.pkgtup): + targets[drpm_po.pkgtup] = [] + targets[drpm_po.pkgtup].append(drpm.xml_dump_metadata()) + + for (n, a, e, v, r) in targets.keys(): + results.append(""" <newpackage name="%s" epoch="%s" version="%s" release="%s" arch="%s">\n""" % ( + n, e, v, r, a)) + results.extend(targets[(n,a,e,v,r)]) +# for src in targets[(n, a, e, v, r)]: +# results.append(src) + + results.append(" </newpackage>\n") + + return ' '.join(results) + + def _createRepoDataObject(self, mdfile, mdtype, compress=True, + compress_type='gzip', attribs={}): + """return random metadata as RepoData object to be added to RepoMD + mdfile = complete path to file + mdtype = the metadata type to use + compress = compress the file before including it + """ + # copy the file over here + sfile = os.path.basename(mdfile) + fo = open(mdfile, 'r') + outdir = os.path.join(self.conf.outputdir, self.conf.tempdir) + if compress: + if compress_type == 'gzip': + sfile = '%s.gz' % sfile + outfn = os.path.join(outdir, sfile) + output = GzipFile(filename = outfn, mode='wb') + elif compress_type == 'bzip2': + sfile = '%s.bz2' % sfile + outfn = os.path.join(outdir, sfile) + output = BZ2File(filename = outfn, mode='wb') + else: + outfn = os.path.join(outdir, sfile) + output = open(outfn, 'w') + + output.write(fo.read()) + output.close() + fo.seek(0) + open_csum = misc.checksum(self.conf.sumtype, fo) + fo.close() + + + if self.conf.unique_md_filenames: + (csum, outfn) = checksum_and_rename(outfn, self.conf.sumtype) + sfile = os.path.basename(outfn) + else: + if compress: + csum = misc.checksum(self.conf.sumtype, outfn) + else: + csum = open_csum + + thisdata = RepoData() + thisdata.type = mdtype + baseloc = None + thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile)) + thisdata.checksum = (self.conf.sumtype, csum) + if compress: + thisdata.openchecksum = (self.conf.sumtype, open_csum) + + thisdata.size = str(os.stat(outfn).st_size) + thisdata.timestamp = str(os.stat(outfn).st_mtime) + for (k, v) in attribs.items(): + setattr(thisdata, k, str(v)) + + return thisdata + + + def doRepoMetadata(self): + """wrapper to generate the repomd.xml file that stores the info + on the other files""" + + repomd = RepoMD('repoid') + repomd.revision = self.conf.revision + + repopath = os.path.join(self.conf.outputdir, self.conf.tempdir) + repofilepath = os.path.join(repopath, self.conf.repomdfile) + + if self.conf.content_tags: + repomd.tags['content'] = self.conf.content_tags + if self.conf.distro_tags: + repomd.tags['distro'] = self.conf.distro_tags + # NOTE - test out the cpeid silliness here + if self.conf.repo_tags: + repomd.tags['repo'] = self.conf.repo_tags + + + sumtype = self.conf.sumtype + workfiles = [(self.conf.otherfile, 'other',), + (self.conf.filelistsfile, 'filelists'), + (self.conf.primaryfile, 'primary')] + + if self.conf.deltas: + workfiles.append((self.conf.deltafile, 'prestodelta')) + + if self.conf.database: + if not self.conf.quiet: self.callback.log('Generating sqlite DBs') + try: + dbversion = str(sqlitecachec.DBVERSION) + except AttributeError: + dbversion = '9' + #FIXME - in theory some sort of try/except here + rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None) + + for (rpm_file, ftype) in workfiles: + complete_path = os.path.join(repopath, rpm_file) + + zfo = _gzipOpen(complete_path) + # This is misc.checksum() done locally so we can get the size too. + data = misc.Checksums([sumtype]) + while data.read(zfo, 2**16): + pass + uncsum = data.hexdigest(sumtype) + unsize = len(data) + zfo.close() + csum = misc.checksum(sumtype, complete_path) + timestamp = os.stat(complete_path)[8] + + db_csums = {} + db_compressed_sums = {} + + if self.conf.database: + if ftype in ['primary', 'filelists', 'other']: + if self.conf.verbose: + self.callback.log("Starting %s db creation: %s" % (ftype, + time.ctime())) + + if ftype == 'primary': + #FIXME - in theory some sort of try/except here + # TypeError appears to be raised, sometimes :( + rp.getPrimary(complete_path, csum) + + elif ftype == 'filelists': + #FIXME and here + rp.getFilelists(complete_path, csum) + + elif ftype == 'other': + #FIXME and here + rp.getOtherdata(complete_path, csum) + + if ftype in ['primary', 'filelists', 'other']: + tmp_result_name = '%s.xml.gz.sqlite' % ftype + tmp_result_path = os.path.join(repopath, tmp_result_name) + good_name = '%s.sqlite' % ftype + resultpath = os.path.join(repopath, good_name) + + # rename from silly name to not silly name + os.rename(tmp_result_path, resultpath) + compressed_name = '%s.bz2' % good_name + result_compressed = os.path.join(repopath, compressed_name) + db_csums[ftype] = misc.checksum(sumtype, resultpath) + + # compress the files + bzipFile(resultpath, result_compressed) + # csum the compressed file + db_compressed_sums[ftype] = misc.checksum(sumtype, + result_compressed) + # timestamp+size the uncompressed file + un_stat = os.stat(resultpath) + # remove the uncompressed file + os.unlink(resultpath) + + if self.conf.unique_md_filenames: + csum_compressed_name = '%s-%s.bz2' % ( + db_compressed_sums[ftype], good_name) + csum_result_compressed = os.path.join(repopath, + csum_compressed_name) + os.rename(result_compressed, csum_result_compressed) + result_compressed = csum_result_compressed + compressed_name = csum_compressed_name + + # timestamp+size the compressed file + db_stat = os.stat(result_compressed) + + # add this data as a section to the repomdxml + db_data_type = '%s_db' % ftype + data = RepoData() + data.type = db_data_type + data.location = (self.conf.baseurl, + os.path.join(self.conf.finaldir, compressed_name)) + data.checksum = (sumtype, db_compressed_sums[ftype]) + data.timestamp = str(db_stat.st_mtime) + data.size = str(db_stat.st_size) + data.opensize = str(un_stat.st_size) + data.openchecksum = (sumtype, db_csums[ftype]) + data.dbversion = dbversion + if self.conf.verbose: + self.callback.log("Ending %s db creation: %s" % (ftype, + time.ctime())) + repomd.repoData[data.type] = data + + data = RepoData() + data.type = ftype + data.checksum = (sumtype, csum) + data.timestamp = str(timestamp) + data.size = str(os.stat(os.path.join(repopath, rpm_file)).st_size) + data.opensize = str(unsize) + data.openchecksum = (sumtype, uncsum) + + if self.conf.unique_md_filenames: + res_file = '%s-%s.xml.gz' % (csum, ftype) + orig_file = os.path.join(repopath, rpm_file) + dest_file = os.path.join(repopath, res_file) + os.rename(orig_file, dest_file) + else: + res_file = rpm_file + rpm_file = res_file + href = os.path.join(self.conf.finaldir, rpm_file) + + data.location = (self.conf.baseurl, href) + repomd.repoData[data.type] = data + + if not self.conf.quiet and self.conf.database: + self.callback.log('Sqlite DBs complete') + + + if self.conf.groupfile is not None: + mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group_gz') + repomd.repoData[mdcontent.type] = mdcontent + + mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group', + compress=False) + repomd.repoData[mdcontent.type] = mdcontent + + + if self.conf.additional_metadata: + for md_type, mdfile in self.conf.additional_metadata.items(): + mdcontent = self._createRepoDataObject(md_file, md_type) + repomd.repoData[mdcontent.type] = mdcontent + + + # FIXME - disabled until we decide how best to use this + #if self.rpmlib_reqs: + # rpmlib = reporoot.newChild(rpmns, 'lib', None) + # for r in self.rpmlib_reqs.keys(): + # req = rpmlib.newChild(rpmns, 'requires', r) + + + # save it down + try: + fo = open(repofilepath, 'w') + fo.write(repomd.dump_xml()) + fo.close() + except (IOError, OSError, TypeError), e: + self.callback.errorlog( + _('Error saving temp file for repomd.xml: %s') % repofilepath) + self.callback.errorlog('Error was: %s') % str(e) + fo.close() + raise MDError, 'Could not save temp file: %s' % repofilepath + + + def doFinalMove(self): + """move the just-created repodata from .repodata to repodata + also make sure to preserve any files we didn't mess with in the + metadata dir""" + + output_final_dir = os.path.join(self.conf.outputdir, self.conf.finaldir) + output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir) + + if os.path.exists(output_final_dir): + try: + os.rename(output_final_dir, output_old_dir) + except: + raise MDError, _('Error moving final %s to old dir %s' % ( + output_final_dir, output_old_dir)) + + output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir) + + try: + os.rename(output_temp_dir, output_final_dir) + except: + # put the old stuff back + os.rename(output_old_dir, output_final_dir) + raise MDError, _('Error moving final metadata into place') + + for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', + 'groupfile']: + if getattr(self.conf, f): + fn = os.path.basename(getattr(self.conf, f)) + else: + continue + oldfile = os.path.join(output_old_dir, fn) + + if os.path.exists(oldfile): + try: + os.remove(oldfile) + except OSError, e: + raise MDError, _( + 'Could not remove old metadata file: %s: %s') % (oldfile, e) + + # Move everything else back from olddir (eg. repoview files) + try: + old_contents = os.listdir(output_old_dir) + except (OSError, IOError), e: + old_contents = [] + + for f in os.listdir(output_old_dir): + oldfile = os.path.join(output_old_dir, f) + finalfile = os.path.join(output_final_dir, f) + if f.find('-') != -1 and f.split('-')[1] in ('primary.sqlite.bz2', + 'filelists.sqlite.bz2', 'primary.xml.gz','other.sqlite.bz2', + 'other.xml.gz','filelists.xml.gz'): + os.remove(oldfile) # kill off the old ones + continue + if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2', + 'primary.sqlite.bz2'): + os.remove(oldfile) + continue + + if os.path.exists(finalfile): + # Hmph? Just leave it alone, then. + try: + if os.path.isdir(oldfile): + shutil.rmtree(oldfile) + else: + os.remove(oldfile) + except OSError, e: + raise MDError, _( + 'Could not remove old metadata file: %s: %s') % (oldfile, e) + else: + try: + os.rename(oldfile, finalfile) + except OSError, e: + msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile) + msg += _('Error was %s') % e + raise MDError, msg + + try: + os.rmdir(output_old_dir) + except OSError, e: + self.errorlog(_('Could not remove old metadata dir: %s') + % self.conf.olddir) + self.errorlog(_('Error was %s') % e) + self.errorlog(_('Please clean up this directory manually.')) + + # write out the read_pkgs_list file with self.read_pkgs + if self.conf.read_pkgs_list: + try: + fo = open(self.conf.read_pkgs_list, 'w') + fo.write('\n'.join(self.read_pkgs)) + fo.flush() + fo.close() + except (OSError, IOError), e: + self.errorlog(_('Could not write out readpkgs list: %s') + % self.conf.read_pkgs_list) + self.errorlog(_('Error was %s') % e) + + def setup_sqlite_dbs(self, initdb=True): + """sets up the sqlite dbs w/table schemas and db_infos""" + destdir = os.path.join(self.conf.outputdir, self.conf.tempdir) + try: + self.md_sqlite = MetaDataSqlite(destdir) + except sqlite.OperationalError, e: + raise MDError, _('Cannot create sqlite databases: %s.\n'\ + 'Maybe you need to clean up a .repodata dir?') % e + + + +class SplitMetaDataGenerator(MetaDataGenerator): + """takes a series of dirs and creates repodata for all of them + most commonly used with -u media:// - if no outputdir is specified + it will create the repodata in the first dir in the list of dirs + """ + def __init__(self, config_obj=None, callback=None): + MetaDataGenerator.__init__(self, config_obj=config_obj, callback=None) + + def _getFragmentUrl(self, url, fragment): + import urlparse + urlparse.uses_fragment.append('media') + if not url: + return url + (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url) + return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment))) + + def getFileList(self, directory, ext): + + extlen = len(ext) + + def extension_visitor(arg, dirname, names): + for fn in names: + if os.path.isdir(fn): + continue + elif fn[-extlen:].lower() == '%s' % (ext): + reldir = os.path.basename(dirname) + if reldir == os.path.basename(directory): + reldir = "" + arg.append(os.path.join(reldir, fn)) + + rpmlist = [] + os.path.walk(directory, extension_visitor, rpmlist) + return rpmlist + + def doPkgMetadata(self): + """all the heavy lifting for the package metadata""" + if len(self.conf.directories) == 1: + MetaDataGenerator.doPkgMetadata(self) + return + + if self.conf.update: + self._setup_old_metadata_lookup() + + filematrix = {} + for mydir in self.conf.directories: + if os.path.isabs(mydir): + thisdir = mydir + else: + if mydir.startswith('../'): + thisdir = os.path.realpath(mydir) + else: + thisdir = os.path.join(self.conf.basedir, mydir) + + filematrix[mydir] = self.getFileList(thisdir, '.rpm') + self.trimRpms(filematrix[mydir]) + self.pkgcount += len(filematrix[mydir]) + + mediano = 1 + self.current_pkg = 0 + self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano) + try: + self.openMetadataDocs() + original_basedir = self.conf.basedir + for mydir in self.conf.directories: + self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano) + self.writeMetadataDocs(filematrix[mydir], mydir) + mediano += 1 + self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, 1) + self.closeMetadataDocs() + except (IOError, OSError), e: + raise MDError, _('Cannot access/write repodata files: %s') % e + + + +class MetaDataSqlite(object): + def __init__(self, destdir): + self.pri_sqlite_file = os.path.join(destdir, 'primary.sqlite') + self.pri_cx = sqlite.Connection(self.pri_sqlite_file) + self.file_sqlite_file = os.path.join(destdir, 'filelists.sqlite') + self.file_cx = sqlite.Connection(self.file_sqlite_file) + self.other_sqlite_file = os.path.join(destdir, 'other.sqlite') + self.other_cx = sqlite.Connection(self.other_sqlite_file) + self.primary_cursor = self.pri_cx.cursor() + + self.filelists_cursor = self.file_cx.cursor() + + self.other_cursor = self.other_cx.cursor() + + self.create_primary_db() + self.create_filelists_db() + self.create_other_db() + + def create_primary_db(self): + # make the tables + schema = [ + """PRAGMA synchronous="OFF";""", + """pragma locking_mode="EXCLUSIVE";""", + """CREATE TABLE conflicts ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""", + """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""", + """CREATE TABLE files ( name TEXT, type TEXT, pkgKey INTEGER);""", + """CREATE TABLE obsoletes ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""", + """CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT, name TEXT, arch TEXT, version TEXT, epoch TEXT, release TEXT, summary TEXT, description TEXT, url TEXT, time_file INTEGER, time_build INTEGER, rpm_license TEXT, rpm_vendor TEXT, rpm_group TEXT, rpm_buildhost TEXT, rpm_sourcerpm TEXT, rpm_header_start INTEGER, rpm_header_end INTEGER, rpm_packager TEXT, size_package INTEGER, size_installed INTEGER, size_archive INTEGER, location_href TEXT, location_base TEXT, checksum_type TEXT);""", + """CREATE TABLE provides ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""", + """CREATE TABLE requires ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER , pre BOOL DEFAULT FALSE);""", + """CREATE INDEX filenames ON files (name);""", + """CREATE INDEX packageId ON packages (pkgId);""", + """CREATE INDEX packagename ON packages (name);""", + """CREATE INDEX pkgconflicts on conflicts (pkgKey);""", + """CREATE INDEX pkgobsoletes on obsoletes (pkgKey);""", + """CREATE INDEX pkgprovides on provides (pkgKey);""", + """CREATE INDEX pkgrequires on requires (pkgKey);""", + """CREATE INDEX providesname ON provides (name);""", + """CREATE INDEX requiresname ON requires (name);""", + """CREATE TRIGGER removals AFTER DELETE ON packages + BEGIN + DELETE FROM files WHERE pkgKey = old.pkgKey; + DELETE FROM requires WHERE pkgKey = old.pkgKey; + DELETE FROM provides WHERE pkgKey = old.pkgKey; + DELETE FROM conflicts WHERE pkgKey = old.pkgKey; + DELETE FROM obsoletes WHERE pkgKey = old.pkgKey; + END;""", + """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION, + ] + + for cmd in schema: + executeSQL(self.primary_cursor, cmd) + + def create_filelists_db(self): + schema = [ + """PRAGMA synchronous="OFF";""", + """pragma locking_mode="EXCLUSIVE";""", + """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""", + """CREATE TABLE filelist ( pkgKey INTEGER, dirname TEXT, filenames TEXT, filetypes TEXT);""", + """CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);""", + """CREATE INDEX dirnames ON filelist (dirname);""", + """CREATE INDEX keyfile ON filelist (pkgKey);""", + """CREATE INDEX pkgId ON packages (pkgId);""", + """CREATE TRIGGER remove_filelist AFTER DELETE ON packages + BEGIN + DELETE FROM filelist WHERE pkgKey = old.pkgKey; + END;""", + """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION, + ] + for cmd in schema: + executeSQL(self.filelists_cursor, cmd) + + def create_other_db(self): + schema = [ + """PRAGMA synchronous="OFF";""", + """pragma locking_mode="EXCLUSIVE";""", + """CREATE TABLE changelog ( pkgKey INTEGER, author TEXT, date INTEGER, changelog TEXT);""", + """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""", + """CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);""", + """CREATE INDEX keychange ON changelog (pkgKey);""", + """CREATE INDEX pkgId ON packages (pkgId);""", + """CREATE TRIGGER remove_changelogs AFTER DELETE ON packages + BEGIN + DELETE FROM changelog WHERE pkgKey = old.pkgKey; + END;""", + """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION, + ] + + for cmd in schema: + executeSQL(self.other_cursor, cmd) diff --git a/createrepo/deltarpms.py b/createrepo/deltarpms.py new file mode 100644 index 0000000..3edcbb5 --- /dev/null +++ b/createrepo/deltarpms.py @@ -0,0 +1,124 @@ +#!/usr/bin/python -tt +# util functions for deltarpms +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# copyright 2009 - Red Hat + +import os.path +import commands +from yum import misc +import deltarpm +from utils import MDError + +class DeltaRPMPackage: + """each drpm is one object, you pass it a drpm file + it opens the file, and pulls the information out in bite-sized chunks :) + """ + + mode_cache = {} + + def __init__(self, po, basedir, filename): + try: + stats = os.stat(os.path.join(basedir, filename)) + self.size = stats[6] + self.mtime = stats[8] + del stats + except OSError, e: + raise MDError, "Error Stat'ing file %s%s" % (basedir, filename) + self.csum_type = 'sha256' + self.relativepath = filename + self.po = po + + fd = os.open(self.po.localpath, os.O_RDONLY) + os.lseek(fd, 0, 0) + fo = os.fdopen(fd, 'rb') + self.csum = misc.checksum(self.csum_type, fo) + del fo + del fd + self._getDRPMInfo(os.path.join(basedir, filename)) + + def _stringToNEVR(self, string): + i = string.rfind("-", 0, string.rfind("-")-1) + name = string[:i] + (epoch, ver, rel) = self._stringToVersion(string[i+1:]) + return (name, epoch, ver, rel) + + def _getLength(self, in_data): + length = 0 + for val in in_data: + length = length * 256 + length += ord(val) + return length + + def _getDRPMInfo(self, filename): + d = deltarpm.readDeltaRPM(filename) + self.oldnevrstring = d['old_nevr'] + self.oldnevr = self._stringToNEVR(d['old_nevr']) + self.sequence = d['seq'] + + def _stringToVersion(self, strng): + i = strng.find(':') + if i != -1: + epoch = strng[:i] + else: + epoch = '0' + j = strng.find('-') + if j != -1: + if strng[i + 1:j] == '': + version = None + else: + version = strng[i + 1:j] + release = strng[j + 1:] + else: + if strng[i + 1:] == '': + version = None + else: + version = strng[i + 1:] + release = None + return (epoch, version, release) + + def xml_dump_metadata(self): + """takes an xml doc object and a package metadata entry node, populates a + package node with the md information""" + + (oldname, oldepoch, oldver, oldrel) = self.oldnevr + sequence = "%s-%s" % (self.oldnevrstring, self.sequence) + + delta_tag = """ <delta oldepoch="%s" oldversion="%s" oldrelease="%s"> + <filename>%s</filename> + <sequence>%s</sequence> + <size>%s</size> + <checksum type="%s">%s</checksum> + </delta>\n""" % (oldepoch, oldver, oldrel, self.relativepath, sequence, + self.size, self.csum_type, self.csum) + return delta_tag + +def create_drpm(old_pkg, new_pkg, destdir): + """make a drpm file, if possible. returns None if nothing could + be created""" + drpmfn = '%s-%s-%s_%s-%s.%s.drpm' % (old_pkg.name, old_pkg.ver, + old_pkg.release, new_pkg.ver, new_pkg.release, + old_pkg.arch) + delta_rpm_path = os.path.join(destdir, drpmfn) + delta_command = '/usr/bin/makedeltarpm %s %s %s' % (old_pkg.localpath, + new_pkg.localpath, + delta_rpm_path) + if not os.path.exists(delta_rpm_path): + #TODO - check/verify the existing one a bit? + (code, out) = commands.getstatusoutput(delta_command) + if code: + print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (old_pkg.name, code, out) + return None + + return delta_rpm_path diff --git a/createrepo/merge.py b/createrepo/merge.py new file mode 100644 index 0000000..b3b2ea1 --- /dev/null +++ b/createrepo/merge.py @@ -0,0 +1,139 @@ +#!/usr/bin/python -tt +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2008 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org + +# merge repos from arbitrary repo urls + +import os +import shutil +import yum +import yum.Errors +from yum.misc import unique, getCacheDir +import yum.update_md +import rpmUtils.arch +import operator +import createrepo +import tempfile + +# take repo paths from cli +# produce new repo metadata from merging the two together. + +#TODO: +# excludes? + + +class RepoMergeBase: + def __init__(self, repolist=[], yumbase=None, mdconf=None, mdbase_class=None ): + self.repolist = repolist + self.outputdir = '%s/merged_repo' % os.getcwd() + self.exclude_tuples = [] + self.sort_func = self._sort_func # callback function to magically sort pkgs + if not mdconf: + self.mdconf = createrepo.MetaDataConfig() + else: + self.mdconf = mdconf + if not mdbase_class: + self.mdbase_class = createrepo.MetaDataGenerator + else: + self.mdbase_class = mdbase_class + if not yumbase: + self.yumbase = yum.YumBase() + else: + self.yumbase = yumbase + self.yumbase.conf.cachedir = getCacheDir() + self.yumbase.conf.cache = 0 + # default to all arches + self.archlist = unique(rpmUtils.arch.arches.keys() + rpmUtils.arch.arches.values()) + self.groups = True + self.updateinfo = True + + def _sort_func(self, repos): + """Default sort func for repomerge. Takes a list of repository objects + any package which is not to be included in the merged repo should be + delPackage()'d""" + # sort the repos by _merge_rank + # - lowest number is the highest rank (1st place, 2ndplace, etc) + repos.sort(key=operator.attrgetter('_merge_rank')) + + for repo in repos: + for pkg in repo.sack: + others = self.yumbase.pkgSack.searchNevra(name=pkg.name, arch=pkg.arch) + # NOTE the above is definitely going to catch other versions which may + # be an invalid comparison + if len(others) > 1: + for thatpkg in others: + if pkg.repoid == thatpkg.repoid: continue + if pkg.repo._merge_rank < thatpkg.repo._merge_rank: + thatpkg.repo.sack.delPackage(thatpkg) + + def merge_repos(self): + self.yumbase.repos.disableRepo('*') + # add our repos and give them a merge rank in the order they appear in + # in the repolist + count = 0 + for r in self.repolist: + count +=1 + rid = 'repo%s' % count + n = self.yumbase.add_enable_repo(rid, baseurls=[r], + metadata_expire=0, + timestamp_check=False) + n._merge_rank = count + + #setup our sacks + self.yumbase._getSacks(archlist=self.archlist) + + myrepos = self.yumbase.repos.listEnabled() + + self.sort_func(myrepos) + + + def write_metadata(self, outputdir=None): + mytempdir = tempfile.mkdtemp() + if self.groups: + comps_fn = mytempdir + '/groups.xml' + compsfile = open(comps_fn, 'w') + compsfile.write(self.yumbase.comps.xml()) + compsfile.close() + self.mdconf.groupfile=comps_fn + + if self.updateinfo: + ui_fn = mytempdir + '/updateinfo.xml' + uifile = open(ui_fn, 'w') + umd = yum.update_md.UpdateMetadata() + for repo in self.yumbase.repos.listEnabled(): + try: # attempt to grab the updateinfo.xml.gz from the repodata + umd.add(repo) + except yum.Errors.RepoMDError: + continue + umd.xml(fileobj=uifile) + uifile.close() + self.mdconf.additional_metadata['updateinfo'] = ui_fn + + + self.mdconf.pkglist = self.yumbase.pkgSack + self.mdconf.directory = self.outputdir + if outputdir: + self.mdconf.directory = outputdir + # clean out what was there + if os.path.exists(self.mdconf.directory + '/repodata'): + shutil.rmtree(self.mdconf.directory + '/repodata') + + if not os.path.exists(self.mdconf.directory): + os.makedirs(self.mdconf.directory) + + mdgen = self.mdbase_class(config_obj=self.mdconf) + mdgen.doPkgMetadata() + mdgen.doRepoMetadata() + mdgen.doFinalMove() diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py new file mode 100644 index 0000000..27d3690 --- /dev/null +++ b/createrepo/readMetadata.py @@ -0,0 +1,217 @@ +#!/usr/bin/python -t + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2006 Red Hat + +import os +import libxml2 +import stat +from utils import errorprint, _ + +from yum import repoMDObject + + +class MetadataIndex(object): + + def __init__(self, outputdir, opts=None): + if opts is None: + opts = {} + self.opts = opts + self.outputdir = outputdir + repodatadir = self.outputdir + '/repodata' + myrepomdxml = repodatadir + '/repomd.xml' + if os.path.exists(myrepomdxml): + repomd = repoMDObject.RepoMD('garbageid', myrepomdxml) + b = repomd.getData('primary').location[1] + f = repomd.getData('filelists').location[1] + o = repomd.getData('other').location[1] + basefile = os.path.join(self.outputdir, b) + filelistfile = os.path.join(self.outputdir, f) + otherfile = os.path.join(self.outputdir, o) + else: + basefile = filelistfile = otherfile = "" + + self.files = {'base' : basefile, + 'filelist' : filelistfile, + 'other' : otherfile} + self.scan() + + def scan(self): + """Read in and index old repo data""" + self.basenodes = {} + self.filesnodes = {} + self.othernodes = {} + self.pkg_ids = {} + if self.opts.get('verbose'): + print _("Scanning old repo data") + for fn in self.files.values(): + if not os.path.exists(fn): + #cannot scan + errorprint(_("Warning: Old repodata file missing: %s") % fn) + return + root = libxml2.parseFile(self.files['base']).getRootElement() + self._scanPackageNodes(root, self._handleBase) + if self.opts.get('verbose'): + print _("Indexed %i base nodes" % len(self.basenodes)) + root = libxml2.parseFile(self.files['filelist']).getRootElement() + self._scanPackageNodes(root, self._handleFiles) + if self.opts.get('verbose'): + print _("Indexed %i filelist nodes" % len(self.filesnodes)) + root = libxml2.parseFile(self.files['other']).getRootElement() + self._scanPackageNodes(root, self._handleOther) + if self.opts.get('verbose'): + print _("Indexed %i other nodes" % len(self.othernodes)) + #reverse index pkg ids to track references + self.pkgrefs = {} + for relpath, pkgid in self.pkg_ids.iteritems(): + self.pkgrefs.setdefault(pkgid,[]).append(relpath) + + def _scanPackageNodes(self, root, handler): + node = root.children + while node is not None: + if node.type != "element": + node = node.next + continue + if node.name == "package": + handler(node) + node = node.next + + def _handleBase(self, node): + top = node + node = node.children + pkgid = None + mtime = None + size = None + relpath = None + do_stat = self.opts.get('do_stat', True) + while node is not None: + if node.type != "element": + node = node.next + continue + if node.name == "checksum": + pkgid = node.content + elif node.name == "time": + mtime = int(node.prop('file')) + elif node.name == "size": + size = int(node.prop('package')) + elif node.name == "location": + relpath = node.prop('href') + node = node.next + if relpath is None: + print _("Incomplete data for node") + return + if pkgid is None: + print _("pkgid missing for %s") % relpath + return + if mtime is None: + print _("mtime missing for %s") % relpath + return + if size is None: + print _("size missing for %s") % relpath + return + if do_stat: + filepath = os.path.join(self.opts['pkgdir'], relpath) + try: + st = os.stat(filepath) + except OSError: + #file missing -- ignore + return + if not stat.S_ISREG(st.st_mode): + #ignore non files + return + #check size and mtime + if st.st_size != size: + if self.opts.get('verbose'): + print _("Size (%i -> %i) changed for file %s") % (size,st.st_size,filepath) + return + if int(st.st_mtime) != mtime: + if self.opts.get('verbose'): + print _("Modification time changed for %s") % filepath + return + #otherwise we index + self.basenodes[relpath] = top + self.pkg_ids[relpath] = pkgid + + def _handleFiles(self, node): + pkgid = node.prop('pkgid') + if pkgid: + self.filesnodes[pkgid] = node + + def _handleOther(self, node): + pkgid = node.prop('pkgid') + if pkgid: + self.othernodes[pkgid] = node + + def getNodes(self, relpath): + """Return base, filelist, and other nodes for file, if they exist + + Returns a tuple of nodes, or None if not found + """ + bnode = self.basenodes.get(relpath,None) + if bnode is None: + return None + pkgid = self.pkg_ids.get(relpath,None) + if pkgid is None: + print _("No pkgid found for: %s") % relpath + return None + fnode = self.filesnodes.get(pkgid,None) + if fnode is None: + return None + onode = self.othernodes.get(pkgid,None) + if onode is None: + return None + return bnode, fnode, onode + + def freeNodes(self,relpath): + #causing problems + """Free up nodes corresponding to file, if possible""" + bnode = self.basenodes.get(relpath,None) + if bnode is None: + print "Missing node for %s" % relpath + return + bnode.unlinkNode() + bnode.freeNode() + del self.basenodes[relpath] + pkgid = self.pkg_ids.get(relpath,None) + if pkgid is None: + print _("No pkgid found for: %s") % relpath + return None + del self.pkg_ids[relpath] + dups = self.pkgrefs.get(pkgid) + dups.remove(relpath) + if len(dups): + #still referenced + return + del self.pkgrefs[pkgid] + for nodes in self.filesnodes, self.othernodes: + node = nodes.get(pkgid) + if node is not None: + node.unlinkNode() + node.freeNode() + del nodes[pkgid] + + +if __name__ == "__main__": + cwd = os.getcwd() + opts = {'verbose':1, + 'pkgdir': cwd} + + idx = MetadataIndex(cwd, opts) + for fn in idx.basenodes.keys(): + a,b,c, = idx.getNodes(fn) + a.serialize() + b.serialize() + c.serialize() + idx.freeNodes(fn) diff --git a/createrepo/utils.py b/createrepo/utils.py new file mode 100644 index 0000000..995c3b9 --- /dev/null +++ b/createrepo/utils.py @@ -0,0 +1,143 @@ +#!/usr/bin/python +# util functions for createrepo +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + + +import os +import os.path +import sys +import bz2 +import gzip +from gzip import write32u, FNAME +from yum import misc + +def errorprint(stuff): + print >> sys.stderr, stuff + +def _(args): + """Stub function for translation""" + return args + + +class GzipFile(gzip.GzipFile): + def _write_gzip_header(self): + self.fileobj.write('\037\213') # magic header + self.fileobj.write('\010') # compression method + if hasattr(self, 'name'): + fname = self.name[:-3] + else: + fname = self.filename[:-3] + flags = 0 + if fname: + flags = FNAME + self.fileobj.write(chr(flags)) + write32u(self.fileobj, long(0)) + self.fileobj.write('\002') + self.fileobj.write('\377') + if fname: + self.fileobj.write(fname + '\000') + + +def _gzipOpen(filename, mode="rb", compresslevel=9): + return GzipFile(filename, mode, compresslevel) + +def bzipFile(source, dest): + + s_fn = open(source, 'rb') + destination = bz2.BZ2File(dest, 'w', compresslevel=9) + + while True: + data = s_fn.read(1024000) + + if not data: break + destination.write(data) + + destination.close() + s_fn.close() + + +def returnFD(filename): + try: + fdno = os.open(filename, os.O_RDONLY) + except OSError: + raise MDError, "Error opening file" + return fdno + +def checkAndMakeDir(directory): + """ + check out the directory and make it, if possible, return 1 if done, else return 0 + """ + if os.path.exists(directory): + if not os.path.isdir(directory): + #errorprint(_('%s is not a dir') % directory) + result = False + else: + if not os.access(directory, os.W_OK): + #errorprint(_('%s is not writable') % directory) + result = False + else: + result = True + else: + try: + os.mkdir(directory) + except OSError, e: + #errorprint(_('Error creating dir %s: %s') % (directory, e)) + result = False + else: + result = True + return result + +def checksum_and_rename(fn_path, sumtype='sha256'): + """checksum the file rename the file to contain the checksum as a prefix + return the new filename""" + csum = misc.checksum(sumtype, fn_path) + fn = os.path.basename(fn_path) + fndir = os.path.dirname(fn_path) + csum_fn = csum + '-' + fn + csum_path = os.path.join(fndir, csum_fn) + os.rename(fn_path, csum_path) + return (csum, csum_path) + + + +def encodefilenamelist(filenamelist): + return '/'.join(filenamelist) + +def encodefiletypelist(filetypelist): + result = '' + ftl = {'file':'f', 'dir':'d', 'ghost':'g'} + for x in filetypelist: + result += ftl[x] + return result + +def split_list_into_equal_chunks(seq, num_chunks): + avg = len(seq) / float(num_chunks) + out = [] + last = 0.0 + while last < len(seq): + out.append(seq[int(last):int(last + avg)]) + last += avg + + return out + + +class MDError(Exception): + def __init__(self, value=None): + Exception.__init__(self) + self.value = value + + def __str__(self): + return self.value diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py new file mode 100644 index 0000000..ac06196 --- /dev/null +++ b/createrepo/yumbased.py @@ -0,0 +1,235 @@ +#!/usr/bin/python -tt +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2007 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org + + +import os +import rpm +import types + +from yum.packages import YumLocalPackage +from yum.Errors import * +from yum import misc +import utils +import tempfile + +class CreateRepoPackage(YumLocalPackage): + def __init__(self, ts, package, sumtype=None, external_data={}): + YumLocalPackage.__init__(self, ts, package) + if sumtype: + self.checksum_type = sumtype + + if external_data: + for (key, val) in external_data.items(): + setattr(self, key, val) + + + def _do_checksum(self): + """return a checksum for a package: + - check if the checksum cache is enabled + if not - return the checksum + if so - check to see if it has a cache file + if so, open it and return the first line's contents + if not, grab the checksum and write it to a file for this pkg + """ + # already got it + if self._checksum: + return self._checksum + + # not using the cachedir + if not hasattr(self, '_cachedir') or not self._cachedir: + self._checksum = misc.checksum(self.checksum_type, self.localpath) + self._checksums = [(self.checksum_type, self._checksum, 1)] + return self._checksum + + + t = [] + if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType: + t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG])) + if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType: + t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP])) + if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType: + t.append("".join(self.hdr[rpm.RPMTAG_HDRID])) + + kcsum = misc.Checksums(checksums=[self.checksum_type]) + kcsum.update("".join(t)) + key = kcsum.hexdigest() + + csumtag = '%s-%s-%s-%s' % (os.path.basename(self.localpath), + key, self.size, self.filetime) + csumfile = '%s/%s' % (self._cachedir, csumtag) + + if os.path.exists(csumfile) and float(self.filetime) <= float(os.stat(csumfile)[-2]): + csumo = open(csumfile, 'r') + checksum = csumo.readline() + csumo.close() + + else: + checksum = misc.checksum(self.checksum_type, self.localpath) + + # This is atomic cache creation via. rename, so we can have two + # tasks using the same cachedir ... mash does this. + try: + (csumo, tmpfilename) = tempfile.mkstemp(dir=self._cachedir) + csumo = os.fdopen(csumo, 'w', -1) + csumo.write(checksum) + csumo.close() + os.rename(tmpfilename, csumfile) + except: + pass + + self._checksum = checksum + self._checksums = [(self.checksum_type, checksum, 1)] + + return self._checksum + + # sqlite-direct dump code below here :-/ + + def _sqlite_null(self, item): + if not item: + return None + return item + + def do_primary_sqlite_dump(self, cur): + """insert primary data in place, this assumes the tables exist""" + if self.crp_reldir and self.localpath.startswith(self.crp_reldir): + relpath = self.localpath.replace(self.crp_reldir, '') + if relpath[0] == '/': relpath = relpath[1:] + else: + relpath = self.localpath + + p = (self.crp_packagenumber, self.checksum, self.name, self.arch, + self.version, self.epoch, self.release, self.summary.strip(), + self.description.strip(), self._sqlite_null(self.url), self.filetime, + self.buildtime, self._sqlite_null(self.license), + self._sqlite_null(self.vendor), self._sqlite_null(self.group), + self._sqlite_null(self.buildhost), self._sqlite_null(self.sourcerpm), + self.hdrstart, self.hdrend, self._sqlite_null(self.packager), + self.packagesize, self.size, self.archivesize, relpath, + self.crp_baseurl, self.checksum_type) + + q = """insert into packages values (?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, + ?, ?, ?)""" + + # write out all of do_primary_sqlite as an executescript - work on the + # quoting for pretty much any contingency - take from sqlutils.py + # + # e + #p = None + #q = """insert into packages values (%s, %s, %s, %s, """ + + cur.execute(q, p) + + # provides, obsoletes, conflicts + for pco in ('obsoletes', 'provides', 'conflicts'): + thispco = [] + for (name, flag, (epoch, ver, rel)) in getattr(self, pco): + thispco.append((name, flag, epoch, ver, rel, self.crp_packagenumber)) + + q = "insert into %s values (?, ?, ?, ?, ?, ?)" % pco + cur.executemany(q, thispco) + + # requires + reqs = [] + for (name, flag, (epoch, ver, rel), pre) in self._requires_with_pre(): + if name.startswith('rpmlib('): + continue + pre_bool = 'FALSE' + if pre == 1: + pre_bool = 'TRUE' + reqs.append((name, flag, epoch, ver,rel, self.crp_packagenumber, pre_bool)) + q = "insert into requires values (?, ?, ?, ?, ?, ?, ?)" + cur.executemany(q, reqs) + + # files + p = [] + for f in self._return_primary_files(): + p.append((f,)) + + if p: + q = "insert into files values (?, 'file', %s)" % self.crp_packagenumber + cur.executemany(q, p) + + # dirs + p = [] + for f in self._return_primary_dirs(): + p.append((f,)) + if p: + q = "insert into files values (?, 'dir', %s)" % self.crp_packagenumber + cur.executemany(q, p) + + + # ghosts + p = [] + for f in self._return_primary_files(list_of_files = self.returnFileEntries('ghost')): + p.append((f,)) + if p: + q = "insert into files values (?, 'ghost', %s)" % self.crp_packagenumber + cur.executemany(q, p) + + + + def do_filelists_sqlite_dump(self, cur): + """inserts filelists data in place, this assumes the tables exist""" + # insert packagenumber + checksum into 'packages' table + q = 'insert into packages values (?, ?)' + p = (self.crp_packagenumber, self.checksum) + + cur.execute(q, p) + + # break up filelists and encode them + dirs = {} + for (filetype, files) in [('file', self.filelist), ('dir', self.dirlist), + ('ghost', self.ghostlist)]: + for filename in files: + (dirname,filename) = (os.path.split(filename)) + if not dirs.has_key(dirname): + dirs[dirname] = {'files':[], 'types':[]} + dirs[dirname]['files'].append(filename) + dirs[dirname]['types'].append(filetype) + + # insert packagenumber|dir|files|types into files table + p = [] + for (dirname,direc) in dirs.items(): + p.append((self.crp_packagenumber, dirname, + utils.encodefilenamelist(direc['files']), + utils.encodefiletypelist(direc['types']))) + if p: + q = 'insert into filelist values (?, ?, ?, ?)' + cur.executemany(q, p) + + + def do_other_sqlite_dump(self, cur): + """inserts changelog data in place, this assumes the tables exist""" + # insert packagenumber + checksum into 'packages' table + q = 'insert into packages values (?, ?)' + p = (self.crp_packagenumber, self.checksum) + + cur.execute(q, p) + + if self.changelog: + q = 'insert into changelog ("pkgKey", "date", "author", "changelog") values (%s, ?, ?, ?)' % self.crp_packagenumber + cur.executemany(q, self.changelog) + + + def do_sqlite_dump(self, md_sqlite): + """write the metadata out to the sqlite dbs""" + self.do_primary_sqlite_dump(md_sqlite.primary_cursor) + md_sqlite.pri_cx.commit() + self.do_filelists_sqlite_dump(md_sqlite.filelists_cursor) + md_sqlite.file_cx.commit() + self.do_other_sqlite_dump(md_sqlite.other_cursor) + md_sqlite.other_cx.commit() @@ -0,0 +1,156 @@ +#!/usr/bin/python + +# dmd - Generate and apply deltas between repository metadata +# +# Copyright (C) 2007 James Bowes <jbowes@redhat.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +import sys +from lxml.etree import parse, tostring, Element + + +class MdType(object): + def __init__(self, namespace, rootelem): + self.ns = "http://linux.duke.edu/metadata/%s" % namespace + self.sns = "{%s}" % self.ns + self.deltasns = "{http://linux.duke.edu/metadata/delta}" + self.root = rootelem + + def get_pkg_id(self, pkg): + return pkg.findtext(self.sns + "checksum") + + def make_hash(self, tree): + pkgshash = {} + for pkg in tree: + pkgid = self.get_pkg_id(pkg) + pkgshash[pkgid] = pkg + + return pkgshash + + def make_pkg_elem(self, pkgid, pkg): + pkgelem = Element("package") + pkgelem.set('name', pkg.findtext(self.sns + 'name')) + pkgelem.set('arch', pkg.findtext(self.sns + 'arch')) + pkgelem.set('pkgid', pkgid) + verelem = pkg.find(self.sns + 'version') + verelem.tag = "version" + pkgelem.append(verelem) + + return pkgelem + + def diff_trees(self, oldtree, newtree): + oldpkgs = oldtree.getroot().getchildren() + newpkgs = newtree.getroot().getchildren() + + oldpkgshash = self.make_hash(oldpkgs) + newpkgshash = self.make_hash(newpkgs) + + diff = Element(self.root, + nsmap = {None : self.ns, + "rpm" : "http://linux.duke.edu/metadata/rpm", + "delta" : "http://linux.duke.edu/metadata/delta"}) + additions = Element("delta:additions") + diff.append(additions) + removals = Element("delta:removals") + + diff.append(removals) + + for pkgid, pkg in newpkgshash.iteritems(): + if not oldpkgshash.has_key(pkgid): + additions.append(pkg) + + for pkgid, pkg in oldpkgshash.iteritems(): + if not newpkgshash.has_key(pkgid): + pkgelem = self.make_pkg_elem(pkgid, pkg) + removals.append(pkgelem) + + diff.set("packages", str(len(removals) + len(additions))) + + print tostring(diff, pretty_print=True) + + def patch_tree(self, oldtree, deltatree): + oldroot = oldtree.getroot() + oldpkgs = oldroot.getchildren() + + oldpkgshash = self.make_hash(oldpkgs) + + additions = deltatree.find(self.deltasns + 'additions').getchildren() + removals = deltatree.find(self.deltasns + 'removals').getchildren() + + for pkg in additions: + pkgid = self.get_pkg_id(pkg) + if oldpkgshash.has_key(pkgid): + print >> sys.stderr, "Package %s already exists" % pkgid + sys.exit(1) + oldroot.append(pkg) + + for pkg in removals: + pkgid = pkg.get('pkgid') + if not oldpkgshash.has_key(pkgid): + print >> sys.stderr, "Package %s does not exist" % pkgid + sys.exit(1) + oldroot.remove(oldpkgshash[pkgid]) + + oldcount = int(oldroot.get('packages')) + newcount = oldcount + len(additions) - len(removals) + oldroot.set('packages', str(newcount)) + print tostring(oldtree, pretty_print=True) + + +class OtherMdType(MdType): + def get_pkg_id(self, pkg): + return pkg.get('pkgid') + + def make_pkg_elem(self, pkgid, pkg): + pkgelem = Element("package") + pkgelem.set('name', pkg.get('name')) + pkgelem.set('arch', pkg.get('arch')) + pkgelem.set('pkgid', pkgid) + verelem = pkg.find(self.sns + 'version') + verelem.tag = "version" + + return pkgelem + + +mdtypeinfo = { + 'primary' : MdType('common', 'metadata'), + 'filelists' : OtherMdType('filelists', 'filelists'), + 'other' : OtherMdType('other', 'other'), + } + + +def usage(progname): + print "usage: %s [diff|patch] MDTYPE FILE1 FILE2" % progname + sys.exit() + +def main(args): + if len(args) != 5: + usage(args[0]) + if args[1] not in ('diff', 'patch'): + usage(args[0]) + if args[2] not in ('primary', 'filelists', 'other'): + usage(args[0]) + + oldtree = parse(args[3]) + newtree = parse(args[4]) + + if args[1] == 'diff': + mdtypeinfo[args[2]].diff_trees(oldtree, newtree) + else: + mdtypeinfo[args[2]].patch_tree(oldtree, newtree) + +if __name__ == "__main__": + main(sys.argv) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..2e70622 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,99 @@ +SHELL = /bin/sh +top_srcdir = .. +srcdir = ../docs +prefix = /usr +exec_prefix = ${prefix} + +bindir = ${exec_prefix}/bin +sbindir = ${exec_prefix}/sbin +libexecdir = ${exec_prefix}/libexec +datadir = ${prefix}/share +sysconfdir = ${prefix}/etc +sharedstatedir = ${prefix}/com +localstatedir = ${prefix}/var +libdir = ${exec_prefix}/lib +infodir = ${prefix}/info +docdir = +includedir = ${prefix}/include +oldincludedir = /usr/include +mandir = ${datadir}/man + +pkgdatadir = $(datadir)/$(PKGNAME) +pkglibdir = $(libdir)/$(PKGNAME) +pkgincludedir = $(includedir)/$(PKGNAME) +top_builddir = ../ + +# all dirs +DIRS = $(DESTDIR)$(bindir) $(DESTDIR)/etc $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(mandir) + + +# INSTALL scripts +INSTALL = install -p --verbose +INSTALL_BIN = $(INSTALL) -m 755 +INSTALL_DIR = $(INSTALL) -m 755 -d +INSTALL_DATA = $(INSTALL) -m 644 +INSTALL_MODULES = $(INSTALL) -m 755 -D +RM = rm -f + + +all: + echo "nothing to do" + +install: all installdirs + mkdir -p $(DESTDIR)$(mandir)/man8 + mkdir -p $(DESTDIR)$(mandir)/man1 + $(INSTALL_DATA) createrepo.8 $(DESTDIR)$(mandir)/man8/createrepo.8 + $(INSTALL_DATA) modifyrepo.1 $(DESTDIR)$(mandir)/man1/modifyrepo.1 + $(INSTALL_DATA) mergerepo.1 $(DESTDIR)$(mandir)/man1/mergerepo.1 + + +uninstall: + $(RM) $(bindir)/$(PKGNAME) + + + +clean: + + +distclean: + $(RM) -rf .libs + $(RM) -f core + $(RM) -f *~ + + +mostlyclean: + $(MAKE) clean + + +maintainer-clean: + $(MAKE) distclean + + +distfiles: + distdir=$(PKGNAME)-$(VERSION); \ + mkdir $(top_srcdir)/.disttmp/$$distdir/docs;\ + cp \ + $(srcdir)/createrepo.8 \ + $(srcdir)/modifyrepo.1 \ + $(srcdir)/mergerepo.1 \ + $(srcdir)/Makefile \ + $(top_srcdir)/.disttmp/$$distdir/docs + +dailyfiles: + distdir=$(PKGNAME); \ + mkdir $(top_srcdir)/.disttmp/$$distdir/docs;\ + cp \ + $(srcdir)/createrepo.8 \ + $(srcdir)/modifyrepo.1 \ + $(srcdir)/mergerepo.1 \ + $(srcdir)/Makefile \ + $(top_srcdir)/.disttmp/$$distdir/docs + +installdirs: + $(MAKE) -C $(top_srcdir) installdirs + + +.PHONY: all install install-strip uninstall clean distclean mostlyclean maintainer-clean info dvi dist distfiles check installcheck installdirs dailyfiles + + + diff --git a/docs/createrepo.8 b/docs/createrepo.8 new file mode 100644 index 0000000..e3c4c3b --- /dev/null +++ b/docs/createrepo.8 @@ -0,0 +1,139 @@ +.TH "createrepo" "8" "2005 Jan 2" "Seth Vidal" "" + +.SH "NAME" +createrepo \- Create repomd (xml-rpm-metadata) repository + +.SH "SYNOPSIS" +\fBcreaterepo\fP [options] <directory> +.PP + +.SH "DESCRIPTION" +\fBcreaterepo\fP is a program that creates a repomd (xml-based rpm metadata) repository from a set of rpms. + +.SH "OPTIONS" +.IP "\fB\-u --baseurl\fP <url>" +Optional base URL location for all files. +.IP "\fB\-o --outputdir\fP <url>" +Optional output directory (useful for read only media). +.IP "\fB\-x --excludes\fP <package>" +File globs to exclude, can be specified multiple times. +.IP "\fB\-i --pkglist\fP <filename>" +specify a text file which contains the complete list of files to +include in the repository from the set found in the directory. File format is one +package per line, no wildcards or globs. +.IP "\fB\-n --includepkg\fP" +specify pkgs to include on the command line. Takes urls as well as local paths. +.IP "\fB\-q --quiet\fP" +Run quietly. +.IP "\fB\-g --groupfile\fP <groupfile>" +A precreated xml filename to point to for group information. +.br +See examples section below for further explanation. +.IP "\fB\-v --verbose\fP" +Run verbosely. +.IP "\fB\-c --cachedir\fP <path>" +Specify a directory to use as a cachedir. This allows createrepo to create a +cache of checksums of packages in the repository. In consecutive runs of +createrepo over the same repository of files that do not have a complete +change out of all packages this decreases the processing time dramatically. +.br +.IP "\fB\--update\fP" +If metadata already exists in the outputdir and an rpm is unchanged +(based on file size and mtime) since the metadata was generated, reuse +the existing metadata rather than recalculating it. In the case of a +large repository with only a few new or modified rpms this can +significantly reduce I/O and processing time. +.br +.IP "\fB\--skip-stat\fP" +skip the stat() call on a --update, assumes if the filename is the same +then the file is still the same (only use this if you're fairly trusting or +gullible). +.br +.IP "\fB\-C --checkts\fP" +Don't generate repo metadata, if their timestamps are newer than its rpms. +This option decreases the processing time drastically again, if you happen +to run it on an unmodified repo, but it is (currently) mutual exclusive +with the --split option. +.br +.IP "\fB\--split\fP" +Run in split media mode. Rather than pass a single directory, take a set of +directories corresponding to different volumes in a media set. +.br +.IP "\fB\-p --pretty\fP" +Output xml files in pretty format. +.IP "\fB\-V --version\fP" +Output version. +.IP "\fB\-h --help\fP" +Show help menu. + +.IP "\fB\-d --database\fP" +Generate sqlite databases for use with yum. This is now the default. + +.IP "\fB\--no-database\fP" +Do not generate sqlite databases in the repository. + +.IP "\fB\-S --skip-symlinks\fP" +Ignore symlinks of packages +.IP "\fB\-s --checksum\fP" +Choose the checksum type used in repomd.xml and for packages in the metadata. +The default is now "sha256" (if python has hashlib). The older default was +"sha", which is actually "sha1", however explicitly using "sha1" doesn't work +on older (3.0.x) versions of yum, you need to specify "sha". +.IP "\fB\--profile\fP" +Output time based profiling information. +.IP "\fB\--changelog-limit\fP CHANGELOG_LIMIT" +Only import the last N changelog entries, from each rpm, into the metadata +.IP "\fB\--unique-md-filenames\fP" +Include the file's checksum in the metadata filename, helps HTTP caching (default) + +.IP "\fB\--simple-md-filenames\fP" +Do not include the file's checksum in the metadata filename. + +.IP "\fB\--distro\fP" +Specify distro tags. Can be specified more than once. Optional syntax specifying a +cpeid(http://cpe.mitre.org/) --distro=cpeid,distrotag +.IP "\fB\--content\fP" +Specify keyword/tags about the content of the repository. Can be specified more than once. +.IP "\fB\--repo\fP" +Specify keyword/tags about the repository itself. Can be specified more than once. +.IP "\fB\--revision\fP" +Arbitrary string for a repository revision. +.IP "\fB\--deltas\fP" +Tells createrepo to generate deltarpms and the delta metadata +.IP "\fB\--oldpackagedirs\fP PATH" +paths to look for older pkgs to delta against. Can be specified multiple times +.IP "\fB\--num-deltas\fP int" +the number of older versions to make deltas against. Defaults to 1 + + +.SH "EXAMPLES" +Here is an example of a repository with a groups file. Note that the +groups file should be in the same directory as the rpm packages +(i.e. /path/to/rpms/comps.xml). +.br +.PP +\fBcreaterepo\fP \-g comps.xml /path/to/rpms + +.SH "FILES" +.nf +repodata/filelists.xml.gz +repodata/other.xml.gz +repodata/primary.xml.gz +repodata/repomd.xml +.fi +.PP +.SH "SEE ALSO" +.I yum (8) yum.conf (5) + +.PP +.SH "AUTHORS" +.nf +See the Authors file +.fi + +.PP +.SH "BUGS" +Any bugs which are found should be emailed to the mailing list: +rpm-metadata@lists.baseurl.org +or reported in trac at: http://createrepo.baseurl.org +.fi diff --git a/docs/mergerepo.1 b/docs/mergerepo.1 new file mode 100644 index 0000000..2529e7a --- /dev/null +++ b/docs/mergerepo.1 @@ -0,0 +1,56 @@ +.TH "mergerepo" "1" "2008 Oct 21" "Seth Vidal" "" + +.SH "NAME" +mergerepo \- Merge multiple repositories together + +.SH "SYNOPSIS" +\fBmergerepo\fP --repo repo1 --repo repo2 +.PP + +.SH "DESCRIPTION" +\fBmergerepo\fP is a program that allows you merge multiple repositories +into a single repository while referring to the remote location for all +packages. + +.SH "OPTIONS" +.IP "\fB\-r --repo\fP <url>" +Url to a repository to be merged. + +.IP "\fB\-o --outputdir <directory>\fP" +Path where merged repository metadata should be written to. If not specified +repository metadata will be written to `pwd`/merged_repo/. + +.IP "\fB\-d --database\fP" +Generate sqlite databases of the merged repository metadata. + +.IP "\fB\-a --archlist\fP" +Specify a comma-separated list of architectures to use. Defaults to ALL. + +.IP "\fB\--nogroups\fP" +Do not merge/include groups metadata in the repository. + +.IP "\fB\--noupdateinfo\fP" +Do not merge/include updateinfo metadata in the repository. + + +.SH "EXAMPLES" +.PP +$ \fBmergerepo\fP --repo=http://myurl.org/repo1 --repo=http://myurl.org/repo2 -d -o /tmp/mymergedrepo + +.PP +.SH "SEE ALSO" +.I createrepo (8) + +.PP +.SH "AUTHORS" +.nf +Seth Vidal <skvidal@fedoraproject.org> +.fi + +.PP +.SH "BUGS" +Any bugs which are found should be emailed to the mailing list: +rpm-metadata@lists.baseurl.org or filed as tickets at: +http://createrepo.baseurl.org/ + +.fi diff --git a/docs/modifyrepo.1 b/docs/modifyrepo.1 new file mode 100644 index 0000000..cc031f5 --- /dev/null +++ b/docs/modifyrepo.1 @@ -0,0 +1,41 @@ +.TH "modifyrepo" "1" "2007 Dec 3" "Luke Macken" "" + +.SH "NAME" +modifyrepo \- Modify a repomd (xml-rpm-metadata) repository + +.SH "SYNOPSIS" +\fBmodifyrepo\fP [options] <input metadata> <output repodata> +.PP + +.SH "DESCRIPTION" +\fBmodifyrepo\fP is a program that allows you to insert arbitrary metadata into a repomd (xml-based rpm metadata) repository. + +.SH "EXAMPLES" +.PP +$ \fBmodifyrepo\fP --mdtype=newmd metadata.xml /repository/repodata +.br +Wrote: /repository/repodata/metadata.xml.gz + type = newmd + location = repodata/metadata.xml.gz + checksum = 1d7ee93db2964e7f85e07ec19b3204591da1050c + timestamp = 1196716296.0 + open-checksum = 824d936dc7dfff029379797b311af0cc66af4115 +.br +Wrote: /repository/repodata/repomd.xml + +.PP +.SH "SEE ALSO" +.I createrepo (8) + +.PP +.SH "AUTHORS" +.nf +Luke Macken <lmacken@redhat.com> +Seth Vidal <skvidal@fedoraproject.org> +.fi + +.PP +.SH "BUGS" +Any bugs which are found should be emailed to the mailing list: +rpm-metadata@lists.baseurl.org +.fi diff --git a/genpkgmetadata.py b/genpkgmetadata.py new file mode 100755 index 0000000..8c98191 --- /dev/null +++ b/genpkgmetadata.py @@ -0,0 +1,277 @@ +#!/usr/bin/python -t +# primary functions and glue for generating the repository metadata +# + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2004 Duke University +# Portions Copyright 2009 Red Hat, Inc - +# written by seth vidal skvidal at fedoraproject.org + +import os +import sys +import re +from optparse import OptionParser +import time + +import createrepo +from createrepo import MDError +from createrepo.utils import errorprint, _ +import yum.misc + + +def parse_args(args, conf): + """ + Parse the command line args. return a config object. + Sanity check all the things being passed in. + """ + + _def = yum.misc._default_checksums[0] + _avail = yum.misc._available_checksums + parser = OptionParser(version = "createrepo %s" % createrepo.__version__) + # query options + parser.add_option("-q", "--quiet", default=False, action="store_true", + help="output nothing except for serious errors") + parser.add_option("-v", "--verbose", default=False, action="store_true", + help="output more debugging info.") + parser.add_option("--profile", default=False, action="store_true", + help="output timing/profile info.") + parser.add_option("-x", "--excludes", default=[], action="append", + help="files to exclude") + parser.add_option("--basedir", default=os.getcwd(), + help="basedir for path to directories") + parser.add_option("-u", "--baseurl", default=None, + help="baseurl to append on all files") + parser.add_option("-g", "--groupfile", default=None, + help="path to groupfile to include in metadata") + parser.add_option("-s", "--checksum", default=_def, dest='sumtype', + help="specify the checksum type to use (default: %s)" % _def) + parser.add_option("-p", "--pretty", default=False, action="store_true", + help="make sure all xml generated is formatted") + parser.add_option("-c", "--cachedir", default=None, + help="set path to cache dir") + parser.add_option("-C", "--checkts", default=False, action="store_true", + help="check timestamps on files vs the metadata to see " \ + "if we need to update") + parser.add_option("-d", "--database", default=True, action="store_true", + help="create sqlite database files: now default, see --no-database to disable") + parser.add_option("--no-database", default=False, dest="nodatabase", action="store_true", + help="do not create sqlite dbs of metadata") + # temporarily disabled + #parser.add_option("--database-only", default=False, action="store_true", + # dest='database_only', + # help="Only make the sqlite databases - does not work with --update, yet") + parser.add_option("--update", default=False, action="store_true", + help="use the existing repodata to speed up creation of new") + parser.add_option("--update-md-path", default=None, dest='update_md_path', + help="use the existing repodata for --update from this path") + parser.add_option("--skip-stat", dest='skip_stat', default=False, + help="skip the stat() call on a --update, assumes if the file" \ + "name is the same then the file is still the same " \ + "(only use this if you're fairly trusting or gullible)", + action="store_true") + parser.add_option("--split", default=False, action="store_true", + help="generate split media") + parser.add_option("-i", "--pkglist", default=None, + help="use only the files listed in this file from the " \ + "directory specified") + parser.add_option("-n", "--includepkg", default=[], action="append", + help="add this pkg to the list - can be specified multiple times") + parser.add_option("-o", "--outputdir", default=None, + help="<dir> = optional directory to output to") + parser.add_option("-S", "--skip-symlinks", dest="skip_symlinks", + default=False, action="store_true", help="ignore symlinks of packages") + parser.add_option("--changelog-limit", dest="changelog_limit", + default=None, help="only import the last N changelog entries") + parser.add_option("--unique-md-filenames", dest="unique_md_filenames", + help="include the file's checksum in the filename, helps with proxies", + default=True, action="store_true") + parser.add_option("--simple-md-filenames", dest="simple_md_filenames", + help="do not include the file's checksum in the filename, helps with proxies", + default=False, action="store_true") + parser.add_option("--distro", default=[], action="append", + help="distro tag and optional cpeid: --distro" "'cpeid,textname'") + parser.add_option("--content", default=[], dest='content_tags', + action="append", help="tags for the content in the repository") + parser.add_option("--repo", default=[], dest='repo_tags', + action="append", help="tags to describe the repository itself") + parser.add_option("--revision", default=None, + help="user-specified revision for this repository") + parser.add_option("--deltas", default=False, action="store_true", + help="create delta rpms and metadata") + parser.add_option("--oldpackagedirs", default=[], dest="oldpackage_paths", + action="append", help="paths to look for older pkgs to delta against") + parser.add_option("--num-deltas", default=1, dest='num_deltas', type='int', + help="the number of older versions to make deltas against") + parser.add_option("--read-pkgs-list", default=None, dest='read_pkgs_list', + help="output the paths to the pkgs actually read useful with --update") + parser.add_option("--max-delta-rpm-size", default=100000000, + dest='max_delta_rpm_size', type='int', + help="max size of an rpm that to run deltarpm against (in bytes)") + + parser.add_option("--workers", default=1, + dest='workers', type='int', + help="number of workers to spawn to read rpms") + + (opts, argsleft) = parser.parse_args(args) + if len(argsleft) > 1 and not opts.split: + errorprint(_('Error: Only one directory allowed per run.')) + parser.print_usage() + sys.exit(1) + + elif len(argsleft) == 0: + errorprint(_('Error: Must specify a directory to index.')) + parser.print_usage() + sys.exit(1) + + else: + directories = argsleft + + if opts.sumtype == 'sha1': + errorprint(_('Warning: It is more compatible to use sha instead of sha1')) + + if opts.sumtype != 'sha' and opts.sumtype not in _avail: + errorprint(_('Error: Checksum %s not available (sha, %s)') % + (opts.sumtype, ", ".join(sorted(_avail)))) + sys.exit(1) + + if opts.split and opts.checkts: + errorprint(_('--split and --checkts options are mutually exclusive')) + sys.exit(1) + + if opts.simple_md_filenames: + opts.unique_md_filenames = False + + if opts.nodatabase: + opts.database = False + + # let's switch over to using the conf object - put all the opts into it + for opt in parser.option_list: + if opt.dest is None: # this is fairly silly + continue + # if it's not set, take the default from the base class + if getattr(opts, opt.dest) is None: + continue + setattr(conf, opt.dest, getattr(opts, opt.dest)) + + directory = directories[0] + conf.directory = directory + conf.directories = directories + + # distro tag parsing + + for spec in opts.distro: + if spec.find(',') == -1: + conf.distro_tags.append((None, spec)) + else: + splitspec = spec.split(',') + conf.distro_tags.append((splitspec[0], splitspec[1])) + + lst = [] + if conf.pkglist: + pfo = open(conf.pkglist, 'r') + for line in pfo.readlines(): + line = line.strip() + if re.match('^\s*\#.*', line) or re.match('^\s*$', line): + continue + lst.append(line) + pfo.close() + + conf.pkglist = lst + + if conf.includepkg: + conf.pkglist.extend(conf.includepkg) + + if conf.changelog_limit: # make sure it is an int, not a string + conf.changelog_limit = int(conf.changelog_limit) + + return conf + +class MDCallBack(object): + """cli callback object for createrepo""" + def __init__(self): + self.__show_progress = os.isatty(1) + + def errorlog(self, thing): + """error log output""" + print >> sys.stderr, thing + + def log(self, thing): + """log output""" + print thing + + def progress(self, item, current, total): + """progress bar""" + + if not self.__show_progress: + return + beg = "%*d/%d - " % (len(str(total)), current, total) + left = 80 - len(beg) + sys.stdout.write("\r%s%-*.*s" % (beg, left, left, item)) + sys.stdout.flush() + +def main(args): + """createrepo from cli main flow""" + start_st = time.time() + conf = createrepo.MetaDataConfig() + conf = parse_args(args, conf) + if conf.profile: + print ('start time: %0.3f' % (time.time() - start_st)) + + mid_st = time.time() + try: + if conf.split: + mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, + callback=MDCallBack()) + else: + mdgen = createrepo.MetaDataGenerator(config_obj=conf, + callback=MDCallBack()) + if mdgen.checkTimeStamps(): + if mdgen.conf.verbose: + print _('repo is up to date') + sys.exit(0) + + if conf.profile: + print ('mid time: %0.3f' % (time.time() - mid_st)) + + pm_st = time.time() + mdgen.doPkgMetadata() + if conf.profile: + print ('pm time: %0.3f' % (time.time() - pm_st)) + rm_st = time.time() + mdgen.doRepoMetadata() + if conf.profile: + print ('rm time: %0.3f' % (time.time() - rm_st)) + fm_st = time.time() + mdgen.doFinalMove() + if conf.profile: + print ('fm time: %0.3f' % (time.time() - fm_st)) + + + except MDError, errormsg: + errorprint(_('%s') % errormsg) + sys.exit(1) + + +if __name__ == "__main__": + if len(sys.argv) > 1: + if sys.argv[1] == 'profile': + import hotshot + p = hotshot.Profile(os.path.expanduser("~/createrepo.prof")) + p.run('main(sys.argv[2:])') + p.close() + else: + main(sys.argv[1:]) + else: + main(sys.argv[1:]) diff --git a/mergerepo.py b/mergerepo.py new file mode 100755 index 0000000..05e5f5e --- /dev/null +++ b/mergerepo.py @@ -0,0 +1,85 @@ +#!/usr/bin/python -tt +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2008 Red Hat, Inc - written by skvidal at fedoraproject.org + +# merge repos from arbitrary repo urls + +import sys +import createrepo.merge +from optparse import OptionParser + +#TODO: +# excludes? +# handle content/distro tags +# support revision? + + +def parse_args(args): + """Parse our opts/args""" + usage = """ + mergerepo: take 2 or more repositories and merge their metadata into a new repo + + mergerepo --repo=url --repo=url --outputdir=/some/path""" + + parser = OptionParser(version = "mergerepo 0.1", usage=usage) + # query options + parser.add_option("-r", "--repo", dest='repos', default=[], action="append", + help="repo url") + parser.add_option("-a", "--archlist", default=[], action="append", + help="Defaults to all arches - otherwise specify arches") + parser.add_option("-d", "--database", default=True, action="store_true") + parser.add_option( "--no-database", default=False, action="store_true", dest="nodatabase") + parser.add_option("-o", "--outputdir", default=None, + help="Location to create the repository") + parser.add_option("", "--nogroups", default=False, action="store_true", + help="Do not merge group(comps) metadata") + parser.add_option("", "--noupdateinfo", default=False, action="store_true", + help="Do not merge updateinfo metadata") + (opts, argsleft) = parser.parse_args(args) + + if len(opts.repos) < 2: + parser.print_usage() + sys.exit(1) + + # sort out the comma-separated crap we somehow inherited. + archlist = [] + for archs in opts.archlist: + for arch in archs.split(','): + archlist.append(arch) + + opts.archlist = archlist + + return opts + +def main(args): + """main""" + opts = parse_args(args) + rmbase = createrepo.merge.RepoMergeBase(opts.repos) + if opts.archlist: + rmbase.archlist = opts.archlist + if opts.outputdir: + rmbase.outputdir = opts.outputdir + if opts.nodatabase: + rmbase.mdconf.database = False + if opts.nogroups: + rmbase.groups = False + if opts.noupdateinfo: + rmbase.updateinfo = False + + rmbase.merge_repos() + rmbase.write_metadata() + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/modifyrepo.py b/modifyrepo.py new file mode 100755 index 0000000..17094a4 --- /dev/null +++ b/modifyrepo.py @@ -0,0 +1,148 @@ +#!/usr/bin/python +# This tools is used to insert arbitrary metadata into an RPM repository. +# Example: +# ./modifyrepo.py updateinfo.xml myrepo/repodata +# or in Python: +# >>> from modifyrepo import RepoMetadata +# >>> repomd = RepoMetadata('myrepo/repodata') +# >>> repomd.add('updateinfo.xml') +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# (C) Copyright 2006 Red Hat, Inc. +# Luke Macken <lmacken@redhat.com> +# modified by Seth Vidal 2008 + +import os +import sys +from createrepo import __version__ +from createrepo.utils import checksum_and_rename, GzipFile, MDError +from yum.misc import checksum + +from yum.repoMDObject import RepoMD, RepoMDError, RepoData +from xml.dom import minidom +from optparse import OptionParser + + +class RepoMetadata: + + def __init__(self, repo): + """ Parses the repomd.xml file existing in the given repo directory. """ + self.repodir = os.path.abspath(repo) + self.repomdxml = os.path.join(self.repodir, 'repomd.xml') + self.checksum_type = 'sha256' + + if not os.path.exists(self.repomdxml): + raise MDError, '%s not found' % self.repomdxml + + try: + self.repoobj = RepoMD(self.repodir) + self.repoobj.parse(self.repomdxml) + except RepoMDError, e: + raise MDError, 'Could not parse %s' % self.repomdxml + + + def add(self, metadata, mdtype=None): + """ Insert arbitrary metadata into this repository. + metadata can be either an xml.dom.minidom.Document object, or + a filename. + """ + md = None + if not metadata: + raise MDError, 'metadata cannot be None' + if isinstance(metadata, minidom.Document): + md = metadata.toxml() + mdname = 'updateinfo.xml' + elif isinstance(metadata, str): + if os.path.exists(metadata): + if metadata.endswith('.gz'): + oldmd = GzipFile(filename=metadata, mode='rb') + else: + oldmd = file(metadata, 'r') + md = oldmd.read() + oldmd.close() + mdname = os.path.basename(metadata) + else: + raise MDError, '%s not found' % metadata + else: + raise MDError, 'invalid metadata type' + + ## Compress the metadata and move it into the repodata + if not mdname.endswith('.gz'): + mdname += '.gz' + if not mdtype: + mdtype = mdname.split('.')[0] + + destmd = os.path.join(self.repodir, mdname) + newmd = GzipFile(filename=destmd, mode='wb') + newmd.write(md) + newmd.close() + print "Wrote:", destmd + + open_csum = checksum(self.checksum_type, metadata) + csum, destmd = checksum_and_rename(destmd, self.checksum_type) + base_destmd = os.path.basename(destmd) + + + ## Remove any stale metadata + if mdtype in self.repoobj.repoData: + del self.repoobj.repoData[mdtype] + + + new_rd = RepoData() + new_rd.type = mdtype + new_rd.location = (None, 'repodata/' + base_destmd) + new_rd.checksum = (self.checksum_type, csum) + new_rd.openchecksum = (self.checksum_type, open_csum) + new_rd.size = str(os.stat(destmd).st_size) + new_rd.timestamp = str(os.stat(destmd).st_mtime) + self.repoobj.repoData[new_rd.type] = new_rd + + print " type =", new_rd.type + print " location =", new_rd.location[1] + print " checksum =", new_rd.checksum[1] + print " timestamp =", new_rd.timestamp + print " open-checksum =", new_rd.openchecksum[1] + + ## Write the updated repomd.xml + outmd = file(self.repomdxml, 'w') + outmd.write(self.repoobj.dump_xml()) + outmd.close() + print "Wrote:", self.repomdxml + + +def main(args): + parser = OptionParser(version='modifyrepo version %s' % __version__) + # query options + parser.add_option("--mdtype", dest='mdtype', + help="specific datatype of the metadata, will be derived from the filename if not specified") + parser.usage = "modifyrepo [options] <input_metadata> <output repodata>" + + (opts, argsleft) = parser.parse_args(args) + if len(argsleft) != 2: + parser.print_usage() + return 0 + metadata = argsleft[0] + repodir = argsleft[1] + try: + repomd = RepoMetadata(repodir) + except MDError, e: + print "Could not access repository: %s" % str(e) + return 1 + try: + repomd.add(metadata, mdtype=opts.mdtype) + except MDError, e: + print "Could not add metadata from file %s: %s" % (metadata, str(e)) + return 1 + +if __name__ == '__main__': + ret = main(sys.argv[1:]) + sys.exit(ret) diff --git a/worker.py b/worker.py new file mode 100755 index 0000000..eb35ef7 --- /dev/null +++ b/worker.py @@ -0,0 +1,99 @@ +#!/usr/bin/python -tt + +import sys +import yum +import createrepo +import os +import rpmUtils +from optparse import OptionParser + + +# pass in dir to make tempdirs in +# make tempdir for this worker +# create 3 files in that tempdir +# return how many pkgs +# return on stderr where things went to hell + +#TODO - take most of read_in_package from createrepo and duplicate it here +# so we can do downloads, etc. +# then replace callers of read_in_package with forked callers of this +# and reassemble at the end + +def main(args): + parser = OptionParser() + parser.add_option('--tmpmdpath', default=None, + help="path where the outputs should be dumped for this worker") + parser.add_option("--pkgoptions", default=[], action='append', + help="pkgoptions in the format of key=value") + parser.add_option("--quiet", default=False, action='store_true', + help="only output errors and a total") + parser.add_option("--verbose", default=False, action='store_true', + help="output errors and a total") + parser.add_option("--globalopts", default=[], action='append', + help="general options in the format of key=value") + + + opts, pkgs = parser.parse_args(args) + external_data = {'_packagenumber': 1} + globalopts = {} + if not opts.tmpmdpath: + print >> sys.stderr, "tmpmdpath required for destination files" + sys.exit(1) + + + for strs in opts.pkgoptions: + k,v = strs.split('=') + if v in ['True', 'true', 'yes', '1', 1]: + v = True + elif v in ['False', 'false', 'no', '0', 0]: + v = False + elif v in ['None', 'none', '']: + v = None + external_data[k] = v + + for strs in opts.globalopts: + k,v = strs.split('=') + if v in ['True', 'true', 'yes', '1', 1]: + v = True + elif v in ['False', 'false', 'no', '0', 0]: + v = False + elif v in ['None', 'none', '']: + v = None + globalopts[k] = v + + + reldir = external_data['_reldir'] + ts = rpmUtils.transaction.initReadOnlyTransaction() + pri = open(opts.tmpmdpath + '/primary.xml' , 'w') + fl = open(opts.tmpmdpath + '/filelists.xml' , 'w') + other = open(opts.tmpmdpath + '/other.xml' , 'w') + + + for pkgfile in pkgs: + pkgpath = reldir + '/' + pkgfile + if not os.path.exists(pkgpath): + print >> sys.stderr, "File not found: %s" % pkgpath + continue + + try: + if not opts.quiet and opts.verbose: + print "reading %s" % (pkgfile) + + pkg = createrepo.yumbased.CreateRepoPackage(ts, package=pkgpath, + external_data=external_data) + pri.write(pkg.xml_dump_primary_metadata()) + fl.write(pkg.xml_dump_filelists_metadata()) + other.write(pkg.xml_dump_other_metadata(clog_limit= + globalopts.get('clog_limit', None))) + except yum.Errors.YumBaseError, e: + print >> sys.stderr, "Error: %s" % e + continue + else: + external_data['_packagenumber']+=1 + + pri.close() + fl.close() + other.close() + +if __name__ == "__main__": + main(sys.argv[1:]) |