Browse Source

Initial import

lmdb
Martchus 8 months ago
commit
c1554f4c87
  1. 48
      .gitignore
  2. 3
      .gitmodules
  3. 1
      3rdparty/tabulate
  4. 29
      CMakeLists.txt
  5. 340
      LICENSE
  6. 322
      README.md
  7. 46
      cli/CMakeLists.txt
  8. 14
      cli/doc/client-config-example.conf
  9. 322
      cli/main.cpp
  10. 1
      cli/tests/cppunit.cpp
  11. 103
      libpkg/CMakeLists.txt
  12. 157
      libpkg/algo/buildorder.cpp
  13. 335
      libpkg/algo/licenses.cpp
  14. 355
      libpkg/algo/search.cpp
  15. 160
      libpkg/data/config.cpp
  16. 185
      libpkg/data/config.h
  17. 492
      libpkg/data/database.cpp
  18. 217
      libpkg/data/database.h
  19. 7
      libpkg/data/lockable.cpp
  20. 36
      libpkg/data/lockable.h
  21. 523
      libpkg/data/package.cpp
  22. 417
      libpkg/data/package.h
  23. 93
      libpkg/data/siglevel.h
  24. 27
      libpkg/global.h
  25. 7
      libpkg/parser/aur.cpp
  26. 48
      libpkg/parser/aur.h
  27. 736
      libpkg/parser/binary.cpp
  28. 115
      libpkg/parser/binary.h
  29. 204
      libpkg/parser/config.cpp
  30. 12
      libpkg/parser/config.h
  31. 55
      libpkg/parser/database.cpp
  32. 10
      libpkg/parser/database.h
  33. 996
      libpkg/parser/package.cpp
  34. 10
      libpkg/parser/package.h
  35. 115
      libpkg/parser/siglevel.cpp
  36. 291
      libpkg/parser/utils.cpp
  37. 100
      libpkg/parser/utils.h
  38. 38
      libpkg/testfiles/c++utilities/PKGBUILD
  39. 39
      libpkg/testfiles/c++utilities/PKGBUILD.newepoch
  40. 38
      libpkg/testfiles/c++utilities/PKGBUILD.newpkgrel
  41. 39
      libpkg/testfiles/c++utilities/PKGBUILD.newpkgver
  42. 18
      libpkg/testfiles/c++utilities/SRCINFO
  43. BIN
      libpkg/testfiles/c++utilities/c++utilities.dll
  44. 48
      libpkg/testfiles/c++utilities/desc
  45. BIN
      libpkg/testfiles/c++utilities/libc++utilities.so.4.3.0
  46. BIN
      libpkg/testfiles/c++utilities/libc++utilities.so.4.5.0
  47. BIN
      libpkg/testfiles/cmake/ccmake
  48. BIN
      libpkg/testfiles/cmake/cmake
  49. BIN
      libpkg/testfiles/cmake/cmake-3.8.2-1-x86_64.pkg.tar.xz
  50. BIN
      libpkg/testfiles/cmake/cmake-gui
  51. BIN
      libpkg/testfiles/cmake/cpack
  52. BIN
      libpkg/testfiles/cmake/ctest
  53. BIN
      libpkg/testfiles/core.db
  54. BIN
      libpkg/testfiles/core.files
  55. 65
      libpkg/testfiles/linux-4.7.6-1-desc
  56. 4596
      libpkg/testfiles/linux-4.7.6-1-files
  57. 147
      libpkg/testfiles/makepkg.conf
  58. BIN
      libpkg/testfiles/mingw-w64-crt/libkernel32.a
  59. BIN
      libpkg/testfiles/mingw-w64-crt/mingw-w64-crt-6.0.0-1-any.pkg.tar.xz
  60. 148
      libpkg/testfiles/mingw-w64-harfbuzz/BUILDINFO
  61. 87
      libpkg/testfiles/mingw-w64-harfbuzz/PKGBUILD
  62. 21
      libpkg/testfiles/mingw-w64-harfbuzz/PKGINFO
  63. 30
      libpkg/testfiles/mingw-w64-harfbuzz/SRCINFO
  64. 54
      libpkg/testfiles/mingw-w64-harfbuzz/desc
  65. BIN
      libpkg/testfiles/mingw-w64-harfbuzz/mingw-w64-harfbuzz-1.4.2-1-any.pkg.tar.xz
  66. 461
      libpkg/testfiles/mirrorlist
  67. 100
      libpkg/testfiles/pacman.conf
  68. 14
      libpkg/testfiles/perl-data-dumper-concise/PKGBUILD
  69. 14
      libpkg/testfiles/perl-data-dumper-concise/PKGBUILD.newpkgrel
  70. BIN
      libpkg/testfiles/perl/perl-linux-desktopfiles-0.22-2-any.pkg.tar.xz
  71. BIN
      libpkg/testfiles/python/sphinxbase-5prealpha-7-i686.pkg.tar.xz
  72. 1
      libpkg/testfiles/repo/bar/cmake-3.8.2-1-x86_64.pkg.tar.xz
  73. 1
      libpkg/testfiles/repo/foo/fake-0-any.pkg.tar.zst
  74. 1
      libpkg/testfiles/repo/foo/mingw-w64-harfbuzz-1.4.2-1-any.pkg.tar.xz
  75. 1
      libpkg/testfiles/repo/foo/missing-0-any.pkg.tar.zst
  76. 1
      libpkg/testfiles/repo/foo/syncthingtray-0.6.2-1-x86_64.pkg.tar.xz
  77. BIN
      libpkg/testfiles/syncthingtray/libsyncthingconnector.so.0.6.2
  78. BIN
      libpkg/testfiles/syncthingtray/libsyncthingmodel.so.0.6.2
  79. BIN
      libpkg/testfiles/syncthingtray/libsyncthingwidgets.so.0.6.2
  80. BIN
      libpkg/testfiles/syncthingtray/syncthingctl
  81. BIN
      libpkg/testfiles/syncthingtray/syncthingtray
  82. BIN
      libpkg/testfiles/syncthingtray/syncthingtray-0.6.2-1-x86_64.pkg.tar.xz
  83. 1
      libpkg/tests/cppunit.cpp
  84. 484
      libpkg/tests/data.cpp
  85. 315
      libpkg/tests/parser.cpp
  86. 390
      libpkg/tests/parser_binary.cpp
  87. 120
      libpkg/tests/parser_helper.cpp
  88. 16
      libpkg/tests/parser_helper.h
  89. 90
      libpkg/tests/utils.cpp
  90. 170
      librepomgr/CMakeLists.txt
  91. 107
      librepomgr/authentication.cpp
  92. 30
      librepomgr/authentication.h
  93. 462
      librepomgr/buildactions/buildaction.cpp
  94. 363
      librepomgr/buildactions/buildaction.h
  95. 14
      librepomgr/buildactions/buildactionfwd.h
  96. 539
      librepomgr/buildactions/buildactionlivestreaming.cpp
  97. 366
      librepomgr/buildactions/buildactionmeta.cpp
  98. 175
      librepomgr/buildactions/buildactionmeta.h
  99. 606
      librepomgr/buildactions/buildactionprivate.h
  100. 12
      librepomgr/buildactions/buildactiontemplate.cpp

48
.gitignore

@ -0,0 +1,48 @@
# C++ objects and libs
*.slo
*.lo
*.o
*.a
*.la
*.lai
*.so
*.dll
*.dylib
# Qt-es
/.qmake.cache
/.qmake.stash
*.pro.user
*.txt.user
*.pro.user.*
*.qbs.user
*.qbs.user.*
*.moc
moc_*.cpp
qrc_*.cpp
ui_*.h
Makefile*
*-build-*
# QtCreator
*.autosave
#QtCtreator Qml
*.qmlproject.user
*.qmlproject.user.*
# Dolphin
.directory
# documentation
/doc
# clang-format
.clang-format
# node modules (only essential JavaScript and CSS files are force-added)
/srv/static/package-lock.json
/srv/static/node_modules/

3
.gitmodules

@ -0,0 +1,3 @@
[submodule "3rdparty/tabulate"]
path = 3rdparty/tabulate
url = git@github.com:p-ranav/tabulate.git

1
3rdparty/tabulate

@ -0,0 +1 @@
Subproject commit 67b010d095e866b651582dbfa0208010cde5afa0

29
CMakeLists.txt

@ -0,0 +1,29 @@
cmake_minimum_required(VERSION 3.1.0 FATAL_ERROR)
# metadata
set(META_PROJECT_NAME repomgr)
set(META_PROJECT_TYPE application)
set(META_APP_AUTHOR "Martchus")
set(META_APP_URL "https://github.com/${META_APP_AUTHOR}/${META_PROJECT_NAME}")
set(META_APP_DESCRIPTION "Repository manager and package builder for Arch Linux")
set(META_APP_CATEGORIES "System;Utility;Network;FileTransfer")
set(META_VERSION_MAJOR 0)
set(META_VERSION_MINOR 0)
set(META_VERSION_PATCH 1)
set(META_VERSION_EXACT_SONAME ON)
set(META_CXX_STANDARD 20)
project(${META_PROJECT_NAME})
enable_testing()
# add subdirectories
add_subdirectory(3rdparty/tabulate)
add_subdirectory(libpkg)
link_directories(${LIBPKG_BINARY_DIR})
add_subdirectory(librepomgr)
link_directories(${LIBREPOMGR_BINARY_DIR})
add_subdirectory(srv)
add_subdirectory(cli)
add_subdirectory(pacfind)

340
LICENSE

@ -0,0 +1,340 @@
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc., <http://fsf.org/>
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
{description}
Copyright (C) {year} {fullname}
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
{signature of Ty Coon}, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License.

322
README.md

@ -0,0 +1,322 @@
# Repository manager and build tool for Arch Linux
This **experimental** project contains **inofficial** tools to manage custom Arch Linux
repositories. It is built on top of the official tools provided by the `pacman` and
`devtools` packages.
So far it consists of:
* libpkg: C++ library to parse Arch Linux packages and databases
* parse pacman config
* parse databases
* extract and parse binary packages (.PKGINFO, dependencies on soname-level)
* parse package source infos (.SRCINFO)
* librepomgr: C++ library for managing custom Arch Linux repositories
* srv: Daemon and web application for building Arch Linux packages and managing custom Arch
Linux repositories
* build packages via `makechrootpkg` and add them to a custom repository via `repo-add`
* check AUR for package updates
* build from locally stored PKGBUILDs and from the AUR
* remove packages from a repository
* move packages from one repository to another
* check for repository issues (missing packages, package misses rebuild, …)
* provides an HTTP API and a web UI with live streaming
* pacfind: Tool to find the package containing a certain file
* requires `*.files`-databases to be present
* does not need the full file path and supports regex (in contrast to `pacman -F`)
* cli: Command line tool to interact with srv
* search for packages and show package details
* TODO: show and submit build actions
* distri: Tool to distribute applications from the packages in a repository
* TODO: bundle an application with its dependencies similar to `linuxdeployqt`
and `windeployqt`
## Setup server
An example config files can be found in this repository, see the `srv/doc`
directory.
### Setting up a working directory
The server needs a place to temporarily store PKGBUILDs, cache files and other stuff.
Just create a directory at any place with enough disk space and set the permissions so the
user you start the server with can read and write there. This directory goes under
`working_directory` in the settings file. Relative paths within the configuration file (used
for other locations) will be treated relative to that directory.
### Setting up a database/repository directories
The databases for the official repositories will be synced from the upstream repositories and
stored in a local directory. The actual packages might be found in the pacman cache directory.
Example configuration:
```
[database/core]
arch = x86_64
sync_from_mirror = on
dbdir = $sync_db_path/$arch
mirror = $default_mirror/$repo/os/$arch
```
Own repositories are not synced from a mirror and all packages are expected to be present in
a dedicated local directory. It makes most sense to store packages and databases in the same
directory. Example configuration:
```
[database/ownstuff]
path = $own_regular_db_path
files = $own_files_db_path
arch = x86_64
depends = core extra community multilib
pkgdir = $local_db_path/$repo/os/$arch
dbdir = $local_db_path/$repo/os/$arch
mirror = $local_mirror/$repo/os/$arch
```
The placeholder/variable values come from the previous "definitions" block, e.g.:
```
[definitions]
sync_db_path = sync-dbs
local_db_path = /run/media/repo/arch
own_sync_db_path = $local_db_path/$repo/os/$arch
own_regular_db_path = $local_db_path/$repo/os/$arch/$repo.db.tar.xz
own_files_db_path = $local_db_path/$repo/os/$arch/$repo.files.tar.xz
default_mirror = http://mirror.23media.de/archlinux
local_mirror = file://$local_db_path
```
The server obviously needs write permissions to add packages to repositories. In my
setup I've just add it as group and set permissions accordingly:
```
sudo chown martchus:buildservice $local_db_path
find $local_db_path -type d -exec chmod g+rwx {} \+
find $local_db_path -type f -exec chmod g+rw {} \+
```
### Setting up the chroot for the build
The overall reasoning and procedure is already outlined
[in the Wiki](https://wiki.archlinux.org/index.php/DeveloperWiki:Building_in_a_clean_chroot).
I also like to be able to build for other architectures than x86_64 so the server
expects a directory layout like this:
```
/the/chroot/directory
├── arch-i686
│   └── root
├── arch-x86_64
│   └── root
├── config-i686
│   ├── makepkg.conf
│   ├── pacman.conf
├── config-x86_64
│   ├── makepkg.conf
│   └── pacman.conf
```
So there's a "root" chroot for every architecture and config files for that architecture
in dedicated directories. To achieve this one just has to call `mkarchroot` for each architecture.
It is invoked like this:
```
mkarchroot -C /path/to/pacman.conf -M /path/to/makepkg.conf /directory/to/store/the/tree base-devel
```
Note that `makechrootpkg` will not use the "root" chroot directories directly. It will create a
working copy using `rsync -a --delete -q -W -x "$chrootdir/root/" "$copydir"` (when using btrfs
it creates a subvolume instead of using rsync).
When following the procedure the "root" chroot is owned by `root` and the working copies by the
user you start the buildservice/makechrootpkg with.
Until I find a better solution the buildservice updates the config files within the "root" chroot
to configure the databases used during the build. It relies on the configuration files stored within
the `config-*` directories for that and needs permissions to update them:
```
sudo chown buildservice:buildservice /the/chroot/directory/arch-x86_64/root/etc/{pacman,makepkg}.conf
```
### sudo configuration
`makechrootpkg` seems to use `sudo` internally so it is required that the user one starts the
server with is allowed to use `sudo`. Currently it is not possible to supply the password
automatically so one has to allow access without password like this:
```
buildservice ALL=(ALL) NOPASSWD:ALL
```
### Sample NGINX config
To make the server publicy accessible one should use a reverse proxy. NGINX example:
```
proxy_http_version 1.1; # this is essential for chunked responses to work
proxy_read_timeout 1d; # prevent timeouts when serving live stream
location /buildservice/api/ {
proxy_buffering off; # for live streaming of logfiles
proxy_pass http://localhost:8090/api/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location ~ ^(?!/buildservice/api/)/buildservice/(.*)$ {
alias /run/media/devel/projects/c++/cmake/auto-makepkg/buildservice/static/$1;
}
```
### Sample GPG config
This minimal GPG config allows `makepkg` to validate signatures which might be present
in some PKGBUILDs:
```
~/.gnupg/gpg.conf
---
utf8-strings
keyring /etc/pacman.d/gnupg/pubring.gpg
keyserver-options auto-key-retrieve
keyserver hkp://keys.gnupg.net
```
#### Notes
* Adding the pacman keyring is actually not very useful because we need to check signatures
of any upstream project and not just arch devs.
* If "auto-key-retrieve" does not work, use `gpg --recv-key <KEYID>` as a workaround
* Also see http://allanmcrae.com/2015/01/two-pgp-keyrings-for-package-management-in-arch-linux/
### ccache configuration
To use ccache one can set the ccache directory in the config (`ccache_dir`) and install the
package `ccache` (and `mingw-w64-ccache` for MinGW packages) into the chroot. Make sure the user
you start the server with has permissions to read and write there. Otherwise the resulting
configure errors can be confusing. Internally the server is mounting that directory like
described in [the wiki](https://wiki.archlinux.org/index.php/Ccache#makechrootpkg).
Note that ccache *slows* down the initial compilation. It is only useful to enable it if rebuilds
with only slight changes are expected. Currently it is *not* possible to enable/disable ccache usage
per package (e.g. via a regex).
## Workflow
### General
* Use the "Reload database" build action to reload one or more databases after databases have been
changes by build actions or manually. So far databases are *not* automatically reloaded.
* Use the "Reload configuration" build action to apply configuration changes without restarting the
service.
### Building packages
0. Ensure the databases are up to date via the "Reload databases" build action.
1. Start the "Prepare building" build action.
* Specify the names of packages to be built.
* If a package already exists the `pkgrel` or `epoch` is bumped as needed so the rebuilt
packages are considered as newer by pacman. A warning is logged when bumping is done.
* Missing dependencies are pulled into the build automatically. Whether dependencies are
considered "missing" or "given" depends on the specified databases (see next points).
* Packages will be splitted into batches. The first batch contains all packages which can be
built immediately. The second batch contains all packages which only depend on packages in
the first batch. The third batch contains all packages which only depend on packages in the
second batch and so on. That means packages do not need to be specified in the correct order.
However, the order in which packages are specified is preserved within each batch so it makes
still sense to specify packages in the order you would prefer to build them.
* Cyclic dependencies can not be added to a batch. A list of cyclic "leftovers" is emitted
if those exist and that is considered a failure. If this is the case you need to add a
bootstrap package to break the cycle. The build system is not clever enough to pull a bootstrap
package automatically into the build so it must be specified explicitely. E.g. to build
`mingw-w64-freetype2` and `mingw-w64-harfbuzz` one needs to add `mingw-w64-freetype2-bootstrap`
explicitely to the list of packages to be built.
* Specify exactly one destination database. The built packages are added to this database.
* Specify source databases.
* All packages contained by the source databases are considered as "given" and not pulled
into the build as dependency.
* If no source databases are specified the destination database and all databases the
destination database is based on are used as source databases.
* Source databases must be the destination database or a database the destination database
is based on.
* Specify a "directory" to store meta-data, logs, PKGBUILDs, sources and packages.
* Check "Clean source directories" when previously prepared sources which are still present in
the specified "directory" should be overridden.
* When the build action has finished, have a look at the results to check whether they are as
expected. If not, just restart the build action with adjusted parameters. The "directory"
can generally be re-used. Use "Clean source directories" as needed. To override only a single
source directory, simply delete it manually before restarting the build action.
* You can amend PKGBUILDs created in the "directory" as needed.
2. Start the "Conduct building" build action.
* Specify the same "directory" as in 1.
* Do *not* specify any databases. The databases as specified in 1. will be used.
* One can optionally specify package names to build only a subset of the initially prepared
build.
* When starting the build, the following steps are performed:
1. All sources of all packages are downloaded. The build is stopped when not all sources
can be downloaded.
2. Packages will now be built in the same order as computed when preparing. Building of
the next batch is stopped when a failure happend within the previous batch.
* When using "Build as far as possible" the build is not stopped as previously explained.
* When using "Save chroot of failures" the chroot directory is renamed when a build failure
happens and therefore not overridden by the next build. This is useful to investigate the
failure later without starting the build from scratch.
* When the build has been stopped due to failures one can simply restart the build action. Packages
which have already been built will be skipped.
* It is useful to modify `build-progress.json` in the "directory".
* Set `finished` back to `"1"` and `addedToRepo` back to `false` to let the build system think
a package has not been built yet. This is useful to build a package again after a non-fatal
packaging mistake.
* Set `addedToRepo` to `true` let the build system think a package has already been built.
This effectively skips a package completely.
* Set `hasSources` back to `false` to recreate the source package and download sources
again as needed. This will also copy over contents of the `src` directory to the `pkg`
directory again.
3. Amend a PKGBUILD file to fix a failure within a "big rebuild".
* Fix the package at the source, e.g. the AUR or the local package directory, then
1. Clean the `src` directory within "directory" for the package(s) or use "Clean source directory" if
all packages need fixing.
2. Restart step 1. Existing sources are reused (except for the removed `src` directories) so it shouldn't
take long. The build progress (`hasSources` in particular) of the affected package(s) should be resetted.
3. Restart step 2.
* or: Amend the PKGBUILD within the `src` directory.
1. Set `hasSources` back to `false` as mentioned under 2. to retrigger building the source directory.
2. Optionally restart step 1. to reevaluate possibly adjusted versions, split packages and dependencies.
Be sure *not* to check "Clean source directory" as this would override your amendment.
2. Restart step 2.
#### Further notes
* One can keep using the same "directory" for different builds. Since the `pkg` dirs will not be cleaned
up (existing files are only updated/overridden) previously downloaded source files can be reused (useful if
only `pkgrel` changes or when building from VCS sources or when some sources just remain the same).
## TODOs and further ideas for improvement
[ ] Add generic locking so build actions don't interfere with each other, e.g. when accessing a database directory
[ ] CLI client
[ ] More advanced search options
[ ] Allow running tasks automatically/periodically
[ ] Refresh build action details automatically while an action is pending
[ ] Fix permission error when stopping a process
[ ] Keep the possibility for a "soft stop" where the build action would finish the current item
[ ] Show statistics like CPU and RAM usage about ongoing build processes
[ ] Stop a build process which doesn't produce output after a certain time
[ ] Don't keep always *all* packages in memory for better scalability
[ ] Find out why the web service sometimes gets stuck
* Weirdly, restarting the client (browser) helps in these cases
* Add "stress" test for live-streaming
* Start process producing lots of output very fast
* Let different clients connect and disconnect fast
[ ] Improve test coverage
[ ] Include `xterm.js` properly
## Build instructions and dependencies
For a PKGBUILD checkout my [PKGBUILDs repository](https://github.com/Martchus/PKGBUILDs).
### C++ stuff
The application depends on [c++utilities](https://github.com/Martchus/cpp-utilities),
[reflective-rapidjson](https://github.com/Martchus/reflective-rapidjson) and some boost modules.
For basic instructions checkout the README file of [c++utilities](https://github.com/Martchus/cpp-utilities).
### Web stuff
The only dependency is `xterm.js` which is bundled. However, only the JavaScript and CSS files are
bundled. For development the full checkout might be useful (e.g. for TypeScript mapping). It can be
retrieved using npm:
```
cd srv/static/node_modules
npm install xterm
```

46
cli/CMakeLists.txt

@ -0,0 +1,46 @@
cmake_minimum_required(VERSION 3.1.0 FATAL_ERROR)
# add project files
set(HEADER_FILES)
set(SRC_FILES main.cpp)
set(TEST_HEADER_FILES)
set(TEST_SRC_FILES tests/cppunit.cpp)
# meta data
set(META_PROJECT_NAME repomgr)
set(META_PROJECT_TYPE application)
set(META_PROJECT_VARNAME BUILD_SERVICE_CLIENT)
set(META_APP_NAME "CLI for repo manager")
set(META_APP_DESCRIPTION "CLI tool to use the repository manager for Arch Linux")
# find c++utilities
set(CONFIGURATION_PACKAGE_SUFFIX
""
CACHE STRING "sets the suffix for find_package() calls to packages configured via c++utilities")
find_package(c++utilities${CONFIGURATION_PACKAGE_SUFFIX} 5.0.0 REQUIRED)
use_cpp_utilities()
# find backend libraries
find_package(libpkg ${META_APP_VERSION} REQUIRED)
use_libpkg()
# find backend libraries
find_package(librepomgr ${META_APP_VERSION} REQUIRED)
use_librepomgr()
# use bundled tabulate
list(APPEND PRIVATE_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/../3rdparty/tabulate/include/")
# include modules to apply configuration
include(BasicConfig)
include(WindowsResources)
include(AppTarget)
include(TestTarget)
include(ShellCompletion)
include(ConfigHeader)
# add install for config file template
install(
FILES doc/client-config-example.conf
DESTINATION "${CMAKE_INSTALL_PREFIX}/${META_DATA_DIR}/skel"
COMPONENT config)

14
cli/doc/client-config-example.conf

@ -0,0 +1,14 @@
[instance/local]
url = http://127.0.0.1:8090
user = martchus
[instance/server]
url = https://martchus.no-ip.biz/buildservice
user = martchus
[instance/bogus]
url = http://martchus.no-ip.biz/foo
user = martchus
[user/martchus]
password = test

322
cli/main.cpp

@ -0,0 +1,322 @@
#include "resources/config.h"
#include "../librepomgr/json.h"
#include "../librepomgr/webapi/params.h"
#include "../librepomgr/webclient/session.h"
#include "../libpkg/data/database.h"
#include "../libpkg/data/package.h"
#include <c++utilities/application/argumentparser.h>
#include <c++utilities/application/commandlineutils.h>
#include <c++utilities/conversion/stringbuilder.h>
#include <c++utilities/conversion/stringconversion.h>
#include <c++utilities/io/ansiescapecodes.h>
#include <c++utilities/io/inifile.h>
#include <c++utilities/io/nativefilestream.h>
#include <tabulate/table.hpp>
#include <fstream>
#include <functional>
#include <iostream>
#include <string_view>
using namespace CppUtilities;
using namespace CppUtilities::EscapeCodes;
using namespace std;
struct ClientConfig {
void parse(const Argument &configFileArg, const Argument &instanceArg);
const char *path = nullptr;
std::string instance;
std::string url;
std::string userName;
std::string password;
};
void ClientConfig::parse(const Argument &configFileArg, const Argument &instanceArg)
{
// parse connfig file
path = configFileArg.firstValue();
if (!path || !*path) {
path = "/etc/buildservice" PROJECT_CONFIG_SUFFIX "/client.conf";
}
auto configFile = NativeFileStream();
configFile.exceptions(std::ios_base::badbit | std::ios_base::failbit);
configFile.open(path, std::ios_base::in);
auto configIni = AdvancedIniFile();
configIni.parse(configFile);
configFile.close();
// read innstance
if (instanceArg.isPresent()) {
instance = instanceArg.values().front();
}
for (const auto &section : configIni.sections) {
if (!section.name.starts_with("instance/")) {
continue;
}
if (!instance.empty() && instance != std::string_view(section.name.data() + 9, section.name.size() - 9)) {
continue;
}
instance = section.name;
if (const auto url = section.findField("url"); url != section.fieldEnd()) {
this->url = std::move(url->value);
} else {
throw std::runtime_error("Config is invalid: No \"url\" specified within \"" % section.name + "\".");
}
if (const auto user = section.findField("user"); user != section.fieldEnd()) {
this->userName = std::move(user->value);
}
break;
}
if (url.empty()) {
throw std::runtime_error("Config is invalid: Instance configuration insufficient.");
}
// read user data
if (userName.empty()) {
return;
}
if (const auto userSection = configIni.findSection("user/" + userName); userSection != configIni.sectionEnd()) {
if (const auto password = userSection->findField("password"); password != userSection->fieldEnd()) {
this->password = std::move(password->value);
} else {
throw std::runtime_error("Config is invalid: No \"password\" specified within \"" % userSection->name + "\".");
}
} else {
throw std::runtime_error("Config is invalid: User \"" % userName + "\" referenced in instance configuration not found.");
}
}
void configureColumnWidths(tabulate::Table &table)
{
const auto terminalSize = determineTerminalSize();
if (!terminalSize.columns) {
return;
}
struct ColumnStats {
std::size_t maxSize = 0;
std::size_t totalSize = 0;
std::size_t rows = 0;
double averageSize = 0.0;
double averagePercentage = 0.0;
std::size_t width = 0;
};
auto columnStats = std::vector<ColumnStats>();
for (const auto &row : table) {
const auto columnCount = row.size();
if (columnStats.size() < columnCount) {
columnStats.resize(columnCount);
}
auto column = columnStats.begin();
for (const auto &cell : row.cells()) {
const auto size = cell->size();
column->maxSize = std::max(column->maxSize, size);
column->totalSize += std::max<std::size_t>(10, size);
column->rows += 1;
++column;
}
}
auto totalAverageSize = 0.0;
for (auto &column : columnStats) {
totalAverageSize += (column.averageSize = static_cast<double>(column.totalSize) / column.rows);
}
for (auto &column : columnStats) {
column.averagePercentage = column.averageSize / totalAverageSize;
column.width = std::max<std::size_t>(terminalSize.columns * column.averagePercentage, std::min<std::size_t>(column.maxSize, 10));
}
for (std::size_t columnIndex = 0; columnIndex != columnStats.size(); ++columnIndex) {
table.column(columnIndex).format().width(columnStats[columnIndex].width);
}
}
void printPackageSearchResults(const LibRepoMgr::WebClient::Response::body_type::value_type &jsonData)
{
const auto packages = ReflectiveRapidJSON::JsonReflector::fromJson<std::list<LibPkg::PackageSearchResult>>(jsonData.data(), jsonData.size());
tabulate::Table t;
t.format().hide_border();
t.add_row({ "Arch", "Repo", "Name", "Version", "Description", "Build date" });
for (const auto &[db, package] : packages) {
const auto &dbInfo = std::get<LibPkg::DatabaseInfo>(db);
t.add_row(
{ package->packageInfo ? package->packageInfo->arch : dbInfo.arch, dbInfo.name, package->name, package->version, package->description,
package->packageInfo && !package->packageInfo->buildDate.isNull() ? package->packageInfo->buildDate.toString() : "?" });
}
t.row(0).format().font_align(tabulate::FontAlign::center).font_style({ tabulate::FontStyle::bold });
configureColumnWidths(t);
std::cout << t << std::endl;
}
template <typename List> inline std::string formatList(const List &list)
{
return joinStrings(list, ", ");
}
std::string formatDependencies(const std::vector<LibPkg::Dependency> &deps)
{
auto asStrings = std::vector<std::string>();
asStrings.reserve(deps.size());
for (const auto &dep : deps) {
asStrings.emplace_back(dep.toString());
}
return formatList(asStrings);
}
void printPackageDetails(const LibRepoMgr::WebClient::Response::body_type::value_type &jsonData)
{
const auto packages = ReflectiveRapidJSON::JsonReflector::fromJson<std::list<LibPkg::Package>>(jsonData.data(), jsonData.size());
for (const auto &package : packages) {
const auto *const pkg = &package;
std::cout << TextAttribute::Bold << pkg->name << ' ' << pkg->version << TextAttribute::Reset << '\n';
tabulate::Table t;
t.format().hide_border();
if (pkg->packageInfo) {
t.add_row({ "Arch", pkg->packageInfo->arch });
} else if (pkg->sourceInfo) {
t.add_row({ "Archs", formatList(pkg->sourceInfo->archs) });
}
t.add_row({ "Description", pkg->description });
t.add_row({ "Upstream URL", pkg->upstreamUrl });
t.add_row({ "License(s)", formatList(pkg->licenses) });
t.add_row({ "Groups", formatList(pkg->groups) });
if (pkg->packageInfo && pkg->packageInfo->size) {
t.add_row({ "Package size", dataSizeToString(pkg->packageInfo->size, true) });
}
if (pkg->installInfo) {
t.add_row({ "Installed size", dataSizeToString(pkg->installInfo->installedSize, true) });
}
if (pkg->packageInfo) {
if (!pkg->packageInfo->packager.empty()) {
t.add_row({ "Packager", pkg->packageInfo->packager });
}
if (!pkg->packageInfo->buildDate.isNull()) {
t.add_row({ "Packager", pkg->packageInfo->buildDate.toString() });
}
}
t.add_row({ "Dependencies", formatDependencies(pkg->dependencies) });
t.add_row({ "Optional dependencies", formatDependencies(pkg->optionalDependencies) });
if (pkg->sourceInfo) {
t.add_row({ "Make dependencies", formatDependencies(pkg->sourceInfo->makeDependencies) });
t.add_row({ "Check dependencies", formatDependencies(pkg->sourceInfo->checkDependencies) });
}
t.add_row({ "Provides", formatDependencies(pkg->provides) });
t.add_row({ "Replaces", formatDependencies(pkg->replaces) });
t.add_row({ "Conflicts", formatDependencies(pkg->conflicts) });
t.add_row({ "Contained libraries", formatList(pkg->libprovides) });
t.add_row({ "Needed libraries", formatList(pkg->libdepends) });
t.column(0).format().font_align(tabulate::FontAlign::right);
configureColumnWidths(t);
std::cout << t << '\n';
}
std::cout.flush();
}
void printRawData(const LibRepoMgr::WebClient::Response::body_type::value_type &rawData)
{
if (!rawData.empty()) {
std::cerr << Phrases::InfoMessage << "Server replied:" << Phrases::End << rawData << '\n';
}
}
void handleResponse(const std::string &url, const LibRepoMgr::WebClient::SessionData &data, const LibRepoMgr::WebClient::HttpClientError &error,
void (*printer)(const LibRepoMgr::WebClient::Response::body_type::value_type &jsonData), int &returnCode)
{
const auto &response = std::get<LibRepoMgr::WebClient::Response>(data.response);
const auto &body = response.body();
if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) {
std::cerr << Phrases::ErrorMessage << "Unable to connect: " << error.what() << Phrases::End;
std::cerr << Phrases::InfoMessage << "URL was: " << url << Phrases::End;
printRawData(body);
return;
}
if (response.result() != boost::beast::http::status::ok) {
std::cerr << Phrases::ErrorMessage << "HTTP request not successful: " << error.what() << Phrases::End;
std::cerr << Phrases::InfoMessage << "URL was: " << url << Phrases::End;
printRawData(body);
return;
}
try {
std::invoke(printer, body);
} catch (const RAPIDJSON_NAMESPACE::ParseResult &e) {
std::cerr << Phrases::ErrorMessage << "Unable to parse responnse: " << tupleToString(LibRepoMgr::serializeParseError(e)) << Phrases::End;
std::cerr << Phrases::InfoMessage << "URL was: " << url << std::endl;
returnCode = 11;
} catch (const std::runtime_error &e) {
std::cerr << Phrases::ErrorMessage << "Unable to display response: " << e.what() << Phrases::End;
std::cerr << Phrases::InfoMessage << "URL was: " << url << std::endl;
returnCode = 12;
}
}
int main(int argc, const char *argv[])
{
// define command-specific parameters
std::string path;
void (*printer)(const LibRepoMgr::WebClient::Response::body_type::value_type &jsonData) = nullptr;
// read CLI args
ArgumentParser parser;
ConfigValueArgument configFileArg("config-file", 'c', "specifies the path of the config file", { "path" });
configFileArg.setEnvironmentVariable(PROJECT_VARNAME_UPPER "_CONFIG_FILE");
ConfigValueArgument instanceArg("instance", 'i', "specifies the instance to connect to", { "instance" });
OperationArgument searchArg("search", 's', "searches packages");
ConfigValueArgument searchTermArg("term", 't', "specifies the search term", { "term" });
searchTermArg.setImplicit(true);
searchTermArg.setRequired(true);
ConfigValueArgument searchModeArg("mode", 'm', "specifies the mode", { "name/name-contains/regex/provides/depends/libprovides/libdepends" });
searchModeArg.setPreDefinedCompletionValues("name name-contains regex provides depends libprovides libdepends");
searchArg.setSubArguments({ &searchTermArg, &searchModeArg });
searchArg.setCallback([&path, &printer, &searchTermArg, &searchModeArg](const ArgumentOccurrence &) {
path = "/api/v0/packages?mode=" % LibRepoMgr::WebAPI::Url::encodeValue(searchModeArg.firstValueOr("name-contains")) % "&name="
+ LibRepoMgr::WebAPI::Url::encodeValue(searchTermArg.values().front());
printer = printPackageSearchResults;
});
OperationArgument packageArg("package", 'p', "shows detauls about a package");
ConfigValueArgument packageNameArg("name", 'n', "specifies the package name", { "name" });
packageNameArg.setImplicit(true);
packageNameArg.setRequired(true);
packageArg.setSubArguments({ &packageNameArg });
packageArg.setCallback([&path, &printer, &packageNameArg](const ArgumentOccurrence &) {
path = "/api/v0/packages?mode=name&details=1&name=" + LibRepoMgr::WebAPI::Url::encodeValue(packageNameArg.values().front());
printer = printPackageDetails;
});
HelpArgument helpArg(parser);
NoColorArgument noColorArg;
parser.setMainArguments({ &searchArg, &packageArg, &instanceArg, &configFileArg, &noColorArg, &helpArg });
parser.parseArgs(argc, argv);
// return early if no operation specified
if (!printer) {
if (!helpArg.isPresent()) {
std::cerr << "No command specified; use --help to list available commands.\n";
}
return 0;
}
// parse config
auto config = ClientConfig();
try {
config.parse(configFileArg, instanceArg);
} catch (const std::runtime_error &e) {
std::cerr << Phrases::ErrorMessage << "Unable to parse config: " << e.what() << Phrases::End;
std::cerr << Phrases::InfoMessage << "Path of config file was: " << (config.path ? config.path : "[none]") << Phrases::End;
return 10;
}
// make HTTP request and show response
const auto url = config.url + path;
auto ioContext = boost::asio::io_context();
auto sslContext = boost::asio::ssl::context{ boost::asio::ssl::context::sslv23_client };
auto returnCode = 0;
sslContext.set_verify_mode(boost::asio::ssl::verify_peer);
sslContext.set_default_verify_paths();
LibRepoMgr::WebClient::runSessionFromUrl(ioContext, sslContext, url,
std::bind(&handleResponse, std::ref(url), std::placeholders::_1, std::placeholders::_2, printer, std::ref(returnCode)), std::string(),
config.userName, config.password);
ioContext.run();
return 0;
}

1
cli/tests/cppunit.cpp

@ -0,0 +1 @@
#include <c++utilities/tests/cppunit.h>

103
libpkg/CMakeLists.txt

@ -0,0 +1,103 @@
cmake_minimum_required(VERSION 3.1.0 FATAL_ERROR)
# add project files
set(HEADER_FILES
data/package.h
data/database.h
data/config.h
data/lockable.h
data/siglevel.h
parser/aur.h
parser/package.h
parser/database.h
parser/config.h
parser/utils.h
parser/binary.h)
set(SRC_FILES
data/package.cpp
data/database.cpp
data/config.cpp
data/lockable.cpp
algo/search.cpp
algo/buildorder.cpp
algo/licenses.cpp
parser/aur.cpp
parser/package.cpp
parser/database.cpp
parser/config.cpp
parser/utils.cpp
parser/binary.cpp
parser/siglevel.cpp)
set(TEST_HEADER_FILES tests/parser_helper.h)
set(TEST_SRC_FILES tests/cppunit.cpp tests/parser.cpp tests/parser_binary.cpp tests/parser_helper.cpp tests/data.cpp
tests/utils.cpp)
# meta data
set(META_PROJECT_NAME libpkg)
set(META_PROJECT_TYPE library)
set(META_PROJECT_VARNAME LIBPKG)
set(META_APP_NAME "Inofficial Arch Linux package library")
set(META_APP_DESCRIPTION "C++ library to parse Arch Linux packages and databases")
set(META_VERSION_MAJOR 0)
set(META_VERSION_MINOR 0)
set(META_VERSION_PATCH 1)
# find c++utilities
set(CONFIGURATION_PACKAGE_SUFFIX
""
CACHE STRING "sets the suffix for find_package() calls to packages configured via c++utilities")
find_package(c++utilities${CONFIGURATION_PACKAGE_SUFFIX} 5.2.0 REQUIRED)
use_cpp_utilities(VISIBILITY PUBLIC)
# use std::filesystem
include(3rdParty)
use_standard_filesystem(VISIBILITY PUBLIC)
# find reflective-rapidjson
find_package(reflective_rapidjson${CONFIGURATION_PACKAGE_SUFFIX} REQUIRED)
use_reflective_rapidjson(VISIBILITY PUBLIC)
# find 3rd party libraries zlib
use_zlib()
# libarchive
find_package(LibArchive)
if (NOT LibArchive_FOUND)
message(FATAL_ERROR "Unable to find libarchive.")
endif ()
add_library(libarchive UNKNOWN IMPORTED)
set_property(TARGET libarchive PROPERTY IMPORTED_LOCATION "${LibArchive_LIBRARIES}")
set_property(TARGET libarchive PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${LibArchive_INCLUDE_DIRS}")
use_target(TARGET_NAME libarchive)
# apply basic configuration
include(BasicConfig)
# trigger code generator for tests because the tests already contain structs to be (de)serialized
include(ReflectionGenerator)
add_reflection_generator_invocation(
INPUT_FILES
data/database.h
data/package.h
data/config.h
data/siglevel.h
parser/aur.h
CLANG_OPTIONS_FROM_TARGETS
"${META_TARGET_NAME}"
CLANG_OPTIONS_FROM_DEPENDENCIES
"${PUBLIC_LIBRARIES};${PRIVATE_LIBRARIES}"
GENERATORS
json
binary
OUTPUT_LISTS
SRC_FILES
JSON_VISIBILITY
${META_PROJECT_VARNAME}_EXPORT
BINARY_VISBILITY
${META_PROJECT_VARNAME}_EXPORT)
# include modules to apply configuration
include(WindowsResources)
include(LibraryTarget)
include(TestTarget)
include(Doxygen)
include(ConfigHeader)

157
libpkg/algo/buildorder.cpp

@ -0,0 +1,157 @@
#include "../data/config.h"
using namespace std;
namespace LibPkg {
/// \cond
struct TopoSortItem {
explicit TopoSortItem(PackageSearchResult &&package, bool onlyDependency)
: package(package)
, onlyDependency(onlyDependency)
, finished(false)
{
}
PackageSearchResult package;
bool onlyDependency;
bool finished;
};
/// \endcond
/*!
* \brief Adds \a dependency and its dependencies in topological order to \a finishedItems.
*
* Also populates \a allItems in arbitrary order which owns the items. Unknown \a dependencies (no matching package found in any of the databases) are
* added to \a ignored and ignored.
*
* The variable \a cycleTracking is supposed to be empty in the beginning and internally used to keep track of the current chain. In case a cycle is detected,
* \a cycleTracking is set to contain the cyclic dependency path. On normal exit it just contains the most recently processed dependency chain (not really
* interesting).
*
* \remarks Dependency cycles are ignored if there are already binary packages of the involved dependencies. It is assumed that in this case it is possible to
* install the existing binaries for building new ones (e.g. just use the current GCC to build the new one instead of taking additional bootstrapping
* effort). We also ignore cycles affecting only indirect dependencies as long as there are binaries for them available (e.g. to build a package requiring
* mesa we just install mesa and don't care about its cyclic dependency with libglvnd).
* \return Returns true if \a dependency could be added to \a finishedItems, has already been present or has been ignored. Returns false if a cycle has been detected.
* \deprecated This function can likely be removed. The build preparation task has its own implementation (to compute batches) which is more useful.
*/
bool Config::addDepsRecursivelyInTopoOrder(vector<unique_ptr<TopoSortItem>> &allItems, vector<TopoSortItem *> &finishedItems,
std::vector<std::string> &ignored, std::vector<PackageSearchResult> &cycleTracking, const Dependency &dependency, BuildOrderOptions options,
bool onlyDependency)
{
// skip if dependency is already present
for (const auto &item : finishedItems) {
if (item->package.pkg->providesDependency(dependency)) {
// ensure the already finished dependency is not treated as dependency
if (!onlyDependency) {
item->onlyDependency = false;
}
return true;
}
}
// search for a package providing the specified dependency (just using the first matching package for now)
auto packageSearchResult = findPackage(dependency);
auto &pkg = packageSearchResult.pkg;
// ignore the dependency if we can't find a package which provides it
if (!pkg) {
ignored.emplace_back(dependency.toString());
return true;
}
// check whether a topo sort item for the current dependency is already pending to detect cycles
for (auto &item : allItems) {
if (pkg != item->package.pkg) {
continue;
}
// skip package if it is only a dependency and there's already a binary package
if (onlyDependency && pkg->packageInfo) {
return true;
}
// report error: remove so far "healy" path in the current chain so it contains only the cyclic path anymore
for (auto i = cycleTracking.begin(), end = cycleTracking.end(); i != end; ++i) {
if (pkg == i->pkg) {
cycleTracking.erase(cycleTracking.begin(), i);
break;
}
}
return false;
}
// add package to the "current chain" (used to comprehend the path in case a cycle occured)
cycleTracking.emplace_back(packageSearchResult);
// add topo sort item for dependency
allItems.emplace_back(make_unique<TopoSortItem>(move(packageSearchResult), onlyDependency));
auto *const currentItem = allItems.back().get();
// add the dependencies of this dependency first
const auto addBuildDependencies = (options & BuildOrderOptions::ConsiderBuildDependencies) && pkg->sourceInfo;
const auto *const runtimeDependencies = &pkg->dependencies;
const auto *const makeDependencies = addBuildDependencies ? &pkg->sourceInfo->makeDependencies : nullptr;
const auto *const checkDependencies = addBuildDependencies ? &pkg->sourceInfo->checkDependencies : nullptr;
for (const auto *dependencies : { runtimeDependencies, makeDependencies, checkDependencies }) {
if (!dependencies) {
continue;
}
for (const auto &dependency : *dependencies) {
// skip dependencies provided by the current package itself (FIXME: right now python 3.n depends on python<3.(n+1) which should be fixed)
if (pkg->providesDependency(dependency)) {
continue;
}
if (!addDepsRecursivelyInTopoOrder(allItems, finishedItems, ignored, cycleTracking, dependency, options, true)) {
// skip if a cycle has been detected
return false;
}
}
}
// remove the package from the "current chain" again
cycleTracking.pop_back();
// mark the current item as visited and add it to finished items
finishedItems.emplace_back(currentItem);
return currentItem->finished = true;
}
BuildOrderResult Config::computeBuildOrder(const std::vector<string> &dependencyDenotations, BuildOrderOptions options)
{
// setup variables to store results
BuildOrderResult result;
vector<unique_ptr<TopoSortItem>> allTopoSortItems;
vector<TopoSortItem *> finishedTopoSortItems;
allTopoSortItems.reserve(dependencyDenotations.size());
// add dependencies
for (const auto &dependency : dependencyDenotations) {
// continue adding dependencies as long as no cycles have been detected
if (addDepsRecursivelyInTopoOrder(allTopoSortItems, finishedTopoSortItems, result.ignored, result.cycle,
Dependency(dependency.data(), dependency.size()), options, false)) {
result.cycle.clear();
continue;
}
// stop on the first cycle
break;
}
// add finished items to result (even if we detected a cycle)
result.order.reserve(allTopoSortItems.size());
for (auto &item : finishedTopoSortItems) {
if (!(options & BuildOrderOptions::IncludeAllDependencies)
&& (item->onlyDependency && ((options & BuildOrderOptions::IncludeSourceOnlyDependencies) || item->package.pkg->packageInfo))) {
continue;
}
result.order.emplace_back(move(item->package));
}
result.success = result.cycle.empty() && result.ignored.empty();
return result;
}
} // namespace LibPkg

335
libpkg/algo/licenses.cpp

@ -0,0 +1,335 @@
#include "../data/config.h"
#include <c++utilities/conversion/stringbuilder.h>
#include <fstream>
#include <sstream>
#include <iostream>
using namespace std;
using namespace CppUtilities;
namespace LibPkg {
bool Config::addLicenseInfo(LicenseResult &result, const Dependency &dependency)
{
// find the referenced package
auto searchResult = findPackage(dependency);
const auto &package = searchResult.pkg;
if (!package) {
result.success = false;
result.notes.emplace_back("Unable to locate " + dependency.toString());
return false;
}
auto packageID = addLicenseInfo(result, searchResult, package);
if (packageID.empty()) {
result.ignoredPackages.emplace_back(package->name % '-' + package->version);
return false;
}
result.consideredPackages.emplace_back(package->name % '-' + package->version);
if (result.mainProject.empty()) {
result.mainProject = move(packageID);
} else {
result.dependendProjects.emplace(move(packageID));
}
return true;
}
std::string Config::addLicenseInfo(LicenseResult &result, PackageSearchResult &searchResult, const std::shared_ptr<Package> &package)
{
// make up some identifier to refer to the package in the license summary
// * use the "regular" package name (e.g. gcc instead of mingw-w64-gcc)
// * include only the upstream version (but not epoch and pkgrel)
// * the licensing files for the whole MinGW-w64 project is contained by the mingw-w64-headers package
// * the licensing files for GCC is contained by the gcc-libs package
// * the licensing files for all Qt modules are contained within qt5-base; don't distinguish the modules for our purposes
// * use the real project name if known
const auto upstreamVersion = PackageVersion::fromString(package->version).upstream;
auto regularPackageName = package->computeRegularPackageName();
auto packageID = regularPackageName.empty() ? package->name : regularPackageName;
static const auto displayNames = unordered_map<string, string>{
{ "mingw-w64-headers", "MinGW-w64" },
{ "gcc", "GCC" },
{ "gcc-libs", "GCC" },
{ "freetype2", "FreeType" },
{ "harfbuzz", "HarfBuzz" },
{ "graphite", "Graphite" },
{ "openssl", "OpenSSL" },
{ "pcre", "PCRE" },
{ "pcre2", "PCRE2" },
{ "glib2", "GLib" },
{ "numix-icon-theme", "Numix icon theme" },
{ "breeze-icons", "Breeze icons (from KDE)" },
{ "go", "Go" },
{ "syncthing", "Syncthing" },
{ "syncthingtray", "Syncthing Tray" },
{ "tageditor", "Tag Editor" },
{ "passwordmanager", "Password Manager" },
};
if (endsWith(packageID, "-git") || endsWith(packageID, "-svn")) {
packageID.resize(packageID.size() - 4);
} else if (endsWith(packageID, "-hg")) {
packageID.resize(packageID.size() - 3);
}
if (const auto displayName = displayNames.find(packageID); displayName != displayNames.cend()) {
packageID = displayName->second;
} else if (startsWith(packageID, "qt5-")) {
packageID = "Qt 5";
regularPackageName = "qt5-base";
}
// skip package if custom license has already been added
if (const auto customLicenses = result.customLicences.find(packageID);
customLicenses != result.customLicences.end() && !customLicenses->second.empty()) {
return packageID;
}
// check whether the package has a standard license and/or a custom license
bool hasCustomLicense = package->licenses.empty();
if (packageID == "Qt 5") {
// consider Qt's licenses custom as it has special variants of the standard licenses FDL, GPL and LGPL
hasCustomLicense = true;
} else {
// read the package's license field (see https://wiki.archlinux.org/index.php/PKGBUILD#license)
for (const auto &license : package->licenses) {
// check for custom licenses and licenses which contain project specific references and are therefore considered custom as well
if (startsWith(license, "custom") || license == "BSD" || license == "ISC" || license == "MIT" || license == "ZLIB"
|| license == "Python") {
hasCustomLicense = true;
continue;
}
// map Arch Linux generic way to say e.g. "GPL2 and above" to a concrete license e.g. "GPL2"
auto concreteLicense = license;
if (license == "GPL") {
concreteLicense = "GPL2";
} else if (license == "LGPL" ||