diff options
| author | DongHun Kwak <dh0128.kwak@samsung.com> | 2022-02-16 16:35:49 +0900 |
|---|---|---|
| committer | DongHun Kwak <dh0128.kwak@samsung.com> | 2022-02-16 16:35:49 +0900 |
| commit | 0bc4568c4b69ef6535079bedda4a578f4e375aeb (patch) | |
| tree | 3b6c2c08031225e8f6cc9032b4ee0ba78512538b | |
| parent | 7acd85d65ef52b08be9cf5cdf057239aa7a21e85 (diff) | |
| download | rsync-0bc4568c4b69ef6535079bedda4a578f4e375aeb.tar.gz rsync-0bc4568c4b69ef6535079bedda4a578f4e375aeb.tar.bz2 rsync-0bc4568c4b69ef6535079bedda4a578f4e375aeb.zip | |
Imported Upstream version 3.2.0upstream/3.2.0
178 files changed, 29839 insertions, 18148 deletions
@@ -15,8 +15,13 @@ config.status aclocal.m4 /proto.h /proto.h-tstamp -/rsync.1 -/rsyncd.conf.5 +/rsync*.1 +/rsync*.5 +/rsync*.html +/help-rsync*.h +/default-cvsignore.h +/default-dont-compress.h +/.md2man-works /autom4te*.cache /confdefs.h /conftest* @@ -24,8 +29,6 @@ aclocal.m4 /getgroups /gmon.out /rsync -/rsync-ssl -/stunnel-rsync /stunnel-rsyncd.conf /shconfig /testdir @@ -45,4 +48,5 @@ aclocal.m4 /testsuite/devices-fake.test /testsuite/xattrs-hlink.test /patches +/SaVeDiR .deps @@ -672,3 +672,12 @@ may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <http://www.gnu.org/philosophy/why-not-lgpl.html>. + +REGARDING OPENSSL AND XXHASH + +In addition, as a special exception, the copyright holders give +permission to dynamically link rsync with the OpenSSL and xxhash +libraries when those libraries are being distributed in compliance +with their license terms, and to distribute a dynamically linked +combination of rsync and these libraries. This is also considered +to be covered under the GPL's System Libraries exception. @@ -1,13 +1,13 @@ To build and install rsync: - $ ./configure - $ make - # make install + $ ./configure + $ make + # make install You may set the installation directory and other parameters by options to ./configure. To see them, use: - $ ./configure --help + $ ./configure --help Configure tries to figure out if the local system uses group "nobody" or "nogroup" by looking in the /etc/group file. (This is only used for the diff --git a/Makefile.in b/Makefile.in index f912f312..31ddc43b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -4,14 +4,17 @@ prefix=@prefix@ datarootdir=@datarootdir@ exec_prefix=@exec_prefix@ -stunnel4=@STUNNEL4@ bindir=@bindir@ +libdir=@libdir@/rsync mandir=@mandir@ LIBS=@LIBS@ CC=@CC@ +AWK=@AWK@ CFLAGS=@CFLAGS@ CPPFLAGS=@CPPFLAGS@ +CXX=@CXX@ +CXXFLAGS=@CXXFLAGS@ EXEEXT=@EXEEXT@ LDFLAGS=@LDFLAGS@ LIBOBJDIR=lib/ @@ -29,7 +32,10 @@ VERSION=@RSYNC_VERSION@ .SUFFIXES: .SUFFIXES: .c .o -GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsyncd.conf.5 +SIMD_x86_64=simd-checksum-x86_64.o lib/md5-asm-x86_64.o + +GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsync.1.html \ + rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \ lib/pool_alloc.h LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \ @@ -44,9 +50,9 @@ OBJS3=progress.o pipe.o DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \ popt/popthelp.o popt/poptparse.o -OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ +OBJS=$(OBJS1) $(OBJS2) $(OBJS3) @SIMD@ $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ -TLS_OBJ = tls.o syscall.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@ +TLS_OBJ = tls.o syscall.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@ # Programs we must have to run the test cases CHECK_PROGS = rsync$(EXEEXT) tls$(EXEEXT) getgroups$(EXEEXT) getfsdev$(EXEEXT) \ @@ -63,21 +69,20 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash. $(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -c $< @CC_SHOBJ_FLAG@ @OBJ_RESTORE@ -all: Makefile rsync$(EXEEXT) rsync-ssl stunnel-rsync stunnel-rsyncd.conf @MAKE_MAN@ +.PHONY: all +all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@ +.PHONY: install install: all -${MKDIR_P} ${DESTDIR}${bindir} ${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir} + ${INSTALLCMD} -m 755 rsync-ssl ${DESTDIR}${bindir} -${MKDIR_P} ${DESTDIR}${mandir}/man1 -${MKDIR_P} ${DESTDIR}${mandir}/man5 if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi + if test -f rsync-ssl.1; then ${INSTALLMAN} -m 644 rsync-ssl.1 ${DESTDIR}${mandir}/man1; fi if test -f rsyncd.conf.5; then ${INSTALLMAN} -m 644 rsyncd.conf.5 ${DESTDIR}${mandir}/man5; fi -install-ssl-client: rsync-ssl stunnel-rsync - -${MKDIR_P} ${DESTDIR}${bindir} - ${INSTALLCMD} -m 755 rsync-ssl ${DESTDIR}${bindir} - ${INSTALLCMD} -m 755 stunnel-rsync ${DESTDIR}${bindir} - install-ssl-daemon: stunnel-rsyncd.conf -${MKDIR_P} ${DESTDIR}/etc/stunnel ${INSTALLCMD} -m 644 stunnel-rsyncd.conf ${DESTDIR}/etc/stunnel/rsyncd.conf @@ -95,9 +100,19 @@ rsync$(EXEEXT): $(OBJS) $(OBJS): $(HEADERS) $(CHECK_OBJS): $(HEADERS) +tls.o xattrs.o: lib/sysxattrs.h +options.o: latest-year.h help-rsync.h help-rsyncd.h +exclude.o: default-cvsignore.h +loadparm.o: default-dont-compress.h flist.o: rounding.h +default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk + $(AWK) -f $(srcdir)/define-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md + +help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk + $(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md + rounding.h: rounding.c rsync.h proto.h @for r in 0 1 3; do \ if $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o rounding -DEXTRA_ROUNDING=$$r -I. $(srcdir)/rounding.c >rounding.out 2>&1; then \ @@ -116,6 +131,12 @@ rounding.h: rounding.c rsync.h proto.h fi @rm -f rounding.out +simd-checksum-x86_64.o: simd-checksum-x86_64.cpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $(srcdir)/simd-checksum-x86_64.cpp + +lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.s + $(CC) -c -o $@ $(srcdir)/lib/md5-asm-x86_64.s + tls$(EXEEXT): $(TLS_OBJ) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TLS_OBJ) $(LIBS) @@ -128,7 +149,7 @@ getgroups$(EXEEXT): getgroups.o getfsdev$(EXEEXT): getfsdev.o $(CC) $(CFLAGS) $(LDFLAGS) -o $@ getfsdev.o $(LIBS) -TRIMSLASH_OBJ = trimslash.o syscall.o lib/compat.o lib/snprintf.o +TRIMSLASH_OBJ = trimslash.o syscall.o t_stub.o lib/compat.o lib/snprintf.o trimslash$(EXEEXT): $(TRIMSLASH_OBJ) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(TRIMSLASH_OBJ) $(LIBS) @@ -136,13 +157,15 @@ T_UNSAFE_OBJ = t_unsafe.o syscall.o util.o util2.o t_stub.o lib/compat.o lib/snp t_unsafe$(EXEEXT): $(T_UNSAFE_OBJ) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_UNSAFE_OBJ) $(LIBS) +.PHONY: conf +conf: configure.sh config.h.in + +.PHONY: gen gen: conf proto.h man +.PHONY: gensend gensend: gen - rsync -aivzc $(GENFILES) $${SAMBA_HOST-samba.org}:/home/ftp/pub/rsync/generated-files/ - -conf: - cd $(srcdir) && $(MAKE) -f prepare-source.mak conf + rsync -aic $(GENFILES) $${SAMBA_HOST-samba.org}:/home/ftp/pub/rsync/generated-files/ aclocal.m4: $(srcdir)/m4/*.m4 aclocal -I $(srcdir)/m4 @@ -174,6 +197,7 @@ configure.sh config.h.in: configure.ac aclocal.m4 fi \ fi +.PHONY: reconfigure reconfigure: configure.sh ./config.status --recheck ./config.status @@ -193,52 +217,46 @@ Makefile: Makefile.in config.status configure.sh config.h.in fi \ fi -rsync-ssl: $(srcdir)/rsync-ssl.in Makefile - sed 's;\@bindir\@;$(bindir);g' <$(srcdir)/rsync-ssl.in >rsync-ssl - @chmod +x rsync-ssl - -stunnel-rsync: $(srcdir)/stunnel-rsync.in Makefile - sed 's;\@stunnel4\@;$(stunnel4);g' <$(srcdir)/stunnel-rsync.in >stunnel-rsync - @chmod +x stunnel-rsync - stunnel-rsyncd.conf: $(srcdir)/stunnel-rsyncd.conf.in Makefile sed 's;\@bindir\@;$(bindir);g' <$(srcdir)/stunnel-rsyncd.conf.in >stunnel-rsyncd.conf +.PHONY: proto proto: proto.h-tstamp proto.h: proto.h-tstamp @if test -f proto.h; then :; else cp -p $(srcdir)/proto.h .; fi proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h - perl $(srcdir)/mkproto.pl $(srcdir)/*.c $(srcdir)/lib/compat.c + $(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c -man: rsync.1 rsyncd.conf.5 man-copy +.PHONY: man +man: rsync.1 rsync-ssl.1 rsyncd.conf.5 -man-copy: - @if test -f rsync.1; then :; elif test -f $(srcdir)/rsync.1; then echo 'Copying srcdir rsync.1'; cp -p $(srcdir)/rsync.1 .; else echo "NOTE: rsync.1 cannot be created."; fi - @if test -f rsyncd.conf.5; then :; elif test -f $(srcdir)/rsyncd.conf.5; then echo 'Copying srcdir rsyncd.conf.5'; cp -p $(srcdir)/rsyncd.conf.5 .; else echo "NOTE: rsyncd.conf.5 cannot be created."; fi +rsync.1: rsync.1.md md2man NEWS.md Makefile + @$(srcdir)/maybe-make-man $(srcdir) rsync.1.md -rsync.1: rsync.yo - yodl2man -o rsync.1 $(srcdir)/rsync.yo - -$(srcdir)/tweak_manpage rsync.1 +rsync-ssl.1: rsync-ssl.1.md md2man NEWS.md Makefile + @$(srcdir)/maybe-make-man $(srcdir) rsync-ssl.1.md -rsyncd.conf.5: rsyncd.conf.yo - yodl2man -o rsyncd.conf.5 $(srcdir)/rsyncd.conf.yo - -$(srcdir)/tweak_manpage rsyncd.conf.5 +rsyncd.conf.5: rsyncd.conf.5.md md2man NEWS.md Makefile + @$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md +.PHONY: clean clean: cleantests rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \ - rounding rounding.h *.old + rounding rounding.h *.old rsync*.1 rsync*.5 rsync*.html +.PHONY: cleantests cleantests: rm -rf ./testtmp* # We try to delete built files from both the source and build # directories, just in case somebody previously configured things in # the source directory. +.PHONY: distclean distclean: clean rm -f Makefile config.h config.status - rm -f rsync-ssl stunnel-rsync stunnel-rsyncd.conf + rm -f stunnel-rsyncd.conf rm -f lib/dummy popt/dummy zlib/dummy rm -f $(srcdir)/Makefile $(srcdir)/config.h $(srcdir)/config.status rm -f $(srcdir)/lib/dummy $(srcdir)/popt/dummy $(srcdir)/zlib/dummy @@ -251,14 +269,14 @@ distclean: clean # this target is really just for my use. It only works on a limited # range of machines and is used to produce a list of potentially # dead (ie. unused) functions in the code. (tridge) +.PHONY: finddead finddead: nm *.o */*.o |grep 'U ' | awk '{print $$2}' | sort -u > nmused.txt nm *.o */*.o |grep 'T ' | awk '{print $$3}' | sort -u > nmfns.txt comm -13 nmused.txt nmfns.txt # 'check' is the GNU name, 'test' is the name for everybody else :-) -.PHONY: check test - +.PHONY: test test: check # There seems to be no standard way to specify some variables as @@ -271,16 +289,19 @@ test: check # catch Bash-isms earlier even if we're running on GNU. Of course, we # might lose in the future where POSIX diverges from old sh. +.PHONY: check check: all $(CHECK_PROGS) $(CHECK_SYMLINKS) rsync_bin=`pwd`/rsync$(EXEEXT) $(srcdir)/runtests.sh +.PHONY: check29 check29: all $(CHECK_PROGS) $(CHECK_SYMLINKS) rsync_bin=`pwd`/rsync$(EXEEXT) $(srcdir)/runtests.sh --protocol=29 +.PHONY: check30 check30: all $(CHECK_PROGS) $(CHECK_SYMLINKS) rsync_bin=`pwd`/rsync$(EXEEXT) $(srcdir)/runtests.sh --protocol=30 -wildtest.o: wildtest.c lib/wildmatch.c rsync.h config.h +wildtest.o: wildtest.c t_stub.o lib/wildmatch.c rsync.h config.h wildtest$(EXEEXT): wildtest.o lib/compat.o lib/snprintf.o @BUILD_POPT@ $(CC) $(CFLAGS) $(LDFLAGS) -o $@ wildtest.o lib/compat.o lib/snprintf.o @BUILD_POPT@ $(LIBS) @@ -297,6 +318,7 @@ testsuite/xattrs-hlink.test: # check a version installed from a binary or some other source tree, # if you want. +.PHONY: installcheck installcheck: $(CHECK_PROGS) $(CHECK_SYMLINKS) POSIXLY_CORRECT=1 TOOLDIR=`pwd` rsync_bin="$(bindir)/rsync$(EXEEXT)" srcdir="$(srcdir)" $(srcdir)/runtests.sh @@ -307,10 +329,12 @@ installcheck: $(CHECK_PROGS) $(CHECK_SYMLINKS) splint: splint +unixlib +gnuextensions -weak rsync.c +.PHONY: doxygen doxygen: cd $(srcdir) && rm dox/html/* && doxygen # for maintainers only +.PHONY: doxygen-upload doxygen-upload: rsync -avzv $(srcdir)/dox/html/ --delete \ $${SAMBA_HOST-samba.org}:/home/httpd/html/rsync/doxygen/head/ @@ -1,47 +0,0 @@ -NEWS for rsync 3.1.3 (28 Jan 2018) -Protocol: 31 (unchanged) -Changes since 3.1.2: - - SECURITY FIXES: - - Fixed a buffer overrun in the protocol's handling of xattr names and - ensure that the received name is null terminated. - - Fix an issue with --protect-args where the user could specify the arg in - the protected-arg list and short-circuit some of the arg-sanitizing code. - - BUG FIXES: - - - Don't output about a new backup dir without appropriate info verbosity. - - Fixed some issues with the sort functions in support/rsyncstats script. - - Added a way to specify daemon config lists (e.g. users, groups, etc) that - contain spaces (see "auth users" in the latest rsyncd.conf manpage). - - If a backup fails (e.g. full disk) rsync exits with an error. - - Fixed a problem with a doubled --fuzzy option combined with --link-dest. - - Avoid invalid output in the summary if either the start or end time had - an error. - - We don't allow a popt alias to affect the --daemon or --server options. - - Fix daemon exclude code to disallow attribute changes in addition to - disallowing transfers. - - Don't force nanoseconds to match if a non-transferred, non-checksummed - file only passed the quick-check w/o comparing nanosecods. - - ENHANCEMENTS: - - - Added the ability for rsync to compare nanosecond times in its file-check - comparisons, and added support nanosecond times on Mac OS X. - - Added a short-option (-@) for --modify-window. - - Added the --checksum-choice=NAME[,NAME] option to choose the checksum - algorithms. - - Added hashing of xattr names (with using -X) to improve the handling of - files with large numbers of xattrs. - - Added a way to filter xattr names using include/exclude/filter rules (see - the --xattrs option in the manpage for details). - - Added "daemon chroot|uid|gid" to the daemon config (in addition to the - old chroot|uid|gid settings that affect the daemon's transfer process). - - Added "syslog tag" to the daemon configuration. - - Some manpage improvements. - - DEVELOPER RELATED: - - - Tweak the "make" output when yodl isn't around to create the man pages. - - Changed an obsolete autoconf compile macro. - - Support newer yodl versions when converting man pages. diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 00000000..845d7379 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,205 @@ +# NEWS for rsync 3.2.0 (19 Jun 2020) + +Protocol: 31 (unchanged) + +## Changes since 3.1.3: + +### BUG FIXES: + + - Avoid a potential out-of-bounds read in daemon mode if argc can be made to + become 0. + + - Fix the default list of skip-compress files for non-daemon transfers. + + - Fix xattr filter rules losing an 'x' attribute in a non-local transfer. + + - Avoid an error when a check for a potential fuzzy file happens to reference + a directory. + + - Make the atomic-rsync helper script have a more consistent error-exit. + + - Make sure that a signal handler calls `_exit()` instead of exit(). + + - Various zlib fixes, including security fixes for CVE-2016-9843, + CVE-2016-9842, CVE-2016-9841, and CVE-2016-9840. + + - Fixed an issue with `--remove-source-files` not removing a source symlink + when combined with `--copy-links`. + + - Fixed a bug where the daemon would fail to write early fatal error messages + to the client, such as refused or unknown command-line options. + + - Fixed the block-size validation logic when dealing with older protocols. + + - Some rrsync fixes and enhancements to handle the latest options. + + - Fixed a problem with the `--link-dest`|`--copy-dest` code when `--xattrs` + was specified along with multiple alternate-destination directories (it + could possibly choose a bad file match while trying to find a better xattr + match). + + - Fixed a couple bugs in the handling of files with the `--sparse` option. + + - Fixed a bug in the writing of the batch.sh file (w/--write-batch) when the + source & destination args were not last on the command-line. + + - Avoid a hang when an overabundance of messages clogs up all the I/O buffers. + + - Fixed a mismatch in the RSYNC_PID values put into the environment of + `pre-xfer exec` and a `post-xfer exec`. + + - Fixed a crash in the `--iconv` code. + + - Fixed a rare crash in the popt_unalias() code. + +### ENHANCEMENTS: + + - Various checksum enhancements, including the optional use of openssl's MD4 & + MD5 checksum algorithms, some x86-64 optimizations for the rolling checksum, + some x86-64 optimizations for the (non-openssl) MD5 checksum, the addition + of xxhash checksum support, and a negotiation heuristic that ensures that it + is easier to add new checksum algorithms in the future. The environment + variable `RSYNC_CHECKSUM_LIST` can be used to customize the preference order + of the negotiation, or use `--checksum-choice` (`--cc`) to force a choice. + + - Various compression enhancements, including the addition of zstd and lz4 + compression algorithms and a negotiation heuristic that picks the best + compression option supported by both sides. The environment variable + `RSYNC_COMPRESS_LIST` can be used to customize the preference order of the + negotiation, or use `--compress-choice` (`--zc`) to force a choice. + + - Added a --debug=NSTR option that outputs details of the new negotiation + strings (for checksums and compression). The first level just outputs the + result of each negotiation on the client, level 2 outputs the values of the + strings that were sent to and received from the server, and level 3 outputs + all those values on the server side too (when the server was given the debug + option). + + - The --debug=OPTS command-line option is no longer auto-forwarded to the + remote rsync which allows for the client and server to have different levels + of debug specified. This also allows for newer debug options to be + specified, such as using --debug=NSTR to see the negotiated hash result, + without having the command fail if the server version is too old to handle + that debug item. Use -M--debug=OPTS to send the options to the remote side. + + - Added the `--atimes` option based on the long-standing patch (just with some + fixes that the patch has been needing). + + - Added `--open-noatime` option to open files using `O_NOATIME`. + + - Added the `--write-devices` option based on the long-standing patch. + + - Added openssl & preliminary gnutls support to the rsync-ssl script, which is + now installed by default. This was unified with the old stunnel-rsync + helper script to simplify packaging. Note that the script accepts the use + of --type=gnutls for gnutls testing, but does not look for gnutls-cli on the + path yet. The use of type=gnutls will not work right until gnutls-cli no + longer drops data. + + - Rsync was enhanced to set the `RSYNC_PORT` environment variable when running + a daemon-over-rsh script. Its value is the user-specified port number (set + via `--port` or an rsync:// URL) or 0 if the user didn't override the port. + + - Added the `proxy protocol` daemon parameter that allows your rsyncd to know + the real remote IP when it is setup behind a proxy. + + - Added negated matching to the daemon's `refuse options` setting by using + match strings that start with a `!` (such as `!compress*`). This lets you + refuse all options except for a particular approved list, for example. It + also lets rsync refuse certain options by default (such as `write-devices`) + while allowing the config to override that, as desired. + + - Added the `early exec` daemon parameter that runs a script before the + transfer parameters are known, allowing some early setup based on module + name. + + - Added status output in response to a signal (via both SIGINFO & SIGVTALRM). + + - Added `--copy-as=USER` option to give some extra security to root-run rsync + commands into/from untrusted directories (such as backups and restores). + + - When resuming the transfer of a file in the `--partial-dir`, rsync will now + update that partial file in-place instead of creating yet another tmp file + copy. This requires both sender & receiver to be at least v3.2.0. + + - Added support for `RSYNC_SHELL` & `RSYNC_NO_XFER_EXEC` environment variables + that affect the early, pre-xfer, and post-xfer exec rsync daemon parameters. + + - Optimize the `--fuzzy --fuzzy` heuristic to avoid the fuzzy directory scan + until all other basis-file options are exhausted (such as `--link-dest`). + + - Have the daemon log include the normal-exit sent/received stats when the + transfer exited with an error when possible (i.e. if it is the sender). + + - The daemon now locks its pid file (when configured to use one) so that it + will not fail to start when the file exists but no daemon is running. + + - Various man page improvements, including some html representations (that + aren't installed by default). + + - Made -V the short option for --version and improved its information. + + - Pass the -4 or -6 option to the ssh command, making it easier to type than + `--rsh='ssh -4'` (or -6). + + - Added example config for rsyncd SSL proxy configs to rsyncd.conf. + + - More errors messages now mention if the error is coming from the sender or + the receiver. + +### PACKAGING RELATED: + + - Add installed binary: /usr/bin/rsync-ssl + + - Add installed man page: /usr/man/man1/rsync-ssl.1 + + - Tweak auxiliary doc file names, such as: README.md, INSTALL.md, NEWS.md, & + OLDNEWS.md. + + - The rsync-ssl script wants to run openssl or stunnel4, so consider adding a + dependency for one of those options (though it's probably fine to just let + it complain about being unable to find the program and let the user decide + if they want to install one or the other). + + - If you packaged rsync + rsync-ssl + rsync-ssl-daemon as separate packages, + the rsync-ssl package is now gone (rsync-ssl should be considered to be + mainstream now that Samba requires SSL for its rsync daemon). + + - Add _build_ dependency for liblz4-dev, libxxhash-dev, libzstd-dev, and + libssl-dev. These development libraries will give rsync extra compression + algorithms, extra checksum algorithms, and allow use of openssl's crypto + lib for (potentially) faster MD4/MD5 checksums. + + - Add _build_ dependency for g++ or clang++ on x86_64 systems to enable the + SIMD checksum optimizations. + + - Add _build_ dependency for _either_ python3-cmarkcfm or python3-commonmark + to allow for patching of man pages or building a git release. This is not + required for a release-tar build, since it comes with pre-built man pages. + Note that cmarkcfm is faster than commonmark, but they generate the same + data. The commonmark dependency is easiest to install since it's native + python, and can be installed via `pip3 install --user commonmark` if you + want to just install it for the build user (or omit `--user`). + + - Remove yodl _build_ dependency (if it was even listed before). + +### DEVELOPER RELATED: + + - Silenced some annoying warnings about major() & minor() by improving an + autoconf include-file check. + + - Converted the man pages from yodl to markdown. They are now processed via a + simple python3 script using the cmarkgfm **or** commonmark library. This + should make it easier to package rsync, since yodl has gotten obscure. + + - Improved some configure checks to work better with strict C99 compilers. + + - Some perl building/packaging scripts were recoded into awk and python3. + + - Some defines in byteorder.h were changed into static inline functions that + will help to ensure that the args don't get evaluated multiple times on + "careful alignment" hosts. + + - Some code typos were fixed (as pointed out by a Fossies run). + +------------------------------------------------------------------------------ diff --git a/OLDNEWS b/OLDNEWS deleted file mode 100644 index 3665794d..00000000 --- a/OLDNEWS +++ /dev/null @@ -1,3765 +0,0 @@ -NEWS for rsync 3.1.2 (21 Dec 2015) -Protocol: 31 (unchanged) -Changes since 3.1.1: - - SECURITY FIXES: - - - Make sure that all transferred files use only path names from inside the - transfer. This makes it impossible for a malicious sender to try to make - the receiver use an unsafe destination path for a transferred file, such - as a just-sent symlink. - - BUG FIXES: - - - Change the checksum seed order in the per-block checksums. This prevents - someone from trying to create checksum blocks that match in sum but not - content. - - Fixed a with the per-dir filter files (using -FF) that could trigger an - assert failure. - - Only skip set_modtime() on a transferred file if the time is exactly - right. - - Don't create an empty backup dir for a transferred file that doesn't - exist yet. - - Fixed a bug where --link-dest and --xattrs could cause rsync to exit if - a filename had a matching dir of the same name in the alt-dest area. - - Allow more than 32 group IDs per user in the daemon's gid=LIST config. - - Fix the logging of %b & %c via --log-file (daemon logging was already - correct, as was --out-format='%b/%c'). - - Fix erroneous acceptance of --info=5 & --debug=5 (an empty flag name is - not valid). - - ENHANCEMENTS: - - - Added "(DRY RUN)" info to the --debug=exit output line. - - Use usleep() for our msleep() function if it is available. - - Added a few extra long-option names to rrsync script, which will make - BackupPC happier. - - Made configure choose to use linux xattrs on netbsd (rather than not - supporting xattrs). - - Added -wo (write-only) option to rrsync support script. - - Misc. manpage tweaks. - - DEVELOPER RELATED: - - - Fixed a bug with the Makefile's use of INSTALL_STRIP. - - Improve a test in the suite that could get an erroneous timestamp error. - - Tweaks for newer versions of git in the packaging tools. - - Improved the m4 generation rules and some autoconf idioms. - -NEWS for rsync 3.1.1 (22 Jun 2014) -Protocol: 31 (unchanged) -Changes since 3.1.0: - - BUG FIXES: - - - If the receiver gets bogus filenames from the sender (an unexpected - leading slash or a ".." infix dir), exit with an error. This prevents a - malicious sender from trying to inject filenames that would affect an - area outside the destination directories. - - - Fixed a failure to remove the partial-transfer temp file when interrupted - (and rsync is not saving the partial files). - - - Changed the chown/group/xattr-set order to avoid losing some security- - related xattr info (that would get cleared by a chown). - - - Fixed a bug in the xattr-finding code that could make a non-root-run - receiver not able to find some xattr numbers. - - - Fixed a bug in the early daemon protocol where a timeout failed to be - honored (e.g. if the remote side fails to send us the initial protocol - greeting). - - - Fixed unintended inclusion of commas in file numbers in the daemon log. - - - We once again send the 'f' sub-flag (of -e) to the server side so it - knows that we can handle incremental-recursion directory errors properly - in older protocols. - - - Fixed an issue with too-aggressive keep-alive messages causing a problem - for older rsync versions early in the transfer. - - - Fixed an incorrect message about backup-directory-creation when using - --dry-run and the backup dir is not an absolute path. - - - Fixed a bug where a failed deletion and/or a failed sender-side removal - would not affect the exit code. - - - Fixed a bug that caused a failure when combining --delete-missing-args - with --xattrs and/or --acls. - - - Fixed a strange dir_depth assertion error that was caused by empty-dir - removals and/or duplicate files in the transfer. - - - Fixed a problem with --info=progress2's output stats where rsync would - only update the stats at the end of each file's transfer. It now uses - the data that is flowing for the current file, making the stats more - accurate and less jumpy. - - - Fixed an itemize bug that affected the combo of --link-dest, -X, and -n. - - - Fixed a problem with delete messages not appearing in the log file when - the user didn't use --verbose. - - - Improve chunked xattr reading for OS X. - - - Removed an attempted hard-link xattr optimization that was causing a - transfer failure. This removal is flagged in the compatibility code, so - if a better fix can be discovered, we have a way to flip it on again. - - - Fixed a bug when the receiver is not configured to be able to hard link - symlimks/devices/special-file items but the sender sent some of these - items flagged as hard-linked. - - - We now generate a better error if the buffer overflows in do_mknod(). - - - Fixed a problem reading more than 16 ACLs on some OSes. - - - Fixed the reading of the secrets file to avoid an infinite wait when - the username is missing. - - - Fixed a parsing problem in the --usermap/--groupmap options when using - MIN-MAX numbers. - - - Switched Cygwin back to using socketpair "pipes" to try to speed it up. - - - Added knowledge of a few new options to rrsync. - - ENHANCEMENTS: - - - Tweaked the temp-file naming when --temp-dir=DIR is used: the temp-file - names will not get a '.' prepended. - - - Added support for a new-compression idiom that does not compress all the - matching data in a transfer. This can help rsync to use less cpu when a - transfer has a lot of matching data, and also makes rsync compatible with - a non-bundled zlib. See the --new-compress and --old-compress options in - the manpage. - - - Added the support/rsync-no-vanished wrapper script. - - - Made configure more prominently mention when we failed to find yodl (in - case the user wants to be able to generate manpages from *.yo files). - - - Have manpage mention how a daemon's max-verbosity setting affects info - and debug options. Also added more clarification on backslash removals - for excludes that contain wildcards. - - - Have configure check if for the attr lib (for getxattr) for those systems - that need to link against it explicitly. - - - Change the early dir-creation logic to only use that idiom in an - inc-recursive copy that is preserving directory times. e.g. using - --omit-dir-times will avoid these early directories being created. - - - Fix a bug in cmp_time() that would return a wrong result if the 2 times - differed by an amount greater than what a time_t can hold. - - DEVELOPER RELATED: - - - We now include an example systemd file (in packaging/systemd). - - - Tweaked configure to make sure that any intended use of the included popt - and/or zlib code is put early in the CFLAGS. - -NEWS for rsync 3.1.0 (28 Sep 2013) -Protocol: 31 (changed) -Changes since 3.0.9: - - OUTPUT CHANGES: - - - Output numbers in 3-digit groups by default (e.g. 1,234,567). See the - --human-readable option for a way to turn it off. See also the daemon's - "log format" parameter and related command-line options (including - --out-format) for a modifier that can be used to request digit-grouping - or human-readable output in log escapes. (Note that log output is - unchanged by default.) - - - The --list-only option is now affected by the --human-readable setting. - It will display digit groupings by default, and unit suffixes if higher - levels of readability are requested. Also, the column width for the size - output has increased from 11 to 14 characters when human readability is - enabled. Use --no-h to get the old-style output and column size. - - - The output of the --progress option has changed: the string "xfer" was - shortened to "xfr", and the string "to-check" was shortened to "to-chk", - both designed to make room for the (by default) wider display of file - size numbers without making the total line-length longer. Also, when - incremental recursion is enabled, the string "ir-chk" will be used - instead of "to-chk" up until the incremental-recursion scan is done, - letting you know that the value to check and the total value will still - be increasing as new files are found. - - - Enhanced the --stats output: 1) to mention how many files were created - (protocol >= 28), 2) to mention how many files were deleted (a new line - for protocol 31, but only output when --delete is in effect), and 3) to - follow the file-count, created-count, and deleted-count with a subcount - list that shows the counts by type. The wording of the transferred count - has also changed so that it is clearer that it is only a count of regular - files. - - BUG FIXES: - - - Fixed a bug in the iconv code when EINVAL or EILSEQ is returned with a - full output buffer. - - - Fixed some rare bugs in --iconv processing that might cause a multibyte - character to get translated incorrectly. - - - Fixed a bogus "vanished file" error if some files were specified with - "./" prefixes and others were not. - - - Fixed a bug in --sparse where an extra gap could get inserted after a - partial write. - - - Changed the way --progress overwrites its prior output in order to make - it nearly impossible for the progress to get overwritten by an error. - - - Improved the propagation of abnormal-exit error messages. This should - help the client side to receive errors from the server when it is exiting - abnormally, and should also avoid dying with an "connection unexpectedly - closed" exit when the closed connection is really expected. - - - The sender now checks each file it plans to remove to ensure that it - hasn't changed from the first stat's info. This helps to avoid losing - file data when the user is not using the option in a safe manner. - - - Fixed a data-duplication bug in the compress option that made compression - less efficient. This improves protocol 31 onward, while behaving in a - compatible (buggy) manner with older rsync protocols. - - - When creating a temp-file, rsync is now a bit smarter about it dot-char - choices, which can fix a problem on OS X with names that start with "..". - - - Rsync now sets a cleanup flag for --inplace and --append transfers that - will flush the write buffer if the transfer aborts. This ensures that - more received data gets written out to the disk on an aborted transfer - (which is quite helpful on a slow, flaky connection). - - - The reads that map_ptr() now does are aligned on 1K boundaries. This - helps some filesystems and/or files that don't like unaligned reads. - - - Fix an issue in the msleep() function if time jumps backwards. - - - Fix daemon-server module-name splitting bug where an arg would get split - even if --protect-args was used. - - ENHANCEMENTS: - - - Added the --remote-option=OPT (-M OPT) command-line option that is useful - for things like sending a remote --log-file=FILE or --fake-super option. - - - Added the --info=FLAGS and --debug=FLAGS options to allow finer-grained - control over what is output. Added an extra type of --progress output - using --info=progress2. - - - The --msgs2stderr option can help with debugging rsync by allowing the - debug messages to get output to stderr rather than travel via the socket - protocol. - - - Added the --delete-missing-args and --ignore-missing-args options to - either delete or ignore user-specified files on the receiver that are - missing on the sender (normally the absence of user-specified files - generates an error). - - - Added a "T" (terabyte) category to the --human-readable size suffixes. - - - Added the --usermap/--groupmap/--chown options for manipulating file - ownership during the copy. - - - Added the "%C" escape to the log-output handling, which will output the - MD5 checksum of any transferred file, or all files if --checksum was - specified (when protocol 30 or above is in effect). - - - Added the "reverse lookup" parameter to the rsync daemon config file to - allow reverse-DNS lookups to be disabled. - - - Added a forward-DNS lookup for the daemon's hosts allow/deny config. Can - be disabled via "forward lookup" parameter (defaults to enabled). - - - Added a way for more than one group to be specified in the daemon's - config file, including a way to specify that you want all of the - specified user's groups without having to name them. Also changed the - daemon to complain about an inability to set explicitly-specified uid/gid - values, even when not run by a super-user. - - - The daemon now tries to send the user the error messages from the - pre-xfer exec script when it fails. - - - Improved the use of alt-dest options into an existing hierarchy of files: - If a match is found in an alt-dir, it takes precedence over an existing - file. (We'll need to wait for a future version before attribute-changes - on otherwise unchanged files are safe when using an existing hierarchy.) - - - Added per-user authorization options and group-authorization support to - the daemon's "auth users" parameter. - - - Added a way to reference environment variables in a daemon's config file - (using %VAR% references). - - - When replacing a non-dir with a symlink/hard-link/device/special-file, - the update should now be done in an atomic manner. - - - Avoid re-sending xattr info for hard-linked files w/the same xattrs - (protocol 31). - - - The backup code was improved to use better logic maintaining the backup - directory hierarchy. Also, when a file is being backed up, rsync tries - to hard-link it into place so that the upcoming replacement of the - destination file will be atomic (for the normal, non-inplace logic). - - - Added the ability to synchronize nano-second modified times. - - - Added a few more default suffixes for the "dont compress" settings. - - - Added the checking of the RSYNC_PROTECT_ARGS environment variable to allow - the default for the --protect-args command-line option to be overridden. - - - Added the --preallocate command-line option. - - - Allow --password-file=- to read the password from stdin (filename "-"). - - - Rsync now comes packaged with an rsync-ssl helper script that can be - used to contact a remote rsync daemon using a piped-stunnel command. - It also includes an stunnel config file to run the server side to - support ssl daemon connections. See the packaging/lsb/rsync.spec - file for one way to package the resulting files. (Suggestions for - how to make this even easier to install & use are welcomed.) - - - Improved the speed of some --inplace updates when there are lots of - identical checksum blocks that end up being unusable. - - - Added the --outbuf=N|L|B option for choosing the output buffering. - - - Repeating the --fuzzy option now causes the code to look for fuzzy - matches inside alt-dest directories too. - - - The --chmod option now supports numeric modes, e.g. --chmod=644,D755 - - - Added some Solaris xattr code. - - - Made an rsync daemon (the listening process) exit with a 0 status when - it was signaled to die. This helps launchd. - - - Improved the RSYNC_* environment variables for the pre-xfer exec script: - when a daemon is sent multiple request args, they are now joined into a - single return value (separated by spaces) so that the RSYNC_REQUEST - environment variable is accurate for any "pre-xfer exec". The values in - RSYNC_ARG# vars are no longer truncated at the "." arg (prior to the - request dirs/files), so that all the requested values are also listed - (separately) in RSYNC_ARG# variables. - - EXTRAS: - - - Added an "instant-rsyncd" script to the support directory, which makes - it easy to configure a simple rsync daemon in the current directory. - - - Added the "mapfrom" and "mapto" scripts to the support directory, which - makes it easier to do user/group mapping in a local transfer based on - passwd/group files from another machine. - - - There's a new, improved version of the lsh script in the support dir: - it's written in perl and supports -u without resorting to using sudo - (when run as root). The old shell version is now named lsh.sh. - - - There is a helper script named rsync-slash-strip in the support directory - for anyone that wants to change the way rsync handles args with trailing - slashes. (e.g. arg/ would get stripped to arg while arg/. would turn into - arg/). - - INTERNAL: - - - The I/O code was rewritten to be simpler and do bigger buffered reads - over the socket. The I/O between the receiver and the generator was - changed to be standard multiplexed-I/O (like that over the socket). - - - The sender tries to use any dead time while the generator is looking for - files to transfer in order to do sender-side directory scanning in a more - parallel manner. - - - A daemon can now inform a client about a daemon-configured timeout value - so that the client can assist in the keep-alive activity (protocol 31). - - - The filter code received some refactoring to make it more extendible, to - read better, and do better sanity checking. - - - Really big numbers are now output using our own big-num routine rather - than casting them to a double and using a %.0f conversion. - - - The pool_alloc library has received some minor improvements in alignment - handling. - - - Added init_stat_x() function to avoid duplication of acl/xattr init code. - - - The included zlib was upgraded from 1.2.3 to 1.2.8. - - - Rsync can now be compiled to use an unmodified zlib library instead of - the tweaked one that is included with rsync. This will eventually - become the default, at which point we'll start the countdown to removing - the included zlib. Until then, feel free to configure using: - - ./configure --with-included-zlib=no - - DEVELOPER RELATED: - - - Added more conditional debug output. - - - Fixed some build issues for android and minix. - -NEWS for rsync 3.0.9 (23 Sep 2011) -Protocol: 30 (unchanged) -Changes since 3.0.8: - - BUG FIXES: - - - Fix a crash bug in checksum scanning when --inplace is used. - - - Fix a hang if a hard-linked file cannot be opened by the sender (e.g. - if it has no read permission). - - - Fix preservation of a symlink's system xattrs (e.g. selinux) on Linux. - - - Fix a memory leak in the xattr code. - - - Fixed a bug with --delete-excluded when a filter merge file has a rule - that specifies a receiver-only side restriction. - - - Fix a bug with the modifying of unwritable directories. - - - Fix --fake-super's interaction with --link-dest same-file comparisons. - - - Fix the updating of the curr_dir buffer to avoid a duplicate slash. - - - Fix the directory permissions on an implied dot-dir when using --relative - (e.g. /outside/path/././send/path). - - - Fixed some too-long sleeping instances when using --bwlimit. - - - Fixed when symlink ownership difference-checking gets compiled into - unchanged_attrs(). - - - Improved the socket-error reporting when multiple protocols fail. - - - Fixed a case where a socket error could reference just-freed memory. - - - Failing to use a password file that was specified on the command-line is - now a fatal error. - - - Fix the non-root updating of directories that don't have the read and/or - execute permission. - - - Make daemon-excluded file errors more error-like. - - - Fix a compilation issue on older C compilers (due to a misplaced var - declaration). - - - Make configure avoid finding socketpair on cygwin. - - - Avoid trying to reference SO_BROADCAST if the OS doesn't support it. - - - Fix some issues with the post-processing of the man pages. - - - Fixed the user home-dir handling in the support/lsh script. - - - Some minor manpage improvements. - -NEWS for rsync 3.0.8 (26 Mar 2011) -Protocol: 30 (unchanged) -Changes since 3.0.7: - - BUG FIXES: - - - Fixed two buffer-overflow issues: one where a directory path that is - exactly MAXPATHLEN was not handled correctly, and one handling a - --backup-dir that is extra extra large. - - - Fixed a data-corruption issue when preserving hard-links without - preserving file ownership, and doing deletions either before or during - the transfer (CVE-2011-1097). This fixes some assert errors in the - hard-linking code, and some potential failed checksums (via -c) that - should have matched. - - - Fixed a potential crash when an rsync daemon has a filter/exclude list - and the transfer is using ACLs or xattrs. - - - Fixed a hang if a really large file is being processed by an rsync that - can't handle 64-bit numbers. Rsync will now complain about the file - being too big and skip it. - - - For devices and special files, we now avoid gathering useless ACL and/or - xattr information for files that aren't being copied. (The un-copied - files are still put into the file list, but there's no need to gather - data that is not going to be used.) This ensures that if the user uses - --no-D, that rsync can't possibly complain about being unable to gather - extended information from special files that are in the file list (but - not in the transfer). - - - Properly handle requesting remote filenames that start with a dash. This - avoids a potential error where a filename could be interpreted as a - (usually invalid) option. - - - Fixed a bug in the comparing of upper-case letters in file suffixes for - --skip-compress. - - - If an rsync daemon has a module configured without a path setting, rsync - will now disallow access to that module. - - - If the destination arg is an empty string, it will be treated as a - reference to the current directory (as 2.x used to do). - - - If rsync was compiled with a newer time-setting function (such as - lutimes), rsync will fall-back to an older function (such as utimes) on a - system where the newer function is not around. This helps to make the - rsync binary more portable in mixed-OS-release situations. - - - Fixed a batch-file writing bug that would not write out the full set of - compatibility flags that the transfer was using. This fixes a potential - protocol problem for a batch file that contains a sender-side I/O error: - it would have been sent in a way that the batch-reader wasn't expecting. - - - Some improvements to the hard-linking code to ensure that device-number - hashing is working right, and to supply more information if the hard-link - code fails. - - - The --inplace code was improved to not search for an impossible checksum - position. The quadruple-verbose chunk[N] message will now mention when - an inplace chunk was handled by a seek rather than a read+write. - - - Improved ACL mask handling, e.g. for Solaris. - - - Fixed a bug that prevented --numeric-ids from disabling the translation - of user/group IDs for ACLs. - - - Fixed an issue where an xattr and/or ACL transfer that used an alt-dest - option (e.g. --link-dest) could output an error trying to itemize the - changes against the alt-dest directory's xattr/ACL info but was instead - trying to access the not-yet-existing new destination directory. - - - Improved xattr system-error messages to mention the full path to the - file. - - - The --link-dest checking for identical symlinks now avoids considering - attribute differences that cannot be changed on the receiver. - - - Avoid trying to read/write xattrs on certain file types for certain OSes. - Improved configure to set NO_SYMLINK_XATTRS, NO_DEVICE_XATTRS, and/or - NO_SPECIAL_XATTRS defines in config.h. - - - Improved the unsafe-symlink errors messages. - - - Fixed a bug setting xattrs on new files that aren't user writable. - - - Avoid re-setting xattrs on a hard-linked file w/the same xattrs. - - - Fixed a bug with --fake-super when copying files and dirs that aren't - user writable. - - - Fixed a bug where a sparse file could have its last sparse block turned - into a real block when rsync sets the file size (requires ftruncate). - - - If a temp-file name is too long, rsync now avoids truncating the name in - the middle of adjacent high-bit characters. This prevents a potential - filename error if the filesystem doesn't allow a name to contain an - invalid multi-byte sequence. - - - If a muli-protocol socket connection fails (i.e., when contacting a - daemon), we now report all the failures, not just the last one. This - avoids losing a relevant error (e.g. an IPv4 connection-refused error) - that happened before the final error (e.g. an IPv6 protocol-not-supported - error). - - - Generate a transfer error if we try to call chown with a -1 for a uid or - a gid (which is not settable). - - - Fixed the working of --force when used with --one-file-system. - - - Fix the popt arg parsing so that an option that doesn't take an arg will - reject an attempt to supply one (can configure --with-included-popt if - your system's popt library doesn't yet have this fix). - - - A couple minor option tweaks to the support/rrsync script, and also some - regex changes that make vim highlighting happier. - - - Fixed some issues in the support/mnt-excl script. - - - Various manpage improvements. - - ENHANCEMENTS: - - - Added ".hg/" to the default cvs excludes (see -C & --cvs-exclude). - - DEVELOPER RELATED: - - - Use lchmod() whenever it is available (not just on symlinks). - - - A couple fixes to the socketpair_tcp() routine. - - - Updated the helper scripts in the packaging subdirectory. - - - Renamed configure.in to configure.ac. - - - Fixed configure's checking for iconv routines for newer OS X versions. - - - Fixed the testsuite/xattrs.test script on OS X. - -NEWS for rsync 3.0.7 (31 Dec 2009) -Protocol: 30 (unchanged) -Changes since 3.0.6: - - BUG FIXES: - - - Fixed a bogus free when using --xattrs with --backup. - - - Avoid an error when --dry-run was trying to stat a prior hard-link file - that hasn't really been created. - - - Fixed a problem with --compress (-z) where the receiving side could - return the error "inflate (token) returned -5". - - - Fixed a bug where --delete-during could delete in a directory before it - noticed that the sending side sent an I/O error for that directory (both - sides of the transfer must be at least 3.0.7). - - - Improved --skip-compress's error handling of bad character-sets and got - rid of a lingering debug fprintf(). - - - Fixed the daemon's conveyance of io_error value from the sender. - - - An rsync daemon use seteuid() (when available) if it used setuid(). - - - Get the permissions right on a --fake-super transferred directory that - needs more owner permissions to emulate root behavior. - - - An absolute-path filter rule (i.e. with a '/' modifier) no longer loses - its modifier when sending the filter rules to the remote rsync. - - - Improved the "--delete does not work without -r or -d" message. - - - Improved rsync's handling of --timeout to avoid a weird timeout case - where the sender could timeout even though it has recently written data - to the socket (but hasn't read data recently, due to the writing). - - - Some misc manpage improvements. - - - Fixed the chmod-temp-dir testsuite on a system without /var/tmp. - - - Make sure that a timeout specified in the daemon's config is used as a - maximum timeout value when the user also specifies a timeout. - - - Improved the error-exit reporting when rsync gets an error trying to - cleanup after an error: the initial error is reported. - - - Improved configure's detection of IPv6 for solaris and cygwin. - - - The AIX sysacls routines will now return ENOSYS if ENOTSUP is missing. - - - Made our (only used if missing) getaddrinfo() routine use inet_pton() - (which we also provide) instead of inet_aton(). - - - The exit-related debug messages now mention the program's role so it is - clear who output what message. - - DEVELOPER RELATED: - - - Got rid of type-punned compiler warnings output by newer gcc versions. - - - The Makefile now ensures that proto.h will be rebuilt if config.h changes. - - - The testsuite no longer uses "id -u", so it works better on solaris. - - -NEWS for rsync 3.0.6 (8 May 2009) -Protocol: 30 (unchanged) -Changes since 3.0.5: - - BUG FIXES: - - - Fixed a --read-batch hang when rsync is reading a batch file that was - created from an incremental-recursion transfer. - - - Fixed the daemon's socket code to handle the simultaneous arrival of - multiple connections. - - - Fix --safe-links/--copy-unsafe-links to properly handle symlinks that - have consecutive slashes in the value. - - - Fixed the parsing of an [IPv6_LITERAL_ADDR] when a USER@ is prefixed. - - - The sender now skips a (bogus) symlink that has a 0-length value, which - avoids a transfer error in the receiver. - - - Fixed a case where the sender could die with a tag-0 error if there was - an I/O during the sending of the file list. - - - Fixed the rrsync script to avoid a server-side problem when -e is at the - start of the short options. - - - Fixed a problem where a vanished directory could turn into an exit code - 23 instead of the proper exit code 24. - - - Fixed the --iconv conversion of symlinks when doing a local copy. - - - Fixed a problem where --one-file-system was not stopping deletions on the - receiving side when a mount-point directory did not match a directory in - the transfer. - - - Fixed the dropping of an ACL mask when no named ACL values were present. - - - Fixed an ACL/xattr corruption issue where the --backup option could cause - rsync to associate the wrong ACL/xattr information with received files. - - - Fixed the use of --xattrs with --only-write-batch. - - - Fixed the use of --dry-run with --read-batch. - - - Fixed configure's erroneous use of target. - - - Fixed configure's --disable-debug option. - - - Fixed a run-time issue for systems that can't find iconv_open() by adding - the --disable-iconv-open configure option. - - - Complain and die if the user tries to combine --remove-source-files (or - the deprecated --remove-sent-files) with --read-batch. - - - Fixed an failure transferring special files from Solaris to Linux. - - -NEWS for rsync 3.0.5 (28 Dec 2008) -Protocol: 30 (unchanged) -Changes since 3.0.4: - - BUG FIXES: - - - Initialize xattr data in a couple spots in the hlink code, which avoids a - crash when the xattr pointer's memory happens to start out non-zero. - Also fixed the itemizing of an alt-dest file's xattrs when hard-linking. - - - Don't send a bogus "-" option to an older server if there were no short - options specified. - - - Fixed skipping of unneeded updates in a batch file when incremental - recursion is active. Added a test for this. Made batch-mode handle - "redo" files properly (and without hanging). - - - Fix the %P logfile escape when the daemon logs from inside a chroot. - - - Fixed the use of -s (--protect-args) when used with a remote source or - destination that had an empty path (e.g. "host:"). Also fixed a problem - when -s was used when accessing a daemon via a remote-shell. - - - Fixed the use of a dot-dir path (e.g. foo/./bar) inside a --files-from - file when the root of the transfer isn't the current directory. - - - Fixed a bug with "-K --delete" removing symlinks to directories when - incremental recursion is active. - - - Fixed a hard to trigger hang when using --remove-source-files. - - - Got rid of an annoying delay when accessing a daemon via a remote-shell. - - - Properly ignore (superfluous) source args on a --read-batch command. - - - Improved the manpage's description of the '*' wildcard to remove the - confusing "non-empty" qualifier. - - - Fixed reverse lookups in the compatibility-library version of - getnameinfo(). - - - Fixed a bug when using --sparse on a sparse file that has over 2GB of - consecutive sparse data. - - - Avoid a hang when using at least 3 --verbose options on a transfer with a - client sender (which includes local copying). - - - Fixed a problem with --delete-delay reporting an error when it was ready - to remove a directory that was now gone. - - - Got rid of a bunch of "warn_unused_result" compiler warnings. - - - If an ftruncate() on a received file fails, it now causes a partial- - transfer warning. - - - Allow a path with a leading "//" to be preserved (CYGWIN only). - - ENHANCEMENTS: - - - Made the support/atomic-rsync script able to perform a fully atomic - update of the copied hierarchy when the destination is setup using a - particular symlink idiom. - - -NEWS for rsync 3.0.4 (6 Sep 2008) -Protocol: 30 (unchanged) -Changes since 3.0.3: - - BUG FIXES: - - - Fixed a bug in the hard-linking code where it would sometimes try to - allocate 0 bytes of memory (which fails on some OSes, such as AIX). - - - Fixed the hard-linking of files from a device that has a device number - of 0 (which seems to be a common device number on NetBSD). - - - Fixed the handling of a --partial-dir that cannot be created. This - particularly impacts the --delay-updates option (since the files cannot - be delayed without a partial-dir), and was potentially destructive if - the --remove-source-files was also specified. - - - Fixed a couple issues in the --fake-super handling of xattrs when the - destination files have root-level attributes (e.g. selinux values) that - a non-root copy can't affect. - - - Improved the keep-alive check in the generator to fire consistently in - incremental-recursion mode when --timeout is enabled. - - - The --iconv option now converts the content of a symlink too, instead - of leaving it in the wrong character-set (requires 3.0.4 on both sides - of the transfer). - - - When using --iconv, if a filename fails to convert on the receiving side, - this no longer makes deletions in the root-dir of the transfer fail - silently (the user now gets a warning about deletions being disabled - due to IO error as long as --ignore-errors was not specified). - - - When using --iconv, if a server-side receiver can't convert a filename, - the error message sent back to the client no longer mangles the name - with the wrong charset conversion. - - - Fixed a potential alignment issue in the IRIX ACL code when allocating - the initial "struct acl" object. Also, cast mallocs to avoid warnings. - - - Changed some errors that were going to stdout to go to stderr. - - - Made human_num() and human_dnum() able to output a negative number - (rather than outputting a cryptic string of punctuation). - - ENHANCEMENTS: - - - Rsync will avoid sending an -e option to the server if an older protocol - is requested (and thus the option would not be useful). This lets the - user specify the --protocol=29 option to access an overly-restrictive - server that is rejecting the protocol-30 use of -e to the server. - - - Improved the message output for an RERR_PARTIAL exit. - - DEVELOPER RELATED: - - - The Makefile will not halt for just a timestamp change on the Makefile - or the configure files, only for actual changes in content. - - - Changed some commands in the testsuite's xattrs.test that called "rsync" - instead of "$RSYNC". - - - Enhanced the release scripts to be able to handle a branch release and - to do even more consistency checks on the files. - - -NEWS for rsync 3.0.3 (29 Jun 2008) -Protocol: 30 (unchanged) -Changes since 3.0.2: - - BUG FIXES: - - - Fixed a wildcard matching problem in the daemon when a module has - "use chroot" enabled. - - - Fixed a crash bug in the hard-link code. - - - Fixed the sending of xattr directory information when the code finds a - --link-dest or --copy-dest directory with unchanged xattrs -- the - destination directory now gets these unchanged xattrs properly applied. - - - Fixed an xattr-sending glitch that could cause an "Internal abbrev" - error. - - - Fixed the combination of --xattrs and --backup. - - - The generator no longer allows a '.' dir to be excluded by a daemon- - exclude rule. - - - Fixed deletion handling when copying a single, empty directory (with no - files) to a differently named, non-existent directory. - - - Fixed the conversion of spaces into dashes in the %M log escape. - - - Fixed several places in the code that were not returning the right - errno when a function failed. - - - Fixed the backing up of a device or special file into a backup dir. - - - Moved the setting of the socket options prior to the connect(). - - - If rsync exits in the middle of a --progress output, it now outputs a - newline to help prevent the progress line from being overwritten. - - - Fixed a problem with how a destination path with a trailing slash or - a trailing dot-dir was compared against the daemon excludes. - - - Fixed the sending of large (size > 16GB) files when talking to an older - rsync (protocols < 30): we now use a compatible block size limit. - - - If a file's length is so huge that we overflow a checksum buffer count - (i.e. several hundred TB), warn the user and avoid sending an invalid - checksum struct over the wire. - - - If a source arg is excluded, --relative no longer adds the excluded - arg's implied dirs to the transfer. This fix also made the exclude - check happen in the better place in the sending code. - - - Use the overflow_exit() function for overflows, not out_of_memory(). - - - Improved the code to better handle a system that has only 32-bit file - offsets. - - ENHANCEMENTS: - - - The rsyncd.conf manpage now consistently refers to the parameters in - the daemon config file as "parameters". - - - The description of the --inplace option was improved. - - EXTRAS: - - - Added a new script in the support directory, deny-rsync, which allows - an admin to (temporarily) replace the rsync command with a script that - sends an error message to the remote client via the rsync protocol. - - DEVELOPER RELATED: - - - Fixed a testcase failure if the tests are run as root and made some - compatibility improvements. - - - Improved the daemon tests, including checking module comments, the - listing of files, and the ensuring that daemon excludes can't affect - a dot-dir arg. - - - Improved some build rules for those that build in a separate directory - from the source, including better install rules for the man pages, and - the fixing of a proto.h-tstamp rule that could make the binaries get - rebuild without cause. - - - Improved the testsuite to work around a problem with some utilities - (e.g. cp -p & touch -r) rounding sub-second timestamps. - - - Ensure that the early patches don't cause any generated-file hunks to - bleed-over into patches that follow. - - -NEWS for rsync 3.0.2 (8 Apr 2008) -Protocol: 30 (unchanged) -Changes since 3.0.1: - - BUG FIXES: - - - Fixed a potential buffer overflow in the xattr code. - - ENHANCEMENTS: - - - None. - - DEVELOPER RELATED: - - - The RPM spec file was improved to install more useful files. - - - A few developer-oriented scripts were moved from the support dir - to the packaging dir. - - -NEWS for rsync 3.0.1 (3 Apr 2008) -Protocol: 30 (unchanged) -Changes since 3.0.0: - - NOTABLE CHANGES IN BEHAVIOR: - - - Added the 'c'-flag to the itemizing of non-regular files so that the - itemized output doesn't get hidden if there were no attribute changes, - and also so that the itemizing of a --copy-links run will distinguish - between copying an identical non-regular file and the creation of a - revised version with a new value (e.g. a changed symlink referent, a - new device number, etc.). - - BUG FIXES: - - - Fixed a crash bug when a single-use rsync daemon (via remote shell) was - run without specifying a --config=FILE option. - - - Fixed a crash when backing up a directory that has a default ACL. - - - Fixed a bug in the handling of xattr values that could cause rsync to - not think that a file's extended attributes are up-to-date. - - - Fixed the working of --fake-super with --link-dest and --xattrs. - - - Fixed a hang when combining --dry-run with --remove-source-files. - - - Fixed a bug with --iconv's handling of files that cannot be converted: - a failed name can no longer cause a transfer failure. - - - Fixed the building of the rounding.h file on systems that need custom - CPPFLAGS to be used. Also improved the error reporting if the building - of rounding.h fails. - - - Fixed the use of the --protect-args (-s) option when talking to a daemon. - - - Fixed the --ignore-existing option's protection of files on the receiver - that are non-regular files on the sender (e.g. if a symlink or a dir on - the sender is trying to replace a file on the receiver). The reverse - protection (protecting a dir/symlink/device from being replaced by a - file) was already working. - - - Fixed an assert failure if --hard-links is combined with an option that - can skip a file in a set of hard-linked files (i.e. --ignore-existing, - --append, etc.), without skipping all the files in the set. - - - Avoid setting the modify time on a directory that already has the right - modify time set. This avoids tweaking the dir's ctime. - - - Improved the daemon-exclude handling to do a better job of applying the - exclude rules to path entries. It also sends the user an error just as - if the files were actually missing (instead of silently ignoring the - user's args), and avoids sending the user the filter-action messages - for these non-user-initiated rules. - - - Fixed some glitches with the dry-run code's missing-directory - handling, including a problem when combined with --fuzzy. - - - Fixed some glitches with the skipped-directory handling. - - - Fixed the 'T'-flag itemizing of symlinks when --time isn't preserved. - - - Fixed a glitch in the itemizing of permissions with the -E option. - - - The --append option's restricting of transfers to those that add data no - longer prevents the updating of non-content changes to otherwise up-to- - date files (i.e. those with the same content but differing permissions, - ownership, xattrs, etc.). - - - Don't allow --fake-super to be specified with -XX (double --xattrs) - because the options conflict. If a daemon has "fake super" enabled, - it automatically downgrades a -XX request to -X. - - - Fixed a couple bugs in the parsing of daemon-config excludes that could - make a floating exclude rule get treated as matching an absolute path. - - - A daemon doesn't try to auto-refuse the "iconv" option if iconv-support - wasn't compiled in to the daemon (avoiding a warning in the logs). - - - Fixed the inclusion of per-dir merge files from implied dirs. - - - Fixed the support/rrsync script to work with the latest options that - rsync sends (including its flag-specifying use of -e to the server). - - ENHANCEMENTS: - - - Added the --old-dirs (--old-d) option to make it easier for a user to - ask for file-listings with older rsync versions (this is easier than - having to type "-r --exclude='/*/*'" manually). - - - When getting an error while asking an older rsync daemon for a file - listing, rsync will try to notice if the error is a rejection of the - --dirs (-d) option and let the user know how to work around the issue. - - - Added a few more --no-OPTION overrides. - - - Improved the documentation of the --append option. - - - Improved the documentation of the filter/exclude/include daemon - parameters. - - INTERNAL: - - - Fixed a couple minor bugs in the included popt library (ones which I - sent to the official popt project for inclusion in the 1.14 release). - - - Fixed a stat() call that should have been do_stat() so that the proper - normal/64-bit stat() function gets called. (Was in an area that should - not have caused problems, though.) - - - Changed the file-glob code to do a directory scan without using the - "glob" and "glob.h". This lets us do the globbing with less memory - churn, and also avoid adding daemon-excluded items to the returned - args. - - DEVELOPER RELATED: - - - The configure script tries to get the user's compiler to not warn about - unused function parameters if the build is not including one or more of - the ACL/xattrs/iconv features. - - - The configure script now has better checks for figuring out if the - included popt code should be used or not. - - - Fixed two testsuite glitches: avoid a failure if someone's "cd" command - outputs the current directory when cd-ing to a relative path, and made - the itemized test query how rsync was built to determine if it should - expect hard-linked symlinks or not. - - - Updated the testsuite to verify that various bug fixes remain fixed. - - - The RPM spec file was updated to have: (1) comments for how to use the - rsync-patch tar file, and (2) an /etc/xinetd.d/rsync file. - - - Updated the build scripts to work with a revised FTP directory - structure. - - -NEWS for rsync 3.0.0 (1 Mar 2008) -Protocol: 30 (changed) -Changes since 2.6.9: - - NOTABLE CHANGES IN BEHAVIOR: - - - The handling of implied directories when using --relative has changed to - send them as directories (e.g. no implied dir is ever sent as a symlink). - This avoids unexpected behavior and should not adversely affect most - people. If you're one of those rare individuals who relied upon having - an implied dir be duplicated as a symlink, you should specify the - transfer of the symlink and the transfer of the referent directory as - separate args. (See also --keep-dirlinks and --no-implied-dirs.) - Also, exclude rules no longer have a partial effect on implied dirs. - - - Requesting a remote file-listing without specifying -r (--recursive) now - sends the -d (--dirs) option to the remote rsync rather than sending -r - along with an extra exclude of /*/*. If the remote rsync does not - understand the -d option (i.e. it is 2.6.3 or older), you will need to - either turn off -d (--no-d), or specify -r --exclude='/*/*' manually. - - - In --dry-run mode, the last line of the verbose summary text is output - with a "(DRY RUN)" suffix to help remind you that no updates were made. - Similarly, --only-write-batch outputs "(BATCH ONLY)". - - - A writable rsync daemon with "use chroot" disabled now defaults to a - symlink-munging behavior designed to make symlinks safer while also - allowing absolute symlinks to be stored and retrieved. This also has - the effect of making symlinks unusable while they're in the daemon's - hierarchy. See the daemon's "munge symlinks" parameter for details. - - - Starting up an extra copy of an rsync daemon will not clobber the pidfile - for the running daemon -- if the pidfile exists, the new daemon will exit - with an error. This means that your wrapper script that starts the rsync - daemon should be made to handle lock-breaking (if you want any automatic - breaking of locks to be done). - - BUG FIXES: - - - A daemon with "use chroot = no" and excluded items listed in the daemon - config file now properly checks an absolute-path arg specified for these - options: --compare-dest, --link-dest, --copy-dest, --partial-dir, - --backup-dir, --temp-dir, and --files-from. - - - A daemon can now be told to disable all user- and group-name translation - on a per-module basis. This avoids a potential problem with a writable - daemon module that has "use chroot" enabled -- if precautions weren't - taken, a user could try to add a missing library and get rsync to use - it. This makes rsync safer by default, and more configurable when id- - translation is not desired. See the daemon's "numeric ids" parameter - for full details. - - - A chroot daemon can now indicate which part of its path should affect the - chroot call, and which part should become an inside-chroot path for the - module. This allows you to have outside-the-transfer paths (such as for - libraries) even when you enable chroot protection. The idiom used in the - rsyncd.conf file is: path = /chroot/dirs/./dirs/inside - - - If a file's data arrived successfully on the receiving side but the - rename of the temporary file to the destination file failed AND the - --remove-source-files (or the deprecated --remove-sent-files) option - was specified, rsync no longer erroneously removes the associated - source file. - - - Fixed the output of -ii when combined with one of the --*-dest options: - it now itemizes all the items, not just the changed ones. - - - Made the output of all file types consistent when using a --*-dest - option. Prior versions would output too many creation events for - matching items. - - - The code that waits for a child pid now handles being interrupted by a - signal. This fixes a problem with the pre-xfer exec function not being - able to get the exit status from the script. - - - A negated filter rule (i.e. with a '!' modifier) no longer loses the - negation when sending the filter rules to the remote rsync. - - - Fixed a problem with the --out-format (aka --log-format) option %f: it - no longer outputs superfluous directory info for a non-daemon rsync. - - - Fixed a problem with -vv (double --verbose) and --stats when "pushing" - files (which includes local copies). Version 2.6.9 would complete the - copy, but exit with an error when the receiver output its memory stats. - - - If --password-file is used on a non-daemon transfer, rsync now complains - and exits. This should help users figure out that they can't use this - option to control a remote shell's password prompt. - - - Make sure that directory permissions of a newly-created destination - directory are handled right when --perms is left off. - - - The itemized output of a newly-created destination directory is now - output as a creation event, not a change event. - - - Improved --hard-link so that more corner cases are handled correctly - when combined with options such as --link-dest and/or --ignore-existing. - - - The --append option no longer updates a file that has the same size. - - - Fixed a bug when combining --backup and --backup-dir with --inplace: - any missing backup directories are now created. - - - Fixed a bug when using --backup and --inplace with --whole-file or - --read-batch: backup files are actually created now. - - - The daemon pidfile is checked and created sooner in the startup sequence. - - - If a daemon module's "path" value is not an absolute pathname, the code - now makes it absolute internally (making it work properly). - - - Ensure that a temporary file always has owner-write permission while we - are writing to it. This avoids problems with some network filesystems - when transfering read-only files. - - - Any errors output about password-file reading no longer cause an error at - the end of the run about a partial transfer. - - - The --read-batch option for protocol 30 now ensures that several more - options are set correctly for the current batch file: --iconv, --acls, - --xattrs, --inplace, --append, and --append-verify. - - - Using --only-write-batch to a daemon receiver now works properly (older - versions would update some files while writing the batch). - - - Avoid outputting a "file has vanished" message when the file is a broken - symlink and --copy-unsafe-links or --copy-dirlinks is used (the code - already handled this for --copy-links). - - - Fixed the combination of --only-write-batch and --dry-run. - - - Fixed rsync's ability to remove files that are not writable by the file's - owner when rsync is running as the same user. - - - When transferring large files, the sender's hashtable of checksums is - kept at a more reasonable state of fullness (no more than 80% full) so - that the scanning of the hashtable will not bog down as the number of - blocks increases. - - ENHANCEMENTS: - - - A new incremental-recursion algorithm is now used when rsync is talking - to another 3.x version. This starts the transfer going more quickly - (before all the files have been found), and requires much less memory. - See the --recursive option in the manpage for some restrictions. - - - Lowered memory use in the non-incremental-recursion algorithm for typical - option values (usually saving from 21-29 bytes per file). - - - The default --delete algorithm is now --delete-during when talking to a - 3.x rsync. This is a faster scan than using --delete-before (which is - the default when talking to older rsync versions), and is compatible with - the new incremental recursion mode. - - - Rsync now allows multiple remote-source args to be specified rather than - having to rely on a special space-splitting side-effect of the remote- - shell. Additional remote args must specify the same host or an empty one - (e.g. empty: :file1 or ::module/file2). For example, this means that - local use of brace expansion now works: rsync -av host:dir/{f1,f2} . - - - Added the --protect-args (-s) option, that tells rsync to send most of - the command-line args at the start of the transfer rather than as args - to the remote-shell command. This protects them from space-splitting, - and only interprets basic wildcard special shell characters (*?[). - - - Added the --delete-delay option, which is a more efficient way to delete - files at the end of the transfer without needing a separate delete pass. - - - Added the --acls (-A) option to preserve Access Control Lists. This is - an improved version of the prior patch that was available, and it even - supports OS X ACLs. If you need to have backward compatibility with old, - ACL-patched versions of rsync, apply the acls.diff file from the patches - dir. - - - Added the --xattrs (-X) option to preserve extended attributes. This is - an improved version of the prior patch that was available, and it even - supports OS X xattrs (which includes their resource fork data). If you - need to have backward compatibility with old, xattr-patched versions of - rsync, apply the xattrs.diff file from the patches dir. - - - Added the --fake-super option that allows a non-super user to preserve - all attributes of a file by using a special extended-attribute idiom. - It even supports the storing of foreign ACL data on your backup server. - There is also an analogous "fake super" parameter for an rsync daemon. - - - Added the --iconv option, which allows rsync to convert filenames from - one character-set to another during the transfer. The default is to - make this feature available as long as your system has iconv_open(). - If compilation fails, specify --disable-iconv to configure, and then - rebuild. If you want rsync to perform character-set conversions by - default, you can specify --enable-iconv=CONVERT_STRING with the default - value for the --iconv option that you wish to use. For example, - "--enable-iconv=." is a good choice. See the rsync manpage for an - explanation of the --iconv option's settings. - - - A new daemon config parameter, "charset", lets you control the character- - set that is used during an --iconv transfer to/from a daemon module. You - can also set your daemon to refuse "no-iconv" if you want to force the - client to use an --iconv transfer (requiring an rsync 3.x client). - - - Added the --skip-compress=LIST option to override the default list of - file suffixes that will not be compressed when using --compress (-z). - - - The daemon's default for "dont compress" was extended to include: - *.7z *.mp[34] *.mov *.avi *.ogg *.jpg *.jpeg - The name-matching routine was also optimized to run more quickly. - - - The --max-delete option now outputs a warning if it skipped any file - deletions, including a count of how many deletions were skipped. (Older - versions just silently stopped deleting things.) - - - You may specify --max-delete=0 to a 3.0.0 client to request that it warn - about extraneous files without deleting anything. If you're not sure - what version the client is, you can use the less-obvious --max-delete=-1, - as both old and new versions will treat that as the same request (though - older versions don't warn). - - - The --hard-link option now uses less memory on both the sending and - receiving side for all protocol versions. For protocol 30, the use of a - hashtable on the sending side allows us to more efficiently convey to the - receiver what files are linked together. This reduces the amount of data - sent over the socket by a considerable margin (rather than adding more - data), and limits the in-memory storage of the device+inode information - to just the sending side for the new protocol 30, or to the receiving - side when speaking an older protocol (note that older rsync versions kept - the device+inode information on both sides). - - - The filter rules now support a perishable ("p") modifier that marks rules - that should not have an effect in a directory that is being deleted. e.g. - -f '-p .svn/' would only affect "live" .svn directories. - - - Rsync checks all the alternate-destination args for validity (e.g. - --link-dest). This lets the user know when they specified a directory - that does not exist. - - - If we get an ENOSYS error setting the time on a symlink, we don't - complain about it anymore (for those systems that even support the - setting of the modify-time on a symlink). - - - Protocol 30 now uses MD5 checksums instead of MD4. - - - Changed the --append option to not checksum the existing data in the - destination file, which speeds up file appending. - - - Added the --append-verify option, which works like the older --append - option (verifying the existing data in the destination file). For - compatibility with older rsync versions, any use of --append that is - talking protocol 29 or older will revert to the --append-verify method. - - - Added the --contimeout=SECONDS option that lets the user specify a - connection timeout for rsync daemon access. - - - Documented and extended the support for the RSYNC_CONNECT_PROG variable - that can be used to enhance the client side of a daemon connection. - - - Improved the dashes and double-quotes in the nroff manpage output. - - - Rsync now supports a lot more --no-OPTION override options. - - INTERNAL: - - - The file-list sorting algorithm now uses a sort that keeps any same- - named items in the same order as they were specified. This allows - rsync to always ensure that the first of the duplicates is the one - that will be included in the copy. The new sort is also faster - than the glibc version of qsort() and mergesort(). - - - Rsync now supports the transfer of 64-bit timestamps (time_t values). - - - Made the file-deletion code use a little less stack when recursing - through a directory hierarchy of extraneous files. - - - Fixed a build problem with older (2.x) versions of gcc. - - - Added some isType() functions that make dealing with signed characters - easier without forcing variables via casts. - - - Changed strcat/strcpy/sprintf function calls to use safer versions. - - - Upgraded the included popt version to 1.10.2 and improved its use of - string-handling functions. - - - Added missing prototypes for compatibility functions from the lib dir. - - - Configure determines if iconv() has a const arg, allowing us to avoid a - compiler warning. - - - Made the sending of some numbers more efficient for protocol 30. - - - Make sure that a daemon process doesn't mind if the client was weird and - omitted the --server option. - - - There are more internal logging categories available in protocol 30 than - the age-old FINFO and FERROR, including FERROR_XFER and FWARN. These new - categories allow some errors and warnings to go to stderr without causing - an erroneous end-of-run warning about some files not being able to be - transferred. - - - Improved the use of "const" on pointers. - - - Improved J.W.'s pool_alloc routines to add a way of incrementally freeing - older sections of a pool's memory. - - - The getaddrinfo.c compatibility code in the "lib" dir was replaced with - some new code (derived from samba, derived from PostgreSQL) that has a - better license than the old code. - - DEVELOPER RELATED: - - - Rsync is now licensed under the GPLv3 or later. - - - Rsync is now being maintained in a "git" repository instead of CVS - (though the old CVS repository still exists for historical access). - Several maintenance scripts were updated to work with git. - - - Generated files are no longer committed into the source repository. The - autoconf and autoheader commands are now automatically run during the - normal use of "configure" and "make". The latest dev versions of all - generated files can also be copied from the samba.org web site (see the - prepare-source script's fetch option). - - - The "patches" directory of diff files is now built from branches in the - rsync git repository (branch patch/FOO creates file patches/FOO.diff). - This directory is now distributed in a separate separate tar file named - rsync-patches-VERSION.tar.gz instead of the main rsync-VERSION.tar.gz. - - - The proto.h file is now built using a simple perl script rather than a - complex awk script, which proved to be more widely compatible. - - - When running the tests, we now put our per-test temp dirs into a sub- - directory named testtmp (which is created, if missing). This allows - someone to symlink the testtmp directory to another filesystem (which is - useful if the build dir's filesystem does not support ACLs and xattrs, - but another filesystem does). - - - Rsync now has a way of handling protocol-version changes during the - development of a new protocol version. This causes any out-of-sync - versions to speak an older protocol rather than fail in a cryptic manner. - This addition makes it safer to deploy a pre-release version that may - interact with the public. This new exchange of sub-version info does not - interfere with the {MIN,MAX}_PROTOCOL_VERSION checking algorithm (which - does not have enough range to allow the main protocol number to be - incremented for every minor tweak in that happens during development). - - - The csprotocol.txt file was updated to mention the daemon protocol change - in the 3.0.0 release. - - -NEWS for rsync 2.6.9 (6 Nov 2006) -Protocol: 29 (unchanged) -Changes since 2.6.8: - - BUG FIXES: - - - If rsync is interrupted via a handled signal (such as SIGINT), it will - once again clean-up its temp file from the destination dir. - - - Fixed an overzealous sanitizing bug in the handling of the --link-dest, - --copy-dest, and --compare-dest options to a daemon without chroot: if - the copy's destination dir is deeper than the top of the module's path, - these options now accept a safe number of parent-dir (../) references - (since these options are relative to the destination dir). The old code - incorrectly chopped off all "../" prefixes for these options, no matter - how deep the destination directory was in the module's hierarchy. - - - Fixed a bug where a deferred info/error/log message could get sent - directly to the sender instead of being handled by rwrite() in the - generator. This fixes an "unexpected tag 3" fatal error, and should - also fix a potential problem where a deferred info/error message from - the receiver might bypass the log file and get sent only to the client - process. (These problems could only affect an rsync daemon that was - receiving files.) - - - Fixed a bug when --inplace was combined with a --*-dest option and we - update a file's data using an alternate basis file. The code now - notices that it needs to copy the matching data from the basis file - instead of (wrongly) assuming that it was already present in the file. - - - Fixed a bug where using --dry-run with a --*-dest option with a path - relative to a directory that does not yet exist: the affected option - gets its proper path value so that the output of the dry-run is right. - - - Fixed a bug in the %f logfile escape when receiving files: the - destination path is now included in the output (e.g. you can now tell - when a user specifies a subdir inside a module). - - - If the receiving side fails to create a directory, it will now skip - trying to update everything that is inside that directory. - - - If --link-dest is specified with --checksum but without --times, rsync - will now allow a hard-link to be created to a matching link-dest file - even when the file's modify-time doesn't match the server's file. - - - The daemon now calls more timezone-using functions prior to doing a - chroot. This should help some C libraries to generate proper timestamps - from inside a chrooted daemon (and to not try to access /etc/timezone - over and over again). - - - Fixed a bug in the handling of an absolute --partial-dir=ABS_PATH option: - it now deletes an alternate basis file from the partial-dir that was used - to successfully update a destination file. - - - Fixed a bug in the handling of --delete-excluded when using a per-dir - merge file: the merge file is now honored on the receiving side, and - only its unqualified include/exclude commands are ignored (just as is - done for global include/excludes). - - - Fixed a recent bug where --delete was not working when transferring from - the root (/) of the filesystem with --relative enabled. - - - Fixed a recent bug where an --exclude='*' could affect the root (/) of - the filesystem with --relative enabled. - - - When --inplace creates a file, it is now created with owner read/write - permissions (0600) instead of no permissions at all. This avoids a - problem continuing a transfer that was interrupted (since --inplace - will not update a file that has no write permissions). - - - If either --remove-source-files or --remove-sent-files is enabled and we - are unable to remove the source file, rsync now outputs an error. - - - Fixed a bug in the daemon's "incoming chmod" rule: newly-created - directories no longer get the 'F' (file) rules applied to them. - - - Fixed an infinite loop bug when a filter rule was rejected due to being - overly long. - - - When the server receives a --partial-dir option from the client, it no - longer runs the client-side code that adds an assumed filter rule (since - the client will be sending us the rules in the usual manner, and they - may have chosen to override the auto-added rule). - - ENHANCEMENTS: - - - Added the --log-file=FILE and --log-file-format=FORMAT options. These - can be used to tell any rsync to output what it is doing to a log file. - They work with a client rsync, a non-daemon server rsync (see the man - page for instructions), and also allows the overriding of rsyncd.conf - settings when starting a daemon. - - - The --log-format option was renamed to be --out-format to avoid confusing - it with affecting the log-file output. (The old option remains as an - alias for the new to preserve backward compatibility.) - - - Made "log file" and "syslog facility" settable on a per-module basis in - the daemon's config file. - - - Added the --remove-source-files option as a replacement for the (now - deprecated) --remove-sent-files option. This new option removes all - non-dirs from the source directories, even if the file was already - up-to-date. This fixes a problem where interrupting an rsync that - was using --remove-sent-files and restarting it could leave behind - a file that the earlier rsync synchronized, but didn't get to remove. - (The deprecated --remove-sent-files is still understood for now, and - still behaves in the same way as before.) - - - Added the option --no-motd to suppress the message-of-the-day output - from a daemon when doing a copy. (See the manpage for a caveat.) - - - Added a new environment variable to the pre-/post-xfer exec commands (in - the daemon's config file): RSYNC_PID. This value will be the same in - both the pre- and post-xfer commands, so it can be used as a unique ID - if the pre-xfer command wants to cache some arg/request info for the - post-xfer command. - - INTERNAL: - - - Did a code audit using IBM's code-checker program and made several - changes, including: replacing most of the strcpy() and sprintf() - calls with strlcpy(), snprintf(), and memcpy(), adding a 0-value to - an enum that had been intermingling a literal 0 with the defined enum - values, silencing some uninitialized memory checks, marking some - functions with a "noreturn" attribute, and changing an "if" that - could never succeed on some platforms into a pre-processor directive - that conditionally compiles the code. - - - Fixed a potential bug in f_name_cmp() when both the args are a - top-level "." dir (which doesn't happen in normal operations). - - - Changed exit_cleanup() so that it can never return instead of exit. - The old code might return if it found the exit_cleanup() function - was being called recursively. The new code is segmented so that - any recursive calls move on to the next step of the exit-processing. - - - The macro WIFEXITED(stat) will now be defined if the OS didn't already - define it. - - DEVELOPER RELATED: - - - The acls.diff and xattrs.diff patches have received a bunch of work to - make them much closer to being acceptable in the main distribution. - The xattrs patch also has some preliminary Mac OS X and FreeBSD - compatibility code that various system types to exchange extended - file-attributes. - - - A new diff in the patches dir, fake-root.diff, allows rsync to - maintain a backup hierarchy with full owner, group, and device info - without actually running as root. It does this using a special - extended attribute, so it depends on xattrs.diff (which depends on - acls.diff). - - - The rsync.yo and rsyncd.conf.yo files have been updated to work - better with the latest yodl 2.x releases. - - - Updated config.guess and config.sub to their 2006-07-02 versions. - - - Updated various files to include the latest FSF address and to have - consistent opening comments. - - -NEWS for rsync 2.6.8 (22 Apr 2006) -Protocol: 29 (unchanged) -Changes since 2.6.7: - - BUG FIXES: - - - Fixed a bug in the exclude code where an anchored exclude without any - wildcards fails to match an absolute source arg, but only when --relative - is in effect. - - - Improved the I/O code for the generator to fix a potential hang when the - receiver gets an EOF on the socket but the generator's select() call - never indicates that the socket is writable for it to be notified about - the EOF. (This can happen when using stunnel). - - - Fixed a problem with the file-reading code where a failed read (such as - that caused by a bad sector) would not advance the file's read-position - beyond the failed read's data. - - - Fixed a logging bug where the "log file" directive was not being honored - in a single-use daemon (one spawned by a remote-shell connection or by - init). - - - If rsync cannot honor the --delete option, we output an error and exit - instead of silently ignoring the option. - - - Fixed a bug in the --link-dest code that prevented special files (such as - fifos) from being linked. - - - The ability to hard-link symlinks and special files is now determined at - configure time instead of at runtime. This fixes a bug with --link-dest - creating a hard-link to a symlink's referent on a BSD system. - - ENHANCEMENTS: - - - In daemon mode, if rsync fails to bind to the requested port, the - error(s) returned by socket() and/or bind() are now logged. - - - When we output a fatal error, we now output the version of rsync in the - message. - - - Improved the documentation for the --owner and --group options. - - - The rsyncstats script in "support" has an improved line-parsing regex - that is easier to read and also makes it to parse syslog-generated lines. - - - A new script in "support": file-attr-restore, can be used to restore the - attributes of a file-set (the permissions, ownership, and group info) - taken from the cached output of a "find ARG... -ls" command. - - DEVELOPER RELATED: - - - Removed the unused function write_int_named(), the unused variable - io_read_phase, and the rarely used variable io_write_phase. This also - elides the confusing 'phase "unknown"' part of one error message. - - - Removed two unused configure checks and two related (also unused) - compatibility functions. - - - The xattrs.diff patch received a security fix that prevents a potential - buffer overflow in the receive_xattr() code. - - - The acls.diff patch has been improved quite a bit, with more to come. - - - A new patch was added: log-file.diff. This contains an early version of - a future option, --log-file=FILE, that will allow any rsync to log its - actions to a file (something that only a daemon supports at present). - - -NEWS for rsync 2.6.7 (11 Mar 2006) -Protocol: 29 (unchanged) -Changes since 2.6.6: - - OUTPUT CHANGES: - - - The letter 'D' in the itemized output was being used for both devices - (character or block) as well as other special files (such as fifos and - named sockets). This has changed to separate non-device special files - under the 'S' designation (e.g. "cS+++++++ path/fifo"). See also the - "--specials" option, below. - - - The way rsync escapes unreadable characters has changed. First, rsync - now has support for recognizing valid multibyte character sequences in - your current locale, allowing it to escape fewer characters than before - for a locale such as UTF-8. Second, it now uses an escape idiom of - "\#123", which is the literal string "\#" followed by exactly 3 octal - digits. Rsync no longer doubles a backslash character in a filename - (e.g. it used to output "foo\\bar" when copying "foo\bar") -- now it only - escapes a backslash that is followed by a hash-sign and 3 digits (0-9) - (e.g. it will output "foo\#134#789" when copying "foo\#789"). See also - the --8-bit-output (-8) option, mentioned below. - - Script writers: the local rsync is the one that outputs escaped names, - so if you need to support unescaping of filenames for older rsyncs, I'd - suggest that you parse the output of "rsync --version" and only use the - old unescaping rules for 2.6.5 and 2.6.6. - - BUG FIXES: - - - Fixed a really old bug that caused --checksum (-c) to checksum all the - files encountered during the delete scan (ouch). - - - Fixed a potential hang in a remote generator: when the receiver gets a - read-error on the socket, it now signals the generator about this so that - the generator does not try to send any of the terminating error messages - to the client (avoiding a potential hang in some setups). - - - Made hard-links work with symlinks and devices again. - - - If the sender gets an early EOF reading a source file, we propagate this - error to the receiver so that it can discard the file and try requesting - it again (which is the existing behavior for other kinds of read errors). - - - If a device-file/special-file changes permissions, rsync now updates the - permissions without recreating the file. - - - If the user specifies a remote-host for both the source and destination, - we now output a syntax error rather than trying to open the destination - hostspec as a filename. - - - When --inplace creates a new destination file, rsync now creates it with - permissions 0600 instead of 0000 -- this makes restarting possible when - the transfer gets interrupted in the middle of sending a new file. - - - Reject the combination of --inplace and --sparse since the sparse-output - algorithm doesn't work when overwriting existing data. - - - Fixed the directory name in the error that is output when pop_dir() - fails. - - - Really fixed the parsing of a "!" entry in .cvsignore files this time. - - - If the generator gets a stat() error on a file, output it (this used to - require at least -vv for the error to be seen). - - - If waitpid() fails or the child rsync didn't exit cleanly, we now handle - the exit status properly and generate a better error. - - - Fixed some glitches in the double-verbose output when using --copy-dest, - --link-dest, or --compare-dest. Also improved how the verbose output - handles hard-links (within the transfer) that had an up-to-date alternate - "dest" file, and copied files (via --copy-dest). - - - Fixed the matching of the dont-compress items (e.g. *.gz) against files - that have a path component containing a slash. - - - If the code reading a filter/exclude file gets an EINTR error, rsync now - clears the error flag on the file handle so it can keep on reading. - - - If --relative is active, the sending side cleans up trailing "/" or "/." - suffixes to avoid triggering a bug in older rsync versions. Also, we now - reject a ".." dir if it would be sent as a relative dir. - - - If a non-directory is in the way of a directory and rsync is run with - --dry-run and --delete, rsync no longer complains about not being able - to opendir() the not-yet present directory. - - - When --list-only is used and a non-existent local destination dir was - also specified as a destination, rsync no longer generates a warning - about being unable to create the missing directory. - - - Fixed some problems with --relative --no-implied-dirs when the - destination directory did not yet exist: we can now create a symlink or - device when it is the first thing in the missing dir, and --fuzzy no - longer complains about being unable to open the missing dir. - - - Fixed a bug where the --copy-links option would not affect implied - directories without --copy-unsafe-links (see --relative). - - - Got rid of the need for --force to be used in some circumstances with - --delete-after (making it consistent with --delete-before/-during). - - - Rsync now ignores the SIGXFSZ signal, just in case your OS sends this - when a file is too large (rsync handles the write error). - - - Fixed a bug in the Proxy-Authorization header's base64-encoded value: it - was not properly padded with trailing '=' chars. This only affects a - user that need to use a password-authenticated proxy for an outgoing - daemon-rsync connection. - - - If we're transferring an empty directory to a new name, rsync no longer - forces S_IWUSR if it wasn't already set, nor does it accidentally leave - it set. - - - Fixed a bug in the debug output (-vvvvv) that could mention the wrong - checksum for the current file offset. - - - Rsync no longer allows a single directory to be copied over a non- - directory destination arg. - - ENHANCEMENTS: - - - Added the --append option that makes rsync append data onto files that - are longer on the source than the destination (this includes new files). - - - Added the --min-size=SIZE option to exclude small files from the - transfer. - - - Added the --compress-level option to allow you to set how aggressive - rsync's compression should be (this option implies --compress). - - - Enhanced the parsing of the SIZE value for --min-size and --max-size to - allow easy entry of multiples of 1000 (instead of just multiples of 1024) - and off-by-one values too (e.g. --max-size=8mb-1). - - - Added the --8-bit-output (-8) option, which tells rsync to avoid escaping - high-bit characters that it thinks are unreadable in the current locale. - - - The new option --human-readable (-h) changes the output of --progress, - --stats, and the end-of-run summary to be easier to read. If repeated, - the units become powers of 1024 instead of powers of 1000. (The old - meaning of -h, as a shorthand for --help, still works as long as you - just use it on its own, as in "rsync -h".) - - - If lutimes() and/or lchmod() are around, use them to allow the - preservation of attributes on symlinks. - - - The --link-dest option now affects symlinks and devices (when possible). - - - Added two config items to the rsyncd.conf parsing: "pre-xfer exec" and - "post-xfer exec". These allow a command to be specified on a per-module - basis that will be run before and/or after a daemon-mode transfer. (See - the man page for a list of the environment variables that are set with - information about the transfer.) - - - When using the --relative option, you can now insert a dot dir in - the source path to indicate where the replication of the source dirs - should start. For example, if you specify a source path of - rsync://host/module/foo/bar/./baz/dir with -R, rsync will now only - replicate the "baz/dir" part of the source path (note: a trailing - dot dir is unaffected unless it also has a trailing slash). - - - Added some new --no-FOO options that make it easier to override unwanted - implied or default options. For example, "-a --no-o" (aka "--archive - --no-owner") can be used to turn off the preservation of file ownership - that is implied by -a. - - - Added the --chmod=MODE option that allows the destination permissions to - be changed from the source permissions. E.g. --chmod=g+w,o-rwx - - - Added the "incoming chmod" and "outgoing chmod" daemon options that allow - a module to specify what permissions changes should be applied to all - files copied to and from the daemon. - - - Allow the --temp-dir option to be specified when starting a daemon, which - sets the default temporary directory for incoming files. - - - If --delete is combined with --dirs without --recursive, rsync will now - delete in any directory whose content is being synchronized. - - - If --backup is combined with --delete without --backup-dir (and without - --delete-excluded), we add a "protect" filter-rule to ensure that files - with the backup suffix are not deleted. - - - The file-count stats that are output by --progress were improved to - better indicate what the numbers mean. For instance, the output: - "(xfer#5, to-check=8383/9999)" indicates that this was the fifth file - to be transferred, and we still need to check 8383 more files out of - a total of 9999. - - - The include/exclude code now allows a dir/*** directive (with 3 trailing - stars) to match both the dir itself as well as all the content below the - dir (dir/** would not match the dir). - - - Added the --prune-empty-dirs (-m) option that makes the receiving rsync - discard empty chains of directories from the file-list. This makes it - easier to selectively copy files from a source hierarchy and end up with - just the directories needed to hold the resulting files. - - - If the --itemize-changes (-i) option is repeated, rsync now includes - unchanged files in the itemized output (similar to -vv, but without all - the other verbose messages that can get in the way). Of course, the - client must be version 2.6.7 for this to work, but the remote rsync only - needs to be 2.6.7 if you're pushing files. - - - Added the --specials option to tell rsync to copy non-device special - files (which rsync now attempts even as a normal user). The --devices - option now requests the copying of just devices (character and block). - The -D option still requests both (e.g. --devices and --specials), -a - still implies -D, and non-root users still get a silent downgrade that - omits device copying. - - - Added the --super option to make the receiver always attempt super-user - activities. This is useful for systems that allow things such as devices - to be created or ownership to be set without being UID 0, and is also - useful for someone who wants to ensure that errors will be output if the - receiving rsync isn't being run as root. - - - Added the --sockopts option for those few who want to customize the TCP - options used to contact a daemon rsync. - - - Added a way for the --temp-dir option to be combined with a partial-dir - setting that lets rsync avoid non-atomic updates (for those times when - --temp-dir is not being used because space is tight). - - - A new support script, files-to-excludes, will transform a list of files - into a set of include/exclude directives that will copy those files. - - - A new option, --executability (-E) can be used to preserve just the - execute bit on files, for those times when using the --perms option is - not desired. - - - The daemon now logs each connection and also each module-list request - that it receives. - - - New log-format options: %M (modtime), %U (uid), %G (gid), and %B - (permission bits, e.g. "rwxr-xrwt"). - - - The --dry-run option no longer forces the enabling of --verbose. - - - The --remove-sent-files option now does a better job of incrementally - removing the sent files on the sending side (older versions tended to - clump up all the removals at the end). - - - A daemon now supersedes its minimal SIGCHLD handler with the standard - PID-remembering version after forking. This ensures that the generator - can get the child-exit status from the receiver. - - - Use of the --bwlimit option no longer interferes with the remote rsync - sending error messages about invalid/refused options. - - - Rsync no longer returns a usage error when used with one local source arg - and no destination: this now implies the --list-only option, just like - the comparable situation with a remote source arg. - - - Added the --copy-dirlinks option, a more limited version of --copy-links. - - - Various documentation improvements, including: a better synopsis, some - improved examples, a better discussion of the presence and absence of - --perms (including how it interacts with the new --executability and - --chmod options), an extended discussion of --temp-dir, an improved - discussion of --partial-dir, a better description of rsync's pattern - matching characters, an improved --no-implied-dirs section, and the - documenting of what the --stats option outputs. - - - Various new and updated diffs in the patches dir, including: acls.diff, - xattrs.diff, atimes.diff, detect-renamed.diff, and slp.diff. - - INTERNAL: - - - We now use sigaction() and sigprocmask() if possible, and fall back on - signal() if not. Using sigprocmask() ensures that rsync enables all the - signals that it needs, just in case it was started in a masked state. - - - Some buffer sizes were expanded a bit, particularly on systems where - MAXPATHLEN is overly small (e.g. cygwin). - - - If io_printf() tries to format more data than fits in the buffer, exit - with an error instead of transmitting a truncated buffer. - - - If a va_copy macro is defined, lib/snprintf.c will use it when defining - the VA_COPY macro. - - - Reduced the amount of stack memory needed for each level of directory - recursion by nearly MAXPATHLEN bytes. - - - The wildmatch function was extended to allow an array of strings to be - supplied as the string to match. This allows the exclude code to do less - string copying. - - - Got rid of the safe_fname() function (and all the myriad calls) and - replaced it with a new function in the log.c code that filters all the - output going to the terminal. - - - Unified the f_name() and the f_name_to() functions. - - - Improved the hash-table code the sender uses to handle checksums to make - it use slightly less memory and run just a little faster. - - DEVELOPER RELATED: - - - The diffs in the patches dir now require "patch -p1 <DIFF" instead of - the previous -p0. Also, the version included in the release tar now - affect generated files (e.g. configure, rsync.1, proto.h, etc.), so - it is no longer necessary to run autoconf and/or yodl unless you're - applying a patch that was checked out from CVS. - - - Several diffs in the patches dir now use the proper --enable-FOO - configure option instead of --with-FOO to turn on the inclusion of - the newly patched feature. - - - There is a new script, "prepare-source" than can be used to update the - various generated files (proto.h, configure, etc.) even before configure - has created the Makefile (this is mainly useful when patching the source - with a patch that doesn't affect generated files). - - - The testsuite now sets HOME so that it won't be affected by a file such - as ~/.popt. - - -NEWS for rsync 2.6.6 (28 Jul 2005) -Protocol: 29 (unchanged) -Changes since 2.6.5: - - SECURITY FIXES: - - - The zlib code was upgraded to version 1.2.3 in order to make it more - secure. While the widely-publicized security problem in zlib 1.2.2 did - not affect rsync, another security problem surfaced that affects rsync's - zlib 1.1.4. - - BUG FIXES: - - - The setting of flist->high in clean_flist() was wrong for an empty list. - This could cause flist_find() to crash in certain rare circumstances - (e.g. if just the right directory setup was around when --fuzzy was - combined with --link-dest). - - - The outputting of hard-linked files when verbosity was > 1 was not right: - (1) Without -i it would output the name of each hard-linked file as - though it had been changed; it now outputs a "is hard linked" message for - the file. (2) With -i it would output all dots for the unchanged - attributes of a hard-link; it now changes those dots to spaces, as is - done for other totally unchanged items. - - - When backing up a changed symlink or device, get rid of any old backup - item so that we don't get an "already exists" error. - - - A couple places that were comparing a local and a remote modification- - time were not honoring the --modify-window option. - - - Fixed a bug where the 'p' (permissions) itemized-changes flag might get - set too often (if some non-significant mode bits differed). - - - Fixed a really old, minor bug that could cause rsync to warn about being - unable to mkdir() a path that ends in "/." because it just created the - directory (required --relative, --no-implied-dirs, a source path that - ended in either a trailing slash or a trailing "/.", and a non-existing - destination dir to tickle the bug in a recent version). - - ENHANCEMENTS: - - - Made the "max verbosity" setting in the rsyncd.conf file settable on a - per-module basis (which now matches the documentation). - - - The support/rrsync script has been upgraded to verify the args of options - that take args (instead of rejecting any such options). The script was - also changed to try to be more secure and to fix a problem in the parsing - of a pull operation that has multiple sources. - - - Improved the documentation that explains the difference between a - normal daemon transfer and a daemon-over remote-shell transfer. - - - Some of the diffs supplied in the patches dir were fixed and/or - improved. - - BUILD CHANGES: - - - Made configure define NOBODY_USER (currently hard-wired to "nobody") and - NOBODY_GROUP (set to either "nobody" or "nogroup" depending on what we - find in the /etc/group file). - - - Added a test to the test suite, itemized.test, that tests the output of - -i (log-format w/%i) and some double-verbose messages. - - -NEWS for rsync 2.6.5 (1 Jun 2005) -Protocol: 29 (unchanged) -Changes since 2.6.4: - - OUTPUT CHANGES: - - - Non-printable chars in filenames are now output using backslash- - escaped characters rather than '?'s. Any non-printable character is - output using 3 digits of octal (e.g. "\n" -> "\012"), and a backslash - is now output as "\\". Rsync also uses your locale setting, which - can make it treat fewer high-bit characters as non-printable. - - - If rsync received an empty file-list when pulling files, it would - output a "nothing to do" message and exit with a 0 (success) exit - status, even if the remote rsync returned an error (it did not do - this under the same conditions when pushing files). This was changed - to make the pulling behavior the same as the pushing behavior: we - now do the normal end-of-run outputting (depending on options) and - exit with the appropriate exit status. - - BUG FIXES: - - - A crash bug was fixed when a daemon had its "path" set to "/", did - not have chroot enabled, and used some anchored excludes in the - rsyncd.conf file. - - - Fixed a bug in the transfer of a single file when -H is specified - (rsync would either infinite loop or perhaps crash). - - - Fixed a case where the generator might try (and fail) to tweak the - write-permissions of a read-only directory in list-only mode (this - only caused an annoying warning message). - - - If --compare-dest or --link-dest uses a locally-copied file as the - basis for an updated version, log this better when --verbose or -i - is in effect. - - - Fixed the accidental disabling of --backup during the --delete-after - processing. - - - Restored the ability to use the --address option in client mode (in - addition to its use in daemon mode). - - - Make sure that some temporary progress information from the delete - processing does not get left on the screen when it is followed by a - newline. - - - When --existing skips a directory with extra verbosity, refer to it - as a "directory", not a "file". - - - When transferring a single file to a different-named file, any - generator messages that are source-file related no longer refer to - the file by the destination filename. - - - Fixed a bug where hard-linking a group of files might fail if the - generator hasn't created a needed destination directory yet. - - - Fixed a bug where a hard-linked group of files that is newly-linked - to a file in a --link-dest dir doesn't link the files from the rest - of the cluster. - - - When deleting files with the --one-file-system (-x) option set, rsync - no longer tries to remove files from inside a mount-point on the - receiving side. Also, we don't complain about being unable to remove - the mount-point dir. - - - Fixed a compatibility problem when using --cvs-ignore (-C) and - sending files to an older rsync without using --delete. - - - Make sure that a "- !" or "+ !" include/exclude pattern does not - trigger the list-clearing action that is reserved for "!". - - - Avoid a timeout in the generator when the sender/receiver aren't - handling the generator's checksum output quickly enough. - - - Fixed the omission of some directories in the delete processing when - --relative (-R) was combined with a source path that had a trailing - slash. - - - Fixed a case where rsync would erroneously delete some files and then - re-transfer them when the options --relative (-R) and --recursive - (-r) were both enabled (along with --delete) and a source path had a - trailing slash. - - - Make sure that --max-size doesn't affect a device or a symlink. - - - Make sure that a system with a really small MAXPATHLEN does not cause - the buffers in readfd_unbuffered() to be too small to receive normal - messages. (This mainly affected Cygwin.) - - - If a source pathname ends with a filename of "..", treat it as if - "../" had been specified (so that we don't copy files to the parent - dir of the destination). - - - If --delete is combined with a file-listing rsync command (i.e. no - transfer is happening), avoid outputting a warning that we couldn't - delete anything. - - - If --stats is specified with --delete-after, ensure that all the - "deleting" messages are output before the statistics. - - - Improved one "if" in the deletion code that was only checking errno - for ENOTEMPTY when it should have also been checking for EEXIST (for - compatibility with OS variations). - - ENHANCEMENTS: - - - Added the --only-write-batch=FILE option that may be used (instead - of --write-batch=FILE) to create a batch file without doing any - actual updating of the destination. This allows you to divert all - the file-updating data away from a slow data link (as long as you - are pushing the data to the remote server when creating the batch). - - - When the generator is taking a long time to fill up its output buffer - (e.g. if the transferred files are few, small, or missing), it now - periodically flushes the output buffer so that the sender/receiver - can get started on the files sooner rather than later. - - - Improved the keep-alive code to handle a long silence between the - sender and the receiver that can occur when the sender is receiving - the checksum data for a large file. - - - Improved the auth-errors that are logged by the daemon to include - some information on why the authorization failed: wrong user, - password mismatch, etc. (The client-visible message is unchanged!) - - - Improved the client's handling of an "@ERROR" from a daemon so that - it does not complain about an unexpectedly closed socket (since we - really did expect the socket to close). - - - If the daemon can't open the log-file specified in rsyncd.conf, fall - back to using syslog and log an appropriate warning. This is better - than what was typically a totally silent (and fatal) failure (since a - daemon is not usually run with the --no-detach option that was - necessary to see the error on stderr). - - - The man pages now consistently refer to an rsync daemon as a "daemon" - instead of a "server" (to distinguish it from the server process in a - non-daemon transfer). - - - Made a small change to the rrsync script (restricted rsync -- in the - support dir) to make a read-only server reject all --remove-* options - when sending files (to future-proof it against the possibility of - other similar options being added at some point). - - INTERNAL: - - - Rsync now calls setlocale(LC_CTYPE, ""). This enables isprint() to - better discern which filename characters need to be escaped in - messages (which should result in fewer escaped characters in some - locales). - - - Improved the naming of the log-file open/reopen/close functions. - - - Removed some protocol-compatibility code that was only needed to help - someone running a pre-release of 2.6.4. - - BUILD CHANGES: - - - Added configure option "--disable-locale" to disable any use of - setlocale() in the binary. - - - Fixed a bug in the SUPPORT{,_HARD}_LINKS #defines which prevented - rsync from being built without symlink or hard-link support. - - - Only #define HAVE_REMSH if it is going to be set to 1. - - - Configure now disables the use of mkstemp() under HP-UX (since they - refuse to fix its broken handling of large files). - - - Configure now explicitly checks for the lseek64() function so that - the code can use HAVE_LSEEK64 instead of inferring lseek64()'s - presence based on the presence of the off64_t type. - - - Configure no longer mentions the change in the default remote-shell - (from rsh to ssh) that occurred for the 2.6.0 release. - - - Some minor enhancements to the test scripts. - - - Added a few new *.diff files to the patches dir, including a patch - that enables the optional copying of extended attributes. - - -NEWS for rsync 2.6.4 (30 March 2005) -Protocol: 29 (changed) -Changes since 2.6.3: - - OUTPUT CHANGES: - - - When rsync deletes a directory and outputs a verbose message about - it, it now appends a trailing slash to the name instead of (only - sometimes) outputting a preceding "directory " string. - - - The --stats output will contain file-list time-statistics if both - sides are 2.6.4, or if the local side is 2.6.4 and the files are - being pushed (since the stats come from the sending side). - (Requires protocol 29 for a pull.) - - - The "%o" (operation) log-format escape now has a third value (besides - "send" and "recv"): "del." (with trailing dot to make it 4 chars). - This changes the way deletions are logged in the daemon's log file. - - - When the --log-format option is combined with --verbose, rsync now - avoids outputting the name of the file twice in most circumstances. - As long as the --log-format item does not refer to any post-transfer - items (such as %b or %c), the --log-format message is output prior to - the transfer, so --verbose is now the equivalent of a --log-format of - '%n%L' (which outputs the name and any link info). If the log output - must occur after the transfer to be complete, the only time the name - is also output prior to the transfer is when --progress was specified - (so that the name will precede the progress stats, and the full - --log-format output will come after). - - - Non-printable characters in filenames are replaced with a '?' to - avoid corrupting the screen or generating empty lines in the output. - - BUG FIXES: - - - Restore the list-clearing behavior of "!" in a .cvsignore file (2.6.3 - was only treating it as a special token in an rsync include/exclude - file). - - - The combination of --verbose and --dry-run now mentions the full list - of changes that would be output without --dry-run. - - - Avoid a mkdir warning when removing a directory in the destination - that already exists in the --backup-dir. - - - An OS that has a binary mode for its files (such as cygwin) needed - setmode(fd, O_BINARY) called on the temp-file we opened with - mkstemp(). (Fix derived from cygwin's 2.6.3 rsync package.) - - - Fixed a potential hang when verbosity is high, the client side is - the sender, and the file-list is large. - - - Fixed a potential protocol-corrupting bug where the generator could - merge a message from the receiver into the middle of a multiplexed - packet of data if only part of that data had been written out to the - socket when the message from the generator arrived. - - - We now check if the OS doesn't support using mknod() for creating - FIFOs and sockets, and compile-in some compatibility code using - mkfifo() and socket() when necessary. - - - Fixed an off-by-one error in the handling of --max-delete=N. Also, - if the --max-delete limit is exceeded during a run, we now output a - warning about this at the end of the run and exit with a new error - code (25). - - - One place in the code wasn't checking if fork() failed. - - - The "ignore nonreadable" daemon parameter used to erroneously affect - readable symlinks that pointed to a non-existent file. - - - If the OS does not have lchown() and a chown() of a symlink will - affect the referent of a symlink (as it should), we no longer try - to set the user and group of a symlink. - - - The generator now properly runs the hard-link loop and the dir-time - rewriting loop after we're sure that the redo phase is complete. - - - When --backup was specified with --partial-dir=DIR, where DIR is a - relative path, the backup code was erroneously trying to backup a - file that was put into the partial-dir. - - - If a file gets resent in a single transfer and the --backup option is - enabled along with --inplace, rsync no longer performs a duplicate - backup (it used to overwrite the first backup with the failed file). - - - One call to flush_write_file() was not being checked for an error. - - - The --no-relative option was not being sent from the client to a - server sender. - - - If an rsync daemon specified "dont compress = ..." for a file and the - client tried to specify --compress, the libz code was not handling a - compression level of 0 properly. This could cause a transfer failure - if the block-size for a file was large enough (e.g. rsync might have - exited with an error for large files). - - - Fixed a bug that would sometimes surface when using --compress and - sending a file with a block-size larger than 64K (either manually - specified, or computed due to the file being really large). Prior - versions of rsync would sometimes fail to decompress the data - properly, and thus the transferred file would fail its verification. - - - If a daemon can't open the specified log file (i.e. syslog is not - being used), die without crashing. We also output an error about - the failure on stderr (which will only be seen if --no-detach was - specified) and exit with a new error code (6). - - - A local transfer no longer duplicates all its include/exclude options - (since the forked process already has a copy of the exclude list, - there's no need to send them a set of duplicates). - - - The output of the items that are being updated by the generator (dirs, - symlinks, devices) is now intermingled in the proper order with the - output from the items that the receiver is updating (regular files) - when pulling. This misordering was particularly bad when --progress - was specified. (Requires protocol 29.) - - - When --timeout is specified, lulls that occur in the transfer while - the generator is doing work that does not generate socket traffic - (looking for changed files, deleting files, doing directory-time - touch-ups, etc.) will cause a new keep-alive packet to be sent that - should keep the transfer going as long as the generator continues to - make progress. (Requires protocol 29.) - - - The stat size of a device is not added to the total file size of the - items in the transfer (the size might be undefined on some OSes). - - - Fixed a problem with refused-option messages sometimes not making it - back to the client side when a remote --files-from was in effect and - the daemon was the receiver. - - - The --compare-dest option was not updating a file that differed in - (the preserved) attributes from the version in the compare-dest DIR. - - - When rsync is copying files into a write-protected directory, fixed - the change-report output for the directory so that we don't report - an identical directory as changed. - - ENHANCEMENTS: - - - Rsync now supports popt's option aliases, which means that you can - use /etc/popt and/or ~/.popt to create your own option aliases. - - - Added the --delete-during (--del) option which will delete files - from the receiving side incrementally as each directory in the - transfer is being processed. This makes it more efficient than the - default, before-the-transfer behavior, which is now also available as - --delete-before (and is still the default --delete-WHEN option that - will be chosen if --delete or --delete-excluded is specified without - a --delete-WHEN choice). All the --del* options infer --delete, so - an rsync daemon that refuses "delete" will still refuse to allow any - file-deleting options (including the new --remove-sent-files option). - - - All the --delete-WHEN options are now more memory efficient: - Previously an duplicate set of file-list objects was created on the - receiving side for the entire destination hierarchy. The new - algorithm only creates one directory of objects at a time (for files - inside the transfer). - - - Added the --copy-dest option, which works like --link-dest except - that it locally copies identical files instead of hard-linking them. - - - Added support for specifying multiple --compare-dest, --copy-dest, or - --link-dest options, but only of a single type. (Promoted from the - patches dir and enhanced.) (Requires protocol 29.) - - - Added the --max-size option. (Promoted from the patches dir.) - - - The daemon-mode options are now separated from the normal rsync - options so that they can't be mixed together. This makes it - impossible to start a daemon that has improper default option values - (which could cause problems when a client connects, such as hanging - or crashing). - - - The --bwlimit option may now be used in combination with --daemon - to specify both a default value for the daemon side and a value - that cannot be exceeded by a user-specified --bwlimit option. - - - Added the "port" parameter to the rsyncd.conf file. (Promoted from - the patches dir.) Also added "address". The command-line options - take precedence over a config-file option, as expected. - - - In _exit_cleanup(): when we are exiting with a partially-received - file, we now flush any data in the write-cache before closing the - partial file. - - - The --inplace support was enhanced to work with --compare-dest, - --link-dest, and (the new) --copy-dest options. (Requires protocol - 29.) - - - Added the --dirs (-d) option for an easier way to copy directories - without recursion. Any directories that are encountered are created - on the destination. Specifying a directory with a trailing slash - copies its immediate contents to the destination. - - - The --files-from option now implies --dirs (-d). - - - Added the --list-only option, which is mainly a way for the client to - put the server into listing mode without needing to resort to any - internal option kluges (e.g. the age-old use of "-r --exclude="/*/*" - for a non-recursive listing). This option is used automatically - (behind the scenes) when a modern rsync speaks to a modern daemon, - but may also be specified manually if you want to force the use of - the --list-only option over a remote-shell connection. - - - Added the --omit-dir-times (-O) option, which will avoid updating - the modified time for directories when --times was specified. This - option will avoid an extra pass through the file-list at the end of - the transfer (to tweak all the directory times), which may provide - an appreciable speedup for a really large transfer. (Promoted from - the patches dir.) - - - Added the --filter (-f) option and its helper option, -F. Filter - rules are an extension to the existing include/exclude handling - that also supports nested filter files as well as per-directory - filter files (like .cvsignore, but with full filter-rule parsing). - This new option was chosen in order to ensure that all existing - include/exclude processing remained 100% compatible with older - versions. Protocol 29 is needed for full filter-rule support, but - backward-compatible rules work with earlier protocol versions. - (Promoted from the patches dir and enhanced.) - - - Added the --delay-updates option that puts all updated files into - a temporary directory (by default ".~tmp~", but settable via the - --partial-dir=DIR option) until the end of the transfer. This - makes the updates a little more atomic for a large transfer. - - - If rsync is put into the background, any output from --progress is - reduced. - - - Documented the "max verbosity" setting for rsyncd.conf. (This - setting was added a couple releases ago, but left undocumented.) - - - The sender and the generator now double-check the file-list index - they are given, and refuse to try to do a file transfer on a - non-file index (since that would indicate that something had gone - very wrong). - - - Added the --itemize-changes (-i) option, which is a way to output a - more detailed list of what files changed and in what way. The effect - is the same as specifying a --log-format of "%i %n%L" (see both the - rsync and rsyncd.conf manpages). Works with --dry-run too. - - - Added the --fuzzy (-y) option, which attempts to find a basis file - for a file that is being created from scratch. The current algorithm - only looks in the destination directory for the created file, but it - does attempt to find a match based on size/mod-time (in case the file - was renamed with no other changes) as well as based on a fuzzy - name-matching algorithm. This option requires protocol 29 because it - needs the new file-sorting order. (Promoted from patches dir and - enhanced.) (Requires protocol 29.) - - - Added the --remove-sent-files option, which lets you move files - between systems. - - - The hostname in HOST:PATH or HOST::PATH may now be an IPv6 literal - enclosed in '[' and ']' (e.g. "[::1]"). (We already allowed IPv6 - literals in the rsync://HOST:PORT/PATH format.) - - - When rsync recurses to build the file list, it no longer keeps open - one or more directory handles from the dir's parent dirs. - - - When building under windows, the default for --daemon is now to - avoid detaching, requiring the new --detach option to force rsync - to detach. - - - The --dry-run option can now be combined with either --write-batch or - --read-batch, allowing you to run a do-nothing test command to see - what would happen without --dry-run. - - - The daemon's "read only" config item now sets an internal read_only - variable that makes extra sure that no write/delete calls on the - read-only side can succeed. - - - The log-format % escapes can now have a numeric field width in - between the % and the escape letter (e.g. "%-40n %08p"). - - - Improved the option descriptions in the --help text. - - SUPPORT FILES: - - - Added atomic-rsync to the support dir: a perl script that will - transfer some files using rsync, and then move the updated files into - place all at once at the end of the transfer. Only works when - pulling, and uses --link-dest and a parallel hierarchy of files to - effect its update. - - - Added mnt-excl to the support dir: a perl script that takes the - /proc/mounts file and translates it into a set of excludes that will - exclude all mount points (even mapped mounts to the same disk). The - excludes are made relative to the specified source dir and properly - anchored. - - - Added savetransfer.c to the support dir: a C program that can make - a copy of all the data that flows over the wire. This lets you test - for data corruption (by saving the data on both the sending side and - the receiving side) and provides one way to debug a protocol error. - - - Added rrsync to the support dir: this is an updated version of Joe - Smith's restricted rsync perl script. This helps to ensure that only - certain rsync commands can be run by an ssh invocation. - - INTERNAL: - - - Added better checking of the checksum-header values that come over - the socket. - - - Merged a variety of file-deleting functions into a single function so - that it is easier to maintain. - - - Improved the type of some variables (particularly blocksize vars) for - consistency and proper size. - - - Got rid of the uint64 type (which we didn't need). - - - Use a slightly more compatible set of core #include directives. - - - Defined int32 in a way that ensures that the build dies if we can't - find a variable with at least 32 bits. - - PROTOCOL DIFFERENCES FOR VERSION 29: - - - A 16-bit flag-word is transmitted after every file-list index. This - indicates what is changing between the sender and the receiver. The - generator now transmits an index and a flag-word to indicate when - dirs and symlinks have changed (instead of producing a message), - which makes the outputting of the information more consistent and - less prone to screen corruption (because the local receiver/sender is - now outputting all the file-change info messages). - - - If a file is being hard-linked, the ITEM_XNAME_FOLLOWS bit is enabled - in the flag-word and the name of the file that was linked immediately - follows in vstring format (see below). - - - If a file is being transferred with an alternate-basis file, the - ITEM_BASIS_TYPE_FOLLOWS bit is enabled in the flag-word and a single - byte follows, indicating what type of basis file was chosen. If that - indicates that a fuzzy-match was selected, the ITEM_XNAME_FOLLOWS bit - is set in the flag-word and the name of the match in vstring format - follows the basis byte. A vstring is a variable length string that - has its size written prior to the string, and no terminating null. - If the string is from 1-127 bytes, the length is a single byte. If - it is from 128-32767 bytes, the length is written as ((len >> 8) | - 0x80) followed by (len % 0x100). - - - The sending of exclude names is done using filter-rule syntax. This - means that all names have a prefixed rule indicator, even excludes - (which used to be sent as a bare pattern, when possible). The -C - option will include the per-dir .cvsignore merge file in the list of - filter rules so it is positioned correctly (unlike in some older - transfer scenarios). - - - Rsync sorts the filename list in a different way: it sorts the subdir - names after the non-subdir names for each dir's contents, and it - always puts a dir's contents immediately after the dir's name in the - list. (Previously an item named "foo.txt" would sort in between - directory "foo/" and "foo/bar".) - - - When talking to a protocol 29 rsync daemon, a list-only request - is able to note this before the options are sent over the wire and - the new --list-only option is included in the options. - - - When the --stats bytes are sent over the wire (or stored in a batch), - they now include two elapsed-time values: one for how long it took to - build the file-list, and one for how long it took to send it over the - wire (each expressed in thousandths of a second). - - - When --delete-excluded is specified with some filter rules (AKA - excludes), a client sender will now initiate a send of the rules to - the receiver (older protocols used to omit the sending of excludes in - this situation since there were no receiver-specific rules that - survived --delete-excluded back then). Note that, as with all the - filter-list sending, only items that are significant to the other - side will actually be sent over the wire, so the filter-rule list - that is sent in this scenario is often empty. - - - An index equal to the file-list count is sent as a keep-alive packet - from the generator to the sender, which then forwards it on to the - receiver. This normally invalid index is only a valid keep-alive - packet if the 16-bit flag-word that follows it contains a single bit - (ITEM_IS_NEW, which is normally an illegal flag to appear alone). - - - A protocol-29 batch file includes a bit for the setting of the --dirs - option and for the setting of the --compress option. Also, the shell - script created by --write-batch will use the --filter option instead - of --exclude-from to capture any filter rules. - - BUILD CHANGES: - - - Handle an operating system that use mkdev() in place of makedev(). - - - Improved configure to better handle cross-compiling. - - -NEWS for rsync 2.6.3 (30 Sep 2004) -Protocol: 28 (unchanged) -Changes since 2.6.2: - - SECURITY FIXES: - - - A bug in the sanitize_path routine (which affects a non-chrooted - rsync daemon) could allow a user to craft a pathname that would get - transformed into an absolute path for certain options (but not for - file-transfer names). If you're running an rsync daemon with chroot - disabled, *please upgrade*, ESPECIALLY if the user privs you run - rsync under is anything above "nobody". - - OUTPUT CHANGES (ATTN: those using a script to parse the verbose output): - - - Please note that the 2-line footer (output when verbose) now uses the - term "sent" instead of "wrote" and "received" instead of "read". If - you are not parsing the numeric values out of this footer, a script - would be better off using the empty line prior to the footer as the - indicator that the verbose output is over. - - - The output from the --stats option was similarly affected to change - "written" to "sent" and "read" to "received". - - - Rsync ensures that a filename that contains a newline gets mentioned - with each newline transformed into a question mark (which prevents a - filename from causing an empty line to be output). - - - The "backed up ..." message that is output when at least 2 --verbose - options are specified is now the same both with and without the - --backup-dir option. - - BUG FIXES: - - - Fixed a crash bug that might appear when --delete was used and - multiple source directories were specified. - - - Fixed a 32-bit truncation of the file length when generating the - checksums. - - - The --backup code no longer attempts to create some directories - over and over again (generating warnings along the way). - - - Fixed a bug in the reading of the secrets file (by the daemon) and - the password file (by the client): the files no longer need to be - terminated by a newline for their content to be read in. - - - If a file has a read error on the sending side or the reconstructed - data doesn't match the expected checksum (perhaps due to the basis - file changing during the transfer), the receiver will no longer - retain the resulting file unless the --partial option was specified. - (Note: for the read-error detection to work, neither side can be - older than 2.6.3 -- older receivers will always retain the file, and - older senders don't tell the receiver that the file had a read - error.) - - - If a file gets resent in a single transfer and the --backup option - is enabled, rsync no longer performs a duplicate backup (it used to - overwrite the original file in the backup area). - - - Files specified in the daemon's "exclude" or "exclude from" config - items are now excluded from being uploaded (assuming that the module - allows uploading at all) in addition to the old download exclusion. - - - Got rid of a potential hang in the receiver when near the end of a - phase. - - - When using --backup without a --backup-dir, rsync no longer preserves - the modify time on directories. This avoids confusing NFS. - - - When --copy-links (-L) is specified, we now output a separate error - for a symlink that has no referent instead of claiming that a file - "vanished". - - - The --copy-links (-L) option no longer has the side-effect of telling - the receiving side to follow symlinks. See the --keep-dirlinks - option (mentioned below) for a way to specify that behavior. - - - Error messages from the daemon server's option-parsing (such as - refused options) are now successfully transferred back to the client - (the server used to fail to send the message because the socket - wasn't in the right state for the message to get through). - - - Most transfer errors that occur during a daemon transfer are now - returned to the user in addition to being logged (some messages are - intended to be daemon-only and are not affected by this). - - - Fixed a bug in the daemon authentication code when using one of the - batch-processing options. - - - We try to work around some buggy IPv6 implementations that fail to - implement IPV6_V6ONLY. This should fix the "address in use" error - that some daemons get when running on an OS with a buggy IPv6 - implementation. Also, if the new code gets this error, we might - suggest that the user specify --ipv4 or --ipv6 (if we think it will - help). - - - When the remote rsync dies, make a better effort to recover any error - messages it may have sent before dying (the local rsync used to just - die with a socket-write error). - - - When using --delete and a --backup-dir that contains files that are - hard-linked to their destination equivalents, rsync now makes sure - that removed files really get removed (avoids a really weird rename() - behavior). - - - Avoid a bogus run-time complaint about a lack of 64-bit integers when - the int64 type is defined as an off_t and it actually has 64-bits. - - - Added a configure check for open64() without mkstemp64() so that we - can avoid using mkstemp() when such a combination is encountered. - This bypasses a problem writing out large temp files on OSes such as - AIX and HP-UX. - - - Fixed an age-old crash problem with --read-batch on a local copy - (rsync was improperly assuming --whole-file for the local copy). - - - When --dry-run (-n) is used and the destination directory does not - exist, rsync now produces a correct report of files that would be - sent instead of dying with a chdir() error. - - - Fixed a bug that could cause a slow-to-connect rsync daemon to die - with an error instead of waiting for the connection to finish. - - - Fixed an ssh interaction that could cause output to be lost when the - user chose to combine the output of rsync's stdout and stderr (e.g. - using the "2>&1"). - - - Fixed an option-parsing bug when --files-from got passed to a daemon. - - ENHANCEMENTS: - - - Added the --partial-dir=DIR option that lets you specify where to - (temporarily) put a partially transferred file (instead of over- - writing the destination file). E.g. --partial-dir=.rsync-partial - Also added support for the RSYNC_PARTIAL_DIR environment variable - that, when found, transforms a regular --partial option (such as - the convenient -P option) into one that also specifies a directory. - - - Added --keep-dirlinks (-K), which allows you to symlink a directory - onto another partition on the receiving side and have rsync treat it - as matching a normal directory from the sender. - - - Added the --inplace option that tells rsync to write each destination - file without using a temporary file. The matching of existing data - in the destination file can be severely limited by this, but there - are also cases where this is more efficient (such as appending data). - Use only when needed (see the man page for more details). - - - Added the "write only" option for the daemon's config file. - - - Added long-option names for -4 and -6 (namely --ipv4 and --ipv6) - and documented all these options in the man page. - - - Improved the handling of the --bwlimit option so that it's less - bursty, more accurate, and works properly over a larger range of - values. - - - The rsync daemon-over-ssh code now looks for SSH_CONNECTION and - SSH2_CLIENT in addition to SSH_CLIENT to figure out the IP address. - - - Added the --checksum-seed=N option for advanced users. - - - Batch writing/reading has a brand-new implementation that is simpler, - fixes a few weird problems with the old code (such as no longer - sprinkling the batch files into different dirs or even onto different - systems), and is much less intrusive into the code (making it easier - to maintain for the future). The new code generates just one data - file instead of three, which makes it possible to read the batch on - stdin via a remote shell. Also, the old requirement of forcing the - same fixed checksum-seed for all batch processing has been removed. - - - If an rsync daemon has a module set with "list = no" (which hides its - presence in the list of available modules), a user that fails to - authenticate gets the same "unknown module" error that they would get - if the module were actually unknown (while still logging the real - error to the daemon's log file). This prevents fishing for module - names. - - - The daemon's "refuse options" config item now allows you to match - option names using wildcards and/or the single-letter option names. - - - Each transferred file now gets its permissions and modified-time - updated before the temp-file gets moved into place. Previously, the - finished file would have a very brief window where its permissions - disallowed all group and world access. - - - Added the ability to parse a literal IPv6 address in an "rsync:" URL - (e.g. rsync://[2001:638:500:101::21]:873/module/dir). - - - The daemon's wildcard expanding code can now handle more than 1000 - filenames (it's now limited by memory instead of having a hard-wired - limit). - - INTERNAL: - - - Some cleanup in the exclude code has saved some per-exclude memory - and made the code easier to maintain. - - - Improved the argv-overflow checking for a remote command that has a - lot of args. - - - Use rsyserr() in the various places that were still calling rprintf() - with strerror() as an arg. - - - If an rsync daemon is listening on multiple sockets (to handle both - IPv4 and IPv6 to a single port), we now close all the unneeded file - handles after we accept a connection (we used to close just one of - them). - - - Optimized the handling of larger block sizes (rsync used to slow to a - crawl if the block size got too large). - - - Optimized away a loop in hash_search(). - - - Some improvements to the sanitize_path() and clean_fname() functions - makes them more efficient and produce better results (while still - being compatible with the file-name cleaning that gets done on both - sides when sending the file-list). - - - Got rid of alloc_sanitize_path() after adding a destination-buffer - arg to sanitize_path() made it possible to put all the former's - functionality into the latter. - - - The file-list that is output when at least 4 verbose options are - specified reports the uid value on the sender even when rsync is - not running as root (since we might be sending to a root receiver). - - BUILD CHANGES: - - - Added a "gen" target to rebuild most of the generated files, - including configure, config.h.in, the man pages, and proto.h. - - - If "make proto" doesn't find some changes in the prototypes, the - proto.h file is left untouched (its time-stamp used to always be - updated). - - - The variable $STRIP (that is optionally set by the install-strip - target's rule) was changed to $INSTALL_STRIP because some systems - have $STRIP already set in the environment. - - - Fixed a build problem when SUPPORT_HARD_LINKS isn't defined. - - - When cross-compiling, the gettimeofday() function is now assumed to - be a modern version that takes two-args (since we can't test it). - - DEVELOPER RELATED: - - - The scripts in the testsuite dir were cleaned up a bit and a few - new tests added. - - - Some new diffs were added to the patches dir, and some accepted - ones were removed. - - -NEWS for rsync 2.6.2 (30 Apr 2004) -Protocol: 28 (unchanged) -Changes since 2.6.1: - - BUG FIXES: - - - Fixed a major bug in the sorting of the filenames when --relative - is used for some sources (just sources such as "/" and "/*" were - affected). This fix ensures that we ask for the right file-list - item when requesting changes from the sender. - - - Rsync now checks the return value of the close() function to - better report disk-full problems on an NFS file system. - - - Restored the old daemon-server behavior of logging error messages - rather than returning them to the user. (A better long-term fix - will be sought in the future.) - - - An obscure uninitialized-variable bug was fixed in the uid/gid - code. (This bug probably had no ill effects.) - - BUILD CHANGES: - - - Got rid of the configure check for sys/sysctl.h (it wasn't used - and was causing a problem on some systems). Also improved the - broken-largefile-locking test to try to avoid failure due to an - NFS build-dir. - - - Fixed a compile problem on systems that don't define - AI_NUMERICHOST. - - - Fixed a compile problem in the popt source for compilers that - don't support __attribute__. - - DEVELOPER RELATED: - - - Improved the testsuite's "merge" test to work on OSF1. - - - Two new diffs were added to the patches dir. - - -NEWS for rsync 2.6.1 (26 Apr 2004) -Protocol: 28 (changed) -Changes since 2.6.0: - - SECURITY FIXES: - - - Paths sent to an rsync daemon are more thoroughly sanitized when - chroot is not used. If you're running a non-read-only rsync - daemon with chroot disabled, *please upgrade*, ESPECIALLY if the - user privs you run rsync under is anything above "nobody". - - ENHANCEMENTS: - - - Lower memory use, more optimal transfer of data over the socket, - and lower CPU usage (see the INTERNAL section for details). - - - The RSYNC_PROXY environment variable can now contain a - "USER:PASS@" prefix before the "HOST:PORT" information. - (Bardur Arantsson) - - - The --progress output now mentions how far along in the transfer - we are, including both a count of files transferred and a - percentage of the total file-count that we've processed. It also - shows better current-rate-of-transfer and remaining-transfer-time - values. - - - Documentation changes now attempt to describe some often mis- - understood features more clearly. - - BUG FIXES: - - - When -x (--one-file-system) is combined with -L (--copy-links) or - --copy-unsafe-links, no symlinked files are skipped, even if the - referent file is on a different filesystem. - - - The --link-dest code now works properly for a non-root user when - (1) the UIDs of the source and destination differ and -o was - specified, or (2) when the group of the source can't be used on - the destination and -g was specified. - - - Fixed a bug in the handling of -H (hard-links) that might cause - the expanded PATH/NAME value of the current item to get - overwritten (due to an expanded-name caching bug). - - - We now reset the "new data has been sent" flag at the start of - each file we send. This makes sure that an interrupted transfer - with the --partial option set doesn't keep a shorter temp file - than the current basis file when no new data has been transferred - over the wire for that file. - - - Fixed a byte-order problem in --batch-mode on big-endian machines. - (Jay Fenlason) - - - When using --cvs-exclude, the exclude items we get from a - per-directory's .cvsignore file once again only affect that one - directory (not all following directories too). The items are also - now properly word-split and parsed without any +/- prefix parsing. - - - When specifying the USER@HOST: prefix for a file, the USER part - can now contain an '@', if needed (i.e. the last '@' is used to - find the HOST, not the first). - - - Fixed some bugs in the handling of group IDs for non-root users: - (1) It properly handles a group that the sender didn't have a name - for (it would previously skip changing the group on any files in - that group). (2) If --numeric-ids is used, rsync no longer - attempts to set groups that the user doesn't have the permission - to set. - - - Fixed the "refuse options" setting in the rsyncd.conf file. - - - Improved the -x (--one-file-system) flag's handling of any mount- - point directories we encounter. It is both more optimal (in that - it no longer does a useless scan of the contents of the mount- - point dirs) and also fixes a bug where a remapped mount of the - original filesystem could get discovered in a subdir we should be - ignoring. - - - Rsync no longer discards a double-slash at the start of a filename - when trying to open the file. It also no longer constructs names - that start with a double slash (unless the user supplied them). - - - Path-specifying options to a daemon should now work the same with - or without chroot turned on. Previously, such a option (such as - --link-dest) would get its absolute path munged into a relative - one if chroot was not on, making that setting fairly useless. - Rsync now transforms the path into one that is based on the - module's base dir when chroot is not enabled. - - - Fixed a compatibility problem interacting with older rsync - versions that might send us an empty --suffix value without - telling us that --backup-dir was specified. - - - The "hosts allow" option for a daemon-over-remote-shell process - now has improved support for IPv6 addresses and a fix for systems - that have a length field in their socket structs. - - - Fixed the ability to request an empty backup --suffix when sending - files to an rsync daemon. - - - Fixed an option-parsing bug when --files-from was sent to a server - sender. - - INTERNAL: - - - Most of the I/O is now buffered, which results in a pretty large - speedup when running under MS Windows. (Craig Barratt) - - - Optimizations to the name-handling/comparing code have made some - significant reductions in user-CPU time for large file sets. - - - Some cleanup of the variable types make the code more consistent. - - - Reduced memory requirements of hard link preservation. - (J.W. Schultz) - - - Implemented a new algorithm for hard-link handling that speeds up - the code significantly. (J.W. Schultz and Wayne Davison) - - - The --hard-link option now uses the first existing file in the - group of linked files as the basis for the transfer. This - prevents the sub-optimal transfer of a file's data when a new - hardlink is added on the sending side and it sorts alphabetically - earlier in the list than the files that are already present on the - receiving side. - - - Dropped support for protocol versions less than 20 (2.3.0 released - 15 Mar 1999) and activated warnings for protocols less than 25 - (2.5.0 released 23 Aug 2001). (Wayne Davison and J.W. Schultz, - severally) - - - More optimal data transmission for --hard-links (protocol 28). - - - More optimal data transmission for --checksum (protocol 28). - - - Less memory is used when --checksum is specified. - - - Less memory is used in the file list (a per-file savings). - - - The generator is now better about not modifying the file list - during the transfer in order to avoid a copy-on-write memory - bifurcation (on systems where fork() uses shared memory). - Previously, rsync's shared memory would slowly become unshared, - resulting in real memory usage nearly doubling on the receiving - side by the end of the transfer. Now, as long as permissions - are being preserved, the shared memory should remain that way - for the entire transfer. - - - Changed hardlink info and file_struct + strings to use allocation - pools. This reduces memory use for large file-sets and permits - freeing memory to the OS. (J.W. Schultz) - - - The 2 pipes used between the receiver and generator processes - (which are forked on the same machine) were reduced to 1 pipe and - the protocol improved so that (1) it is now impossible to have the - "redo" pipe fill up and hang rsync, and (2) trailing messages from - the receiver don't get lost on their way through the generator - over to the sender (which mainly affected hard-link messages and - verbose --stats output). - - - Improved the internal uid/gid code to be more portable and a - little more optimized. - - - The device numbers sent when using --devices are now sent as - separate major/minor values with 32-bit accuracy (protocol 28). - Previously, the copied devices were sent as a single 32-bit - number. This will make inter-operation of 64-bit binaries more - compatible with their 32-bit brethren (with both ends of the - connection are using protocol 28). Note that optimizations in the - binary protocol for sending the device numbers often results in - fewer bytes being used than before, even though more precision is - now available. - - - Some cleanup of the exclude/include structures and its code made - things clearer (internally), simpler, and more efficient. - - - The reading & writing of the file-list in batch-mode is now - handled by the same code that sends & receives the list over the - wire. This makes it much easier to maintain. (Note that the - batch code is still considered to be experimental.) - - BUILD CHANGES: - - - The configure script now accepts --with-rsyncd-conf=PATH to - override the default value of the /etc/rsyncd.conf file. - - - Fixed configure bug when running "./configure --disable-ipv6". - - - Fixed compilation problem on Tru64 Unix (having to do with - sockaddr.sa_len and sockaddr.sin_len). - - DEVELOPER RELATED: - - - Fixed "make test" bug when build dir is not the source dir. - - - Added a couple extra diffs in the "patches" dir, removed the ones - that got applied, and rebuilt the rest. - - -NEWS for rsync 2.6.0 (1 Jan 2004) -Protocol: 27 (changed) -Changes since 2.5.7: - - ENHANCEMENTS: - - * "ssh" is now the default remote shell for rsync. If you want to - change this, configure like this: "./configure --with-rsh=rsh". - - * Added --files-from, --no-relative, --no-implied-dirs, and --from0. - Note that --from0 affects the line-ending character for all the - files read by the --*-from options. (Wayne Davison) - - * Length of csum2 is now per-file starting with protocol version - 27. (J.W. Schultz) - - * Per-file dynamic block size is now sqrt(file length). The - per-file checksum size is determined according to an algorithm - provided by Donovan Baarda which reduces the probability of rsync - algorithm corrupting data and falling back using the whole md4 - checksums. (J.W. Schultz, Donovan Baarda) - - * The --stats option no longer includes the (debug) malloc summary - unless the verbose option was specified at least twice. - - * Added a new error/warning code for when files vanish from the - sending side. Made vanished source files not interfere with the - file-deletion pass when --delete-after was specified. - - * Various trailing-info sections are now preceded by a newline. - - BUG FIXES: - - * Fixed several exclude/include matching bugs when using wild-cards. - This has a several user-visible effects, all of which make the - matching more consistent and intuitive. This should hopefully not - cause anyone problems since it makes the matching work more like - what people are expecting. (Wayne Davison) - - - A pattern with a "**" no longer causes a "*" to match slashes. - For example, with "/*/foo/**", "foo" must be 2 levels deep. - [If your string has BOTH "*" and "**" wildcards, changing the - "*" wildcards to "**" will provide the old behavior in all - versions.] - - - "**/foo" now matches at the base of the transfer (like /foo - does). [Use "/**/foo" to get the old behavior in all versions.] - - - A non-anchored wildcard term floats to match beyond the base of - the transfer. E.g. "CVS/R*" matches at the end of the path, - just like the non-wildcard term "CVS/Root" does. [Use "/CVS/R*" - to get the old behavior in all versions.] - - - Including a "**" in the match term causes it to be matched - against the entire path, not just the name portion, even if - there aren't any interior slashes in the term. E.g. "foo**bar" - would exclude "/path/foo-bar" (just like before) as well as - "/foo-path/baz-bar" (unlike before). [Use "foo*bar" to get the - old behavior in all versions.] - - * The exclude list specified in the daemon's config file is now - properly applied to the pulled items no matter how deep the - user's file-args are in the source tree. (Wayne Davison) - - * For protocol version >= 27, mdfour_tail() is called when the - block size (including checksum_seed) is a multiple of 64. - Previously it was not called, giving the wrong MD4 checksum. - (Craig Barratt) - - * For protocol version >= 27, a 64 bit bit counter is used in - mdfour.c as required by the RFC. Previously only a 32 bit bit - counter was used, causing incorrect MD4 file checksums for - file sizes >= 512MB - 4. (Craig Barratt) - - * Fixed a crash bug when interacting with older rsync versions and - multiple files of the same name are destined for the same dir. - (Wayne Davison) - - * Keep tmp names from overflowing MAXPATHLEN. - - * Make --link-dest honor the absence of -p, -o, and -g. - - * Made rsync treat a trailing slash in the destination in a more - consistent manner. - - * Fixed file I/O error detection. (John Van Essen) - - * Fixed bogus "malformed address {hostname}" message in rsyncd log - when checking IP address against hostnames from "hosts allow" - and "hosts deny" parameters in config file. - - * Print heap statistics when verbose >= 2 instead of when >= 1. - - * Fixed a compression (-z) bug when syncing a mostly-matching file - that contains already-compressed data. (Yasuoka Masahiko and - Wayne Davison) - - * Fixed a bug in the --backup code that could cause deleted files - to not get backed up. - - * When the backup code makes new directories, create them with mode - 0700 instead of 0755 (since the directory permissions in the - backup tree are not yet copied from the main tree). - - * Call setgroups() in a more portable manner. - - * Improved file-related error messages to better indicate exactly - what pathname failed. (Wayne Davison) - - * Fixed some bugs in the handling of --delete and --exclude when - using the --relative (-R) option. (Wayne Davison) - - * Fixed bug that prevented regular files from replacing - special files and caused a directory in --link-dest or - --compare-dest to block the creation of a file with the - same path. A directory still cannot be replaced by a - regular file unless --delete specified. (J.W. Schultz) - - * Detect and report when open or opendir succeed but read and - readdir fail caused by network filesystem issues and truncated - files. (David Norwood, Michael Brown, J.W. Schultz) - - * Added a fix that should give ssh time to restore the tty settings - if the user presses Ctrl-C at an ssh password prompt. - - INTERNAL: - - * Eliminated vestigial support for old versions that we stopped - supporting. (J.W. Schultz) - - * Simplified some of the option-parsing code. (Wayne Davison) - - * Some cleanup made to the exclude code, as well as some new - defines added to enhance readability. (Wayne Davison) - - * Changed the protocol-version code so that it can interact at a - lower protocol level than the maximum supported by both sides. - Added an undocumented option, --protocol=N, to force the value - we advertise to the other side (primarily for testing purposes). - (Wayne Davison) - - -NEWS for rsync 2.5.7 (4 Dec 2003) -Protocol: 26 (unchanged) -Changes since 2.5.6: - - SECURITY FIXES: - - * Fix buffer handling bugs. (Andrew Tridgell, Martin Pool, Paul - Russell, Andrea Barisani) - - -NEWS for rsync 2.5.6, aka "the dwd-between-jobs release" (26 Jan 2003) -Protocol: 26 (unchanged) -Changes since 2.5.5: - - ENHANCEMENTS: - - * The --delete-after option now implies --delete. (Wayne Davison) - - * The --suffix option can now be used with --backup-dir. (Michael - Zimmerman) - - * Combining "::" syntax with the --rsh/-e option now uses the - specified remote-shell as a transport to talk to a (newly-spawned) - server-daemon. This allows someone to use daemon features, such - as modules, over a secure protocol, such as ssh. (JD Paul) - - * The rsync:// syntax for daemon connections is now accepted in the - destination field. - - * If the file name given to --include-from or --exclude-from is "-", - rsync will read from standard input. (J.W. Schultz) - - * New option --link-dest which is like --compare-dest except that - unchanged files are hard-linked in to the destination directory. - (J.W. Schultz) - - * Don't report an error if an excluded file disappears during an - rsync run. (Eugene Chupriyanov and Bo Kersey) - - * Added .svn to --cvs-exclude list to support subversion. (Jon - Middleton) - - * Properly support IPv6 addresses in the rsyncd.conf "hosts allow" - and "hosts deny" fields. (Hideaki Yoshifuji) - - * Changed exclude file handling to permit DOS or MAC style line - terminations. (J.W. Schultz) - - * Ignore errors from chmod when -p/-a/--preserve-perms is not set. - (Dave Dykstra) - - BUG FIXES: - - * Fix "forward name lookup failed" errors on AIX 4.3.3. (John - L. Allen, Martin Pool) - - * Generate each file's rolling-checksum data as we send it, not - in a separate (memory-eating) pass before hand. This prevents - timeout errors on really large files. (Stefan Nehlsen) - - * Fix compilation on Tru64. (Albert Chin, Zoong Pham) - - * Better handling of some client-server errors. (Martin Pool) - - * Fixed a crash that would occur when sending a list of files that - contains a duplicate name (if it sorts to the end of the file - list) and using --delete. (Wayne Davison) - - * Fixed the file-name duplicate-removal code when dealing with multiple - dups in a row. (Wayne Davison) - - * Fixed a bug that caused rsync to lose the exit status of its child - processes and sometimes return an exit code of 0 instead of showing - an error. (David R. Staples, Dave Dykstra) - - * Fixed bug in --copy-unsafe-links that caused it to be completely - broken. (Dave Dykstra) - - * Prevent infinite recursion in cleanup code under certain circumstances. - (Sviatoslav Sviridov and Marc Espie) - - * Fixed a bug that prevented rsync from creating intervening directories - when --relative-paths/-R is set. (Craig Barratt) - - * Prevent "Connection reset by peer" messages from Cygwin. (Randy O'Meara) - - INTERNAL: - - * Many code cleanups and improved internal documentation. (Martin - Pool, Nelson Beebe) - - * Portability fixes. (Dave Dykstra and Wayne Davison) - - * More test cases. (Martin Pool) - - * Some test-case fixes. (Brian Poole, Wayne Davison) - - * Updated included popt to the latest vendor drop, version 1.6.4. - (Jos Backus) - - * Updated config.guess and config.sub to latest versions; this - means rsync should build on more platforms. (Paul Green) - - -NEWS for rsync 2.5.5, aka Snowy River (2 Apr 2002) -Protocol: 26 (unchanged) -Changes since 2.5.4: - - ENHANCEMENTS: - - * With --progress, when a transfer is complete show the time taken; - otherwise show expected time to complete. (Cameron Simpson) - - * Make "make install-strip" works properly, and "make install" - accepts a DESTDIR variable for help in building binary packages. - (Peter Breitenlohner, Greg Louis) - - * If configured with --enable-maintainer-mode, then on receipt of - a fatal signal rsync will try to open an xterm running gdb, - similarly to Samba's "panic action" or GNOME's bug-buddy. - (Martin Pool) - - - BUG FIXES: - - * Fix situation where failure to fork (e.g. because out of process - slots) would cause rsync to kill all processes owned by the - current user. Yes, really! (Paul Haas, Martin Pool) - - * Fix test suite on Solaris. (Jos Backus, Martin Pool) - - * Fix minor memory leak in socket code. (Dave Dykstra, Martin - Pool.) - - * Fix --whole-file problem that caused it to be the default even - for remote connections. (Martin Pool, Frank Schulz) - - * Work around bug in Mac OS X mkdir(2), which cannot handle - trailing slashes. - <http://www.opensource.apple.com/bugs/X/BSD%20Kernel/2734739.html> - (Martin Pool) - - * Improved network error handling. (Greg A. Woods) - - -NEWS for rsync 2.5.4, aka "Imitation lizard skin" (13 Mar 2002) -Protocol: 26 (unchanged) -Changes since 2.5.3: - - BUG FIXES: - - * Additional fix for zlib double-free bug. (Martin Pool, Andrew - Tridgell) (CVE CAN-2002-0059) - - ENHANCEMENTS: - - * Merge in changes from zlib 1.1.3 to zlib 1.1.4. (Jos Backus) - (Note that rsync still uses a custom version of zlib; you can - not just link against a system library. See zlib/README.rsync) - - * Additional test cases for --compress. (Martin Pool) - - -NEWS for rsync 2.5.3, aka "Happy 26" (11 Mar 2002) -Protocol: 26 (unchanged) -Changes since 2.5.2: - - SECURITY FIXES: - - * Make sure that supplementary groups are removed from a server - process after changing uid and gid. (Ethan Benson) (Debian bug - #132272, CVE CAN-2002-0080) - - BUG FIXES: - - * Fix zlib double-free bug. (Owen Taylor, Mark J Cox) (CVE - CAN-2002-0059) - - * Fixed problem that in many cases caused the error message - unexpected read size of 0 in map_ptr - and resulted in the wrong data being copied. - - * Fixed compilation errors on some systems caused by the use of - "unsigned int64" in rsync.h. - - * Fixed problem on systems such as Sunos4 that do not support realloc - on a NULL pointer; error was "out of memory in flist_expand". - - * Fix for rsync server processes hanging around after the client - unexpectedly disconnects. (Colin Walters) (Debian bug #128632) - - * Cope with BSD systems on which mkdir() will not accept a trailing - slash. - - ENHANCEMENTS: - - * Merge in changes from zlib 1.1.2 to zlib 1.1.3. (Note that - rsync still uses a custom version of zlib; you can not just link - against a system library. See zlib/README.rsync) - - * Command to initiate connections is only shown with -vv, rather - than -v as in 2.5.2. Output from plain -v is more similar to - what was historically used so as not to break scripts that try - to parse the output. - - * Added --no-whole-file and --no-blocking-io options (Dave Dykstra) - - * Made the --write-batch and --read-batch options actually work - and added documentation in the man page (Jos Backus) - - * If the daemon is unable to fork a child to accept a connection, - print an error message. (Colin Walters) - - -NEWS for rsync 2.5.2 (26 Jan 2002) -Protocol: 26 (changed) -Changes since 2.5.1: - - SECURITY FIXES: - - * Signedness security patch from Sebastian Krahmer - <krahmer@suse.de> -- in some cases we were not sufficiently - careful about reading integers from the network. - - BUG FIXES: - - * Fix possible string mangling in log files. - - * Fix for setting local address of outgoing sockets. - - * Better handling of hardlinks and devices on platforms with - 64-bit dev_t or ino_t. - - * Name resolution on machines supporting IPv6 is improved. - - * Fix for device nodes. (dann frazier) (Debian #129135) - - ENHANCEMENTS: - - * With -v, rsync now shows the command used to initiate an ssh/rsh - connection. - - * --statistics now shows memory heap usage on platforms that - support mallinfo(). - - * "The Ted T'so school of program optimization": make progress - visible and people will think it's faster. (With --progress, - rsync will show you how many files it has seen as it builds the - file_list, giving some indication that it has not hung.) - - * Improvements to batch mode support. This is still experimental - but testing would be welcome. (Jos Backus) - - * New --ignore-existing option, patch previously distributed with - Vipul's Razor. (Debian #124286) - - -NEWS for rsync 2.5.1 (3 Jan 2002) -Protocol: 25 (unchanged) -Changes since 2.5.0: - - BUG FIXES: - - * Fix for segfault in --daemon mode configuration parser. (Paul - Mackerras) - - * Correct string<->address parsing for both IPv4 and 6. - (YOSHIFUJI Hideaki, SUMIKAWA Munechika and Jun-ichiro "itojun" - Hagino) - - * Various fixes for IPv6 support. (Dave Dykstra) - - * rsync.1 typo fix. (Matt Kraai) - - * Test suite typo fixes. (Tom Schmidt) - - * rsync.1 grammar and clarity improvements. (Edward - Welbourne) - - * Correction to ./configure tests for inet_ntop. (Jeff Garzik) - - ENHANCEMENTS: - - * --progress and -P now show estimated data transfer rate (in a - multiple of bytes/s) and estimated time to completion. (Rik - Faith) - - * --no-detach option, required to run as a W32 service and also - useful when running on Unix under daemontools, AIX's SRC, or a - debugger. (Max Bowsher, Jos Backus) - - * Clearer error messages for some conditions. - - -NEWS for rsync 2.5.0 (30 Nov 2001) -Protocol: 25 (changed) -Changes since 2.4.6: - - ANNOUNCEMENTS - - * Martin Pool <mbp@samba.org> is now a co-maintainer. - - NEW FEATURES - - * Support for LSB-compliant packaging <http://www.linuxbase.org/> - - * Shell wildcards are allowed in "auth users" lines. - - * Merged UNC rsync+ patch to support creation of standalone patch - sets. By Bert J. Dempsey and Debra Weiss, updated by Jos - Backus. <http://www.ils.unc.edu/i2dsi/unc_rsync+.html> - - * IPv6 support based on a patch from KAME.net, on systems - including modern versions of Linux, Solaris, and HP-UX. Also - includes IPv6 compatibility functions for old OSs by the - Internet Software Consortium, Paul Vixie, the OpenSSH - portability project, and OpenBSD. - - ENHANCEMENTS - - * Include/exclude cluestick: with -vv, print out whether files are - included or excluded and why. - - * Many error messages have more friendly explanations and more - details. - - * Manual page improvements plus scanty protocol documentation. - - * When running as --daemon in the background and using a "log - file" rsyncd.conf directive, close the log file every time it is - open when going to sleep on the socket. This allows the log - file to get cleaned out by another process. - - * Change to using libpopt rather than getopt for processing - options. This makes the code cleaner and the behaviour more - consistent across platforms. popt is included and built if not - installed on the platform. - - * More details in --version, including note about whether 64-bit - files, symlinks and hardlinks are supported. - - * MD4 code may use less CPU cycles. - - * Use mkstemp on systems where it is secure. If we use mktemp, - explain that we do it in a secure way. - - * --whole-file is the default when source and target are on the - local machine. - - BUG FIXES: - - * Fix for various bugs causing rsync to hang. - - * Attempt to fix Large File Summit support on AIX. - - * Attempt to fix error handling lockup bug. - - * Give a non-0 exit code if *any* of the files we have been asked - to transfer fail to transfer. - - * For log messages containing ridiculously long strings that might - overflow a buffer rsync no longer aborts, but rather prints an - ellipsis at the end of the string. (Patch from Ed Santiago.) - - PLATFORMS: - - * Improved support for UNICOS (tested on Cray T3E and Cray SV1) - - * autoconf2.52 (or later) is now required to rebuild the autoconf - scripts. It is not required to simply build rsync. - - * Platforms thought to work in this release: - - Cray SV1 UNICOS 10.0.0.8 cc - Debian Linux 2.2 UltraSparc gcc - Debian Linux testing/unstable ARM gcc - FreeBSD 3.3-RELEASE i386 cc - FreeBSD 4.1.1-RELEASE i386 cc - FreeBSD 4.3-STABLE i386 cc - HP PA-RISC HP-UX 10.20 gcc - HP PA-RISC HP-UX 11.11 cc - IRIX 6.5 MIPS cc - IRIX 6.5 MIPS gcc - Mac OS X PPC (--disable-ipv6) cc - NetBSD 1.5 i386 gcc - NetBSD Current i386 cc - OpenBSD 2.5 Sparc gcc - OpenBSD 2.9 i386 cc - OpenBSD Current i386 cc - RedHat 6.2 i386 gcc - RedHat 6.2 i386 insure++ - RedHat 7.0 i386 gcc - RedHat 7.1 i386 (Kernel 2.4.10) gcc - Slackware 8.0 i686 (Kernel 2.4.10) - Solaris 8 UltraSparc cc - Solaris 8 UltraSparc gcc - Solaris 8 i386 gcc - SuSE 7.1 i386 gcc2.95.2 - SuSE 7.1 ppc gcc2.95.2 - i386-pc-sco3.2v5.0.5 cc - i386-pc-sco3.2v5.0.5 gcc - powerpc-ibm-aix4.3.3.0 cc - i686-unknown-sysv5UnixWare7.1.0 gcc - i686-unknown-sysv5UnixWare7.1.0 cc - - TESTING: - - * The existing test.sh script by Phil Hands has been merged into a - test framework that works from both "make check" and the Samba - build farm. - -Partial Protocol History - RELEASE DATE VER. DATE OF COMMIT* PROTOCOL - 28 Jan 2018 3.1.3 31 - 21 Dec 2015 3.1.2 31 - 22 Jun 2014 3.1.1 31 - 28 Sep 2013 3.1.0 31 Aug 2008 31 - 23 Sep 2011 3.0.9 30 - 26 Mar 2011 3.0.8 30 - 31 Dec 2009 3.0.7 30 - 08 May 2009 3.0.6 30 - 28 Dec 2008 3.0.5 30 - 06 Sep 2008 3.0.4 30 - 29 Jun 2008 3.0.3 30 - 08 Apr 2008 3.0.2 30 - 03 Apr 2008 3.0.1 30 - 01 Mar 2008 3.0.0 11 Nov 2006 30 - 06 Nov 2006 2.6.9 29 - 22 Apr 2006 2.6.8 29 - 11 Mar 2006 2.6.7 29 - 28 Jul 2005 2.6.6 29 - 01 Jun 2005 2.6.5 29 - 30 Mar 2005 2.6.4 17 Jan 2005 29 - 30 Sep 2004 2.6.3 28 - 30 Apr 2004 2.6.2 28 - 26 Apr 2004 2.6.1 08 Jan 2004 28 - 01 Jan 2004 2.6.0 10 Apr 2003 27 (MAX=40) - 04 Dec 2003 2.5.7 26 - 26 Jan 2003 2.5.6 26 - 02 Apr 2002 2.5.5 26 - 13 Mar 2002 2.5.4 26 - 11 Mar 2002 2.5.3 26 - 26 Jan 2002 2.5.2 11 Jan 2002 26 - 03 Jan 2002 2.5.1 25 - 30 Nov 2001 2.5.0 23 Aug 2001 25 - 06 Sep 2000 2.4.6 24 - 19 Aug 2000 2.4.5 24 - 29 Jul 2000 2.4.4 24 - 09 Apr 2000 2.4.3 24 - 30 Mar 2000 2.4.2 24 - 30 Jan 2000 2.4.1 29 Jan 2000 24 - 29 Jan 2000 2.4.0 28 Jan 2000 23 - 25 Jan 2000 2.3.3 23 Jan 2000 22 - 08 Nov 1999 2.3.2 26 Jun 1999 21 - 06 Apr 1999 2.3.1 20 - 15 Mar 1999 2.3.0 15 Mar 1999 20 - 25 Nov 1998 2.2.1 19 - 03 Nov 1998 2.2.0 19 - 09 Sep 1998 2.1.1 19 - 20 Jul 1998 2.1.0 19 - 17 Jul 1998 2.0.19 19 - 18 Jun 1998 2.0.17 19 - 01 Jun 1998 2.0.16 19 - 27 May 1998 2.0.13 27 May 1998 19 - 26 May 1998 2.0.12 18 - 22 May 1998 2.0.11 18 - 18 May 1998 2.0.9 18 May 1998 18 - 17 May 1998 2.0.8 17 - 15 May 1998 2.0.1 17 - 14 May 1998 2.0.0 17 - 17 Apr 1998 1.7.4 17 - 13 Apr 1998 1.7.3 17 - 05 Apr 1998 1.7.2 17 - 26 Mar 1998 1.7.1 17 - 26 Mar 1998 1.7.0 26 Mar 1998 17 (MAX=30) - 13 Jan 1998 1.6.9 13 Jan 1998 15 (MAX=20) - -* DATE OF COMMIT is the date the protocol change was committed to CVS. diff --git a/OLDNEWS.md b/OLDNEWS.md new file mode 100644 index 00000000..077a288b --- /dev/null +++ b/OLDNEWS.md @@ -0,0 +1,3863 @@ +# NEWS for rsync 3.1.3 (28 Jan 2018) + +Protocol: 31 (unchanged) + +## Changes since 3.1.2: + +### SECURITY FIXES: + + - Fixed a buffer overrun in the protocol's handling of xattr names and ensure + that the received name is null terminated. + + - Fix an issue with `--protect-args` where the user could specify the arg in the + protected-arg list and short-circuit some of the arg-sanitizing code. + +### BUG FIXES: + + - Don't output about a new backup dir without appropriate info verbosity. + + - Fixed some issues with the sort functions in support/rsyncstats script. + + - Added a way to specify daemon config lists (e.g. users, groups, etc) that + contain spaces (see `auth users` in the latest rsyncd.conf manpage). + + - If a backup fails (e.g. full disk) rsync exits with an error. + + - Fixed a problem with a doubled `--fuzzy` option combined with `--link-dest`. + + - Avoid invalid output in the summary if either the start or end time had an + error. + + - We don't allow a popt alias to affect the `--daemon` or `--server` options. + + - Fix daemon exclude code to disallow attribute changes in addition to + disallowing transfers. + + - Don't force nanoseconds to match if a non-transferred, non-checksummed file + only passed the quick-check w/o comparing nanosecods. + +### ENHANCEMENTS: + + - Added the ability for rsync to compare nanosecond times in its file-check + comparisons, and added support nanosecond times on Mac OS X. + + - Added a short-option (`-@`) for `--modify-window`. + + - Added the `--checksum-choice=NAME[,NAME]` option to choose the checksum + algorithms. + + - Added hashing of xattr names (with using `-X`) to improve the handling of + files with large numbers of xattrs. + + - Added a way to filter xattr names using include/exclude/filter rules (see + the `--xattrs` option in the manpage for details). + + - Added `daemon chroot|uid|gid` to the daemon config (in addition to the old + chroot|uid|gid settings that affect the daemon's transfer process). + + - Added `syslog tag` to the daemon configuration. + + - Some manpage improvements. + +### DEVELOPER RELATED: + + - Tweak the `make` output when yodl isn't around to create the man pages. + + - Changed an obsolete autoconf compile macro. + + - Support newer yodl versions when converting man pages. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.1.2 (21 Dec 2015) + +Protocol: 31 (unchanged) + +## Changes since 3.1.1: + +### SECURITY FIXES: + + - Make sure that all transferred files use only path names from inside the + transfer. This makes it impossible for a malicious sender to try to make the + receiver use an unsafe destination path for a transferred file, such as a + just-sent symlink. + +### BUG FIXES: + + - Change the checksum seed order in the per-block checksums. This prevents + someone from trying to create checksum blocks that match in sum but not + content. + + - Fixed a with the per-dir filter files (using `-FF`) that could trigger an + assert failure. + + - Only skip `set_modtime()` on a transferred file if the time is exactly + right. + + - Don't create an empty backup dir for a transferred file that doesn't exist + yet. + + - Fixed a bug where `--link-dest` and `--xattrs` could cause rsync to exit if + a filename had a matching dir of the same name in the alt-dest area. + + - Allow more than 32 group IDs per user in the daemon's gid=LIST config. + + - Fix the logging of %b & %c via `--log-file` (daemon logging was already + correct, as was `--out-format='%b/%c'`). + + - Fix erroneous acceptance of `--info=5` & `--debug=5` (an empty flag name is + not valid). + +### ENHANCEMENTS: + + - Added `(DRY RUN)` info to the `--debug=exit` output line. + + - Use usleep() for our msleep() function if it is available. + + - Added a few extra long-option names to rrsync script, which will make + BackupPC happier. + + - Made configure choose to use linux xattrs on netbsd (rather than not + supporting xattrs). + + - Added `-wo` (write-only) option to rrsync support script. + + - Misc. manpage tweaks. + +### DEVELOPER RELATED: + + - Fixed a bug with the Makefile's use of `INSTALL_STRIP`. + + - Improve a test in the suite that could get an erroneous timestamp error. + + - Tweaks for newer versions of git in the packaging tools. + + - Improved the m4 generation rules and some autoconf idioms. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.1.1 (22 Jun 2014) + +Protocol: 31 (unchanged) + +## Changes since 3.1.0: + +### BUG FIXES: + + - If the receiver gets bogus filenames from the sender (an unexpected leading + slash or a `..` infix dir), exit with an error. This prevents a malicious + sender from trying to inject filenames that would affect an area outside the + destination directories. + + - Fixed a failure to remove the partial-transfer temp file when interrupted + (and rsync is not saving the partial files). + + - Changed the chown/group/xattr-set order to avoid losing some security- + related xattr info (that would get cleared by a chown). + + - Fixed a bug in the xattr-finding code that could make a non-root-run + receiver not able to find some xattr numbers. + + - Fixed a bug in the early daemon protocol where a timeout failed to be + honored (e.g. if the remote side fails to send us the initial protocol + greeting). + + - Fixed unintended inclusion of commas in file numbers in the daemon log. + + - We once again send the 'f' sub-flag (of `-e`) to the server side so it knows + that we can handle incremental-recursion directory errors properly in older + protocols. + + - Fixed an issue with too-aggressive keep-alive messages causing a problem for + older rsync versions early in the transfer. + + - Fixed an incorrect message about backup-directory-creation when using + `--dry-run` and the backup dir is not an absolute path. + + - Fixed a bug where a failed deletion and/or a failed sender-side removal + would not affect the exit code. + + - Fixed a bug that caused a failure when combining `--delete-missing-args` + with `--xattrs` and/or `--acls`. + + - Fixed a strange `dir_depth` assertion error that was caused by empty-dir + removals and/or duplicate files in the transfer. + + - Fixed a problem with `--info=progress2`'s output stats where rsync would + only update the stats at the end of each file's transfer. It now uses the + data that is flowing for the current file, making the stats more accurate + and less jumpy. + + - Fixed an itemize bug that affected the combo of `--link-dest`, `-X`, and + `-n`. + + - Fixed a problem with delete messages not appearing in the log file when the + user didn't use `--verbose`. + + - Improve chunked xattr reading for OS X. + + - Removed an attempted hard-link xattr optimization that was causing a + transfer failure. This removal is flagged in the compatibility code, so if a + better fix can be discovered, we have a way to flip it on again. + + - Fixed a bug when the receiver is not configured to be able to hard link + symlimks/devices/special-file items but the sender sent some of these items + flagged as hard-linked. + + - We now generate a better error if the buffer overflows in `do_mknod()`. + + - Fixed a problem reading more than 16 ACLs on some OSes. + + - Fixed the reading of the secrets file to avoid an infinite wait when the + username is missing. + + - Fixed a parsing problem in the `--usermap`/`--groupmap` options when using + MIN-MAX numbers. + + - Switched Cygwin back to using socketpair `pipes` to try to speed it up. + + - Added knowledge of a few new options to rrsync. + +### ENHANCEMENTS: + + - Tweaked the temp-file naming when `--temp-dir=DIR` is used: the temp-file + names will not get a '.' prepended. + + - Added support for a new-compression idiom that does not compress all the + matching data in a transfer. This can help rsync to use less cpu when a + transfer has a lot of matching data, and also makes rsync compatible with a + non-bundled zlib. See the `--new-compress` and `--old-compress` options in + the manpage. + + - Added the support/rsync-no-vanished wrapper script. + + - Made configure more prominently mention when we failed to find yodl (in case + the user wants to be able to generate manpages from `*.yo` files). + + - Have manpage mention how a daemon's max-verbosity setting affects info and + debug options. Also added more clarification on backslash removals for + excludes that contain wildcards. + + - Have configure check if for the attr lib (for getxattr) for those systems + that need to link against it explicitly. + + - Change the early dir-creation logic to only use that idiom in an + inc-recursive copy that is preserving directory times. e.g. using + `--omit-dir-times` will avoid these early directories being created. + + - Fix a bug in `cmp_time()` that would return a wrong result if the 2 times + differed by an amount greater than what a `time_t` can hold. + +### DEVELOPER RELATED: + + - We now include an example systemd file (in packaging/systemd). + + - Tweaked configure to make sure that any intended use of the included popt + and/or zlib code is put early in the CFLAGS. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.1.0 (28 Sep 2013) + +Protocol: 31 (changed) + +## Changes since 3.0.9: + +### OUTPUT CHANGES: + + - Output numbers in 3-digit groups by default (e.g. 1,234,567). See the + `--human-readable` option for a way to turn it off. See also the daemon's + `log format` parameter and related command-line options (including + `--out-format`) for a modifier that can be used to request digit-grouping or + human-readable output in log escapes. (Note that log output is unchanged by + default.) + + - The `--list-only` option is now affected by the `--human-readable` setting. + It will display digit groupings by default, and unit suffixes if higher + levels of readability are requested. Also, the column width for the size + output has increased from 11 to 14 characters when human readability is + enabled. Use `--no-h` to get the old-style output and column size. + + - The output of the `--progress` option has changed: the string `xfer` was + shortened to `xfr`, and the string `to-check` was shortened to `to-chk`, + both designed to make room for the (by default) wider display of file size + numbers without making the total line-length longer. Also, when incremental + recursion is enabled, the string `ir-chk` will be used instead of `to-chk` + up until the incremental-recursion scan is done, letting you know that the + value to check and the total value will still be increasing as new files are + found. + + - Enhanced the `--stats` output: 1) to mention how many files were created + (protocol >= 28), 2) to mention how many files were deleted (a new line for + protocol 31, but only output when `--delete` is in effect), and 3) to follow + the file-count, created-count, and deleted-count with a subcount list that + shows the counts by type. The wording of the transferred count has also + changed so that it is clearer that it is only a count of regular files. + +### BUG FIXES: + + - Fixed a bug in the iconv code when EINVAL or EILSEQ is returned with a full + output buffer. + + - Fixed some rare bugs in `--iconv` processing that might cause a multibyte + character to get translated incorrectly. + + - Fixed a bogus `vanished file` error if some files were specified with `./` + prefixes and others were not. + + - Fixed a bug in `--sparse` where an extra gap could get inserted after a + partial write. + + - Changed the way `--progress` overwrites its prior output in order to make it + nearly impossible for the progress to get overwritten by an error. + + - Improved the propagation of abnormal-exit error messages. This should help + the client side to receive errors from the server when it is exiting + abnormally, and should also avoid dying with an `connection unexpectedly + closed` exit when the closed connection is really expected. + + - The sender now checks each file it plans to remove to ensure that it hasn't + changed from the first stat's info. This helps to avoid losing file data + when the user is not using the option in a safe manner. + + - Fixed a data-duplication bug in the compress option that made compression + less efficient. This improves protocol 31 onward, while behaving in a + compatible (buggy) manner with older rsync protocols. + + - When creating a temp-file, rsync is now a bit smarter about it dot-char + choices, which can fix a problem on OS X with names that start with `..`. + + - Rsync now sets a cleanup flag for `--inplace` and `--append` transfers that + will flush the write buffer if the transfer aborts. This ensures that more + received data gets written out to the disk on an aborted transfer (which is + quite helpful on a slow, flaky connection). + + - The reads that `map_ptr()` now does are aligned on 1K boundaries. This helps + some filesystems and/or files that don't like unaligned reads. + + - Fix an issue in the msleep() function if time jumps backwards. + + - Fix daemon-server module-name splitting bug where an arg would get split + even if `--protect-args` was used. + +### ENHANCEMENTS: + + - Added the `--remote-option=OPT` (`-M OPT`) command-line option that is + useful for things like sending a remote `--log-file=FILE` or `--fake-super` + option. + + - Added the `--info=FLAGS` and `--debug=FLAGS` options to allow finer-grained + control over what is output. Added an extra type of `--progress` output + using `--info=progress2`. + + - The `--msgs2stderr` option can help with debugging rsync by allowing the + debug messages to get output to stderr rather than travel via the socket + protocol. + + - Added the `--delete-missing-args` and `--ignore-missing-args` options to + either delete or ignore user-specified files on the receiver that are + missing on the sender (normally the absence of user-specified files + generates an error). + + - Added a `T` (terabyte) category to the `--human-readable` size suffixes. + + - Added the `--usermap`/`--groupmap`/`--chown` options for manipulating file + ownership during the copy. + + - Added the `%C` escape to the log-output handling, which will output the MD5 + checksum of any transferred file, or all files if `--checksum` was specified + (when protocol 30 or above is in effect). + + - Added the `reverse lookup` parameter to the rsync daemon config file to + allow reverse-DNS lookups to be disabled. + + - Added a forward-DNS lookup for the daemon's hosts allow/deny config. Can be + disabled via `forward lookup` parameter (defaults to enabled). + + - Added a way for more than one group to be specified in the daemon's config + file, including a way to specify that you want all of the specified user's + groups without having to name them. Also changed the daemon to complain + about an inability to set explicitly-specified uid/gid values, even when not + run by a super-user. + + - The daemon now tries to send the user the error messages from the pre-xfer + exec script when it fails. + + - Improved the use of alt-dest options into an existing hierarchy of files: If + a match is found in an alt-dir, it takes precedence over an existing file. + (We'll need to wait for a future version before attribute-changes on + otherwise unchanged files are safe when using an existing hierarchy.) + + - Added per-user authorization options and group-authorization support to the + daemon's `auth users` parameter. + + - Added a way to reference environment variables in a daemon's config file + (using %VAR% references). + + - When replacing a non-dir with a symlink/hard-link/device/special-file, the + update should now be done in an atomic manner. + + - Avoid re-sending xattr info for hard-linked files w/the same xattrs + (protocol 31). + + - The backup code was improved to use better logic maintaining the backup + directory hierarchy. Also, when a file is being backed up, rsync tries to + hard-link it into place so that the upcoming replacement of the destination + file will be atomic (for the normal, non-inplace logic). + + - Added the ability to synchronize nano-second modified times. + + - Added a few more default suffixes for the `dont compress` settings. + + - Added the checking of the `RSYNC_PROTECT_ARGS` environment variable to allow + the default for the `--protect-args` command-line option to be overridden. + + - Added the `--preallocate` command-line option. + + - Allow `--password-file=-` to read the password from stdin (filename `-`). + + - Rsync now comes packaged with an rsync-ssl helper script that can be used to + contact a remote rsync daemon using a piped-stunnel command. It also + includes an stunnel config file to run the server side to support ssl daemon + connections. See the packaging/lsb/rsync.spec file for one way to package + the resulting files. (Suggestions for how to make this even easier to + install & use are welcomed.) + + - Improved the speed of some `--inplace` updates when there are lots of + identical checksum blocks that end up being unusable. + + - Added the `--outbuf=N|L|B` option for choosing the output buffering. + + - Repeating the `--fuzzy` option now causes the code to look for fuzzy matches + inside alt-dest directories too. + + - The `--chmod` option now supports numeric modes, e.g. `--chmod=644,D755` + + - Added some Solaris xattr code. + + - Made an rsync daemon (the listening process) exit with a 0 status when it + was signaled to die. This helps launchd. + + - Improved the `RSYNC_*` environment variables for the pre-xfer exec script: + when a daemon is sent multiple request args, they are now joined into a + single return value (separated by spaces) so that the `RSYNC_REQUEST` + environment variable is accurate for any `pre-xfer exec`. The values in + `RSYNC_ARG#` vars are no longer truncated at the `.` arg (prior to the + request dirs/files), so that all the requested values are also listed + (separately) in `RSYNC_ARG#` variables. + +### EXTRAS: + + - Added an `instant-rsyncd` script to the support directory, which makes it + easy to configure a simple rsync daemon in the current directory. + + - Added the `mapfrom` and `mapto` scripts to the support directory, which + makes it easier to do user/group mapping in a local transfer based on + passwd/group files from another machine. + + - There's a new, improved version of the lsh script in the support dir: it's + written in perl and supports `-u` without resorting to using sudo (when run + as root). The old shell version is now named lsh.sh. + + - There is a helper script named rsync-slash-strip in the support directory + for anyone that wants to change the way rsync handles args with trailing + slashes. (e.g. arg/ would get stripped to arg while arg/. would turn into + arg/). + +### INTERNAL: + + - The I/O code was rewritten to be simpler and do bigger buffered reads over + the socket. The I/O between the receiver and the generator was changed to be + standard multiplexed-I/O (like that over the socket). + + - The sender tries to use any dead time while the generator is looking for + files to transfer in order to do sender-side directory scanning in a more + parallel manner. + + - A daemon can now inform a client about a daemon-configured timeout value so + that the client can assist in the keep-alive activity (protocol 31). + + - The filter code received some refactoring to make it more extendible, to + read better, and do better sanity checking. + + - Really big numbers are now output using our own big-num routine rather than + casting them to a double and using a %.0f conversion. + + - The `pool_alloc` library has received some minor improvements in alignment + handling. + + - Added `init_stat_x()` function to avoid duplication of acl/xattr init code. + + - The included zlib was upgraded from 1.2.3 to 1.2.8. + + - Rsync can now be compiled to use an unmodified zlib library instead of the + tweaked one that is included with rsync. This will eventually become the + default, at which point we'll start the countdown to removing the included + zlib. Until then, feel free to configure using: + + ./configure `--with-included-zlib=no` + +### DEVELOPER RELATED: + + - Added more conditional debug output. + + - Fixed some build issues for android and minix. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.9 (23 Sep 2011) + +Protocol: 30 (unchanged) + +## Changes since 3.0.8: + +### BUG FIXES: + + - Fix a crash bug in checksum scanning when `--inplace` is used. + + - Fix a hang if a hard-linked file cannot be opened by the sender (e.g. if it + has no read permission). + + - Fix preservation of a symlink's system xattrs (e.g. selinux) on Linux. + + - Fix a memory leak in the xattr code. + + - Fixed a bug with `--delete-excluded` when a filter merge file has a rule + that specifies a receiver-only side restriction. + + - Fix a bug with the modifying of unwritable directories. + + - Fix `--fake-super`'s interaction with `--link-dest` same-file comparisons. + + - Fix the updating of the `curr_dir` buffer to avoid a duplicate slash. + + - Fix the directory permissions on an implied dot-dir when using `--relative` + (e.g. /outside/path/././send/path). + + - Fixed some too-long sleeping instances when using `--bwlimit`. + + - Fixed when symlink ownership difference-checking gets compiled into + `unchanged_attrs()`. + + - Improved the socket-error reporting when multiple protocols fail. + + - Fixed a case where a socket error could reference just-freed memory. + + - Failing to use a password file that was specified on the command-line is now + a fatal error. + + - Fix the non-root updating of directories that don't have the read and/or + execute permission. + + - Make daemon-excluded file errors more error-like. + + - Fix a compilation issue on older C compilers (due to a misplaced var + declaration). + + - Make configure avoid finding socketpair on cygwin. + + - Avoid trying to reference `SO_BROADCAST` if the OS doesn't support it. + + - Fix some issues with the post-processing of the man pages. + + - Fixed the user home-dir handling in the support/lsh script. + + - Some minor manpage improvements. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.8 (26 Mar 2011) + +Protocol: 30 (unchanged) + +## Changes since 3.0.7: + +### BUG FIXES: + + - Fixed two buffer-overflow issues: one where a directory path that is exactly + MAXPATHLEN was not handled correctly, and one handling a `--backup-dir` that + is extra extra large. + + - Fixed a data-corruption issue when preserving hard-links without preserving + file ownership, and doing deletions either before or during the transfer + (CVE-2011-1097). This fixes some assert errors in the hard-linking code, and + some potential failed checksums (via `-c`) that should have matched. + + - Fixed a potential crash when an rsync daemon has a filter/exclude list and + the transfer is using ACLs or xattrs. + + - Fixed a hang if a really large file is being processed by an rsync that + can't handle 64-bit numbers. Rsync will now complain about the file being + too big and skip it. + + - For devices and special files, we now avoid gathering useless ACL and/or + xattr information for files that aren't being copied. (The un-copied files + are still put into the file list, but there's no need to gather data that is + not going to be used.) This ensures that if the user uses `--no-D`, that + rsync can't possibly complain about being unable to gather extended + information from special files that are in the file list (but not in the + transfer). + + - Properly handle requesting remote filenames that start with a dash. This + avoids a potential error where a filename could be interpreted as a (usually + invalid) option. + + - Fixed a bug in the comparing of upper-case letters in file suffixes for + `--skip-compress`. + + - If an rsync daemon has a module configured without a path setting, rsync + will now disallow access to that module. + + - If the destination arg is an empty string, it will be treated as a reference + to the current directory (as 2.x used to do). + + - If rsync was compiled with a newer time-setting function (such as lutimes), + rsync will fall-back to an older function (such as utimes) on a system where + the newer function is not around. This helps to make the rsync binary more + portable in mixed-OS-release situations. + + - Fixed a batch-file writing bug that would not write out the full set of + compatibility flags that the transfer was using. This fixes a potential + protocol problem for a batch file that contains a sender-side I/O error: it + would have been sent in a way that the batch-reader wasn't expecting. + + - Some improvements to the hard-linking code to ensure that device-number + hashing is working right, and to supply more information if the hard-link + code fails. + + - The `--inplace` code was improved to not search for an impossible checksum + position. The quadruple-verbose chunk[N] message will now mention when an + inplace chunk was handled by a seek rather than a read+write. + + - Improved ACL mask handling, e.g. for Solaris. + + - Fixed a bug that prevented `--numeric-ids` from disabling the translation of + user/group IDs for ACLs. + + - Fixed an issue where an xattr and/or ACL transfer that used an alt-dest + option (e.g. `--link-dest`) could output an error trying to itemize the + changes against the alt-dest directory's xattr/ACL info but was instead + trying to access the not-yet-existing new destination directory. + + - Improved xattr system-error messages to mention the full path to the file. + + - The `--link-dest` checking for identical symlinks now avoids considering + attribute differences that cannot be changed on the receiver. + + - Avoid trying to read/write xattrs on certain file types for certain OSes. + Improved configure to set `NO_SYMLINK_XATTRS`, `NO_DEVICE_XATTRS`, and/or + `NO_SPECIAL_XATTRS` defines in config.h. + + - Improved the unsafe-symlink errors messages. + + - Fixed a bug setting xattrs on new files that aren't user writable. + + - Avoid re-setting xattrs on a hard-linked file w/the same xattrs. + + - Fixed a bug with `--fake-super` when copying files and dirs that aren't user + writable. + + - Fixed a bug where a sparse file could have its last sparse block turned into + a real block when rsync sets the file size (requires ftruncate). + + - If a temp-file name is too long, rsync now avoids truncating the name in the + middle of adjacent high-bit characters. This prevents a potential filename + error if the filesystem doesn't allow a name to contain an invalid + multi-byte sequence. + + - If a muli-protocol socket connection fails (i.e., when contacting a daemon), + we now report all the failures, not just the last one. This avoids losing a + relevant error (e.g. an IPv4 connection-refused error) that happened before + the final error (e.g. an IPv6 protocol-not-supported error). + + - Generate a transfer error if we try to call chown with a `-1` for a uid or a + gid (which is not settable). + + - Fixed the working of `--force` when used with `--one-file-system`. + + - Fix the popt arg parsing so that an option that doesn't take an arg will + reject an attempt to supply one (can configure `--with-included-popt` if + your system's popt library doesn't yet have this fix). + + - A couple minor option tweaks to the support/rrsync script, and also some + regex changes that make vim highlighting happier. + + - Fixed some issues in the support/mnt-excl script. + + - Various manpage improvements. + +### ENHANCEMENTS: + + - Added `.hg/` to the default cvs excludes (see `-C` & `--cvs-exclude`). + +### DEVELOPER RELATED: + + - Use lchmod() whenever it is available (not just on symlinks). + + - A couple fixes to the `socketpair_tcp()` routine. + + - Updated the helper scripts in the packaging subdirectory. + + - Renamed configure.in to configure.ac. + + - Fixed configure's checking for iconv routines for newer OS X versions. + + - Fixed the testsuite/xattrs.test script on OS X. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.7 (31 Dec 2009) + +Protocol: 30 (unchanged) + +## Changes since 3.0.6: + +### BUG FIXES: + + - Fixed a bogus free when using `--xattrs` with `--backup`. + + - Avoid an error when `--dry-run` was trying to stat a prior hard-link file + that hasn't really been created. + + - Fixed a problem with `--compress` (`-z`) where the receiving side could + return the error "inflate (token) returned -5". + + - Fixed a bug where `--delete-during` could delete in a directory before it + noticed that the sending side sent an I/O error for that directory (both + sides of the transfer must be at least 3.0.7). + + - Improved `--skip-compress`'s error handling of bad character-sets and got + rid of a lingering debug fprintf(). + + - Fixed the daemon's conveyance of `io_error` value from the sender. + + - An rsync daemon use seteuid() (when available) if it used setuid(). + + - Get the permissions right on a `--fake-super` transferred directory that + needs more owner permissions to emulate root behavior. + + - An absolute-path filter rule (i.e. with a '/' modifier) no longer loses its + modifier when sending the filter rules to the remote rsync. + + - Improved the "--delete does not work without -r or -d" message. + + - Improved rsync's handling of `--timeout` to avoid a weird timeout case where + the sender could timeout even though it has recently written data to the + socket (but hasn't read data recently, due to the writing). + + - Some misc manpage improvements. + + - Fixed the chmod-temp-dir testsuite on a system without /var/tmp. + + - Make sure that a timeout specified in the daemon's config is used as a + maximum timeout value when the user also specifies a timeout. + + - Improved the error-exit reporting when rsync gets an error trying to cleanup + after an error: the initial error is reported. + + - Improved configure's detection of IPv6 for solaris and cygwin. + + - The AIX sysacls routines will now return ENOSYS if ENOTSUP is missing. + + - Made our (only used if missing) getaddrinfo() routine use `inet_pton()` + (which we also provide) instead of `inet_aton()`. + + - The exit-related debug messages now mention the program's role so it is + clear who output what message. + +### DEVELOPER RELATED: + + - Got rid of type-punned compiler warnings output by newer gcc versions. + + - The Makefile now ensures that proto.h will be rebuilt if config.h changes. + + - The testsuite no longer uses `id -u`, so it works better on solaris. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.6 (8 May 2009) + +Protocol: 30 (unchanged) + +## Changes since 3.0.5: + +### BUG FIXES: + + - Fixed a `--read-batch` hang when rsync is reading a batch file that was + created from an incremental-recursion transfer. + + - Fixed the daemon's socket code to handle the simultaneous arrival of + multiple connections. + + - Fix `--safe-links`/`--copy-unsafe-links` to properly handle symlinks that + have consecutive slashes in the value. + + - Fixed the parsing of an `[IPv6_LITERAL_ADDR]` when a USER@ is prefixed. + + - The sender now skips a (bogus) symlink that has a 0-length value, which + avoids a transfer error in the receiver. + + - Fixed a case where the sender could die with a tag-0 error if there was an + I/O during the sending of the file list. + + - Fixed the rrsync script to avoid a server-side problem when `-e` is at the + start of the short options. + + - Fixed a problem where a vanished directory could turn into an exit code 23 + instead of the proper exit code 24. + + - Fixed the `--iconv` conversion of symlinks when doing a local copy. + + - Fixed a problem where `--one-file-system` was not stopping deletions on the + receiving side when a mount-point directory did not match a directory in the + transfer. + + - Fixed the dropping of an ACL mask when no named ACL values were present. + + - Fixed an ACL/xattr corruption issue where the `--backup` option could cause + rsync to associate the wrong ACL/xattr information with received files. + + - Fixed the use of `--xattrs` with `--only-write-batch`. + + - Fixed the use of `--dry-run` with `--read-batch`. + + - Fixed configure's erroneous use of target. + + - Fixed configure's `--disable-debug` option. + + - Fixed a run-time issue for systems that can't find `iconv_open()` by adding + the `--disable-iconv-open` configure option. + + - Complain and die if the user tries to combine `--remove-source-files` (or + the deprecated `--remove-sent-files`) with `--read-batch`. + + - Fixed an failure transferring special files from Solaris to Linux. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.5 (28 Dec 2008) + +Protocol: 30 (unchanged) + +## Changes since 3.0.4: + +### BUG FIXES: + + - Initialize xattr data in a couple spots in the hlink code, which avoids a + crash when the xattr pointer's memory happens to start out non-zero. Also + fixed the itemizing of an alt-dest file's xattrs when hard-linking. + + - Don't send a bogus `-` option to an older server if there were no short + options specified. + + - Fixed skipping of unneeded updates in a batch file when incremental + recursion is active. Added a test for this. Made batch-mode handle `redo` + files properly (and without hanging). + + - Fix the %P logfile escape when the daemon logs from inside a chroot. + + - Fixed the use of `-s` (`--protect-args`) when used with a remote source or + destination that had an empty path (e.g. `host:`). Also fixed a problem when + `-s` was used when accessing a daemon via a remote-shell. + + - Fixed the use of a dot-dir path (e.g. foo/./bar) inside a `--files-from` + file when the root of the transfer isn't the current directory. + + - Fixed a bug with `-K --delete` removing symlinks to directories when + incremental recursion is active. + + - Fixed a hard to trigger hang when using `--remove-source-files`. + + - Got rid of an annoying delay when accessing a daemon via a remote-shell. + + - Properly ignore (superfluous) source args on a `--read-batch` command. + + - Improved the manpage's description of the `*` wildcard to remove the + confusing `non-empty` qualifier. + + - Fixed reverse lookups in the compatibility-library version of getnameinfo(). + + - Fixed a bug when using `--sparse` on a sparse file that has over 2GB of + consecutive sparse data. + + - Avoid a hang when using at least 3 `--verbose` options on a transfer with a + client sender (which includes local copying). + + - Fixed a problem with `--delete-delay` reporting an error when it was ready + to remove a directory that was now gone. + + - Got rid of a bunch of `warn_unused_result` compiler warnings. + + - If an ftruncate() on a received file fails, it now causes a partial- + transfer warning. + + - Allow a path with a leading `//` to be preserved (CYGWIN only). + +### ENHANCEMENTS: + + - Made the support/atomic-rsync script able to perform a fully atomic update + of the copied hierarchy when the destination is setup using a particular + symlink idiom. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.4 (6 Sep 2008) + +Protocol: 30 (unchanged) + +## Changes since 3.0.3: + +### BUG FIXES: + + - Fixed a bug in the hard-linking code where it would sometimes try to + allocate 0 bytes of memory (which fails on some OSes, such as AIX). + + - Fixed the hard-linking of files from a device that has a device number of 0 + (which seems to be a common device number on NetBSD). + + - Fixed the handling of a `--partial-dir` that cannot be created. This + particularly impacts the `--delay-updates` option (since the files cannot be + delayed without a partial-dir), and was potentially destructive if the + `--remove-source-files` was also specified. + + - Fixed a couple issues in the `--fake-super` handling of xattrs when the + destination files have root-level attributes (e.g. selinux values) that a + non-root copy can't affect. + + - Improved the keep-alive check in the generator to fire consistently in + incremental-recursion mode when `--timeout` is enabled. + + - The `--iconv` option now converts the content of a symlink too, instead of + leaving it in the wrong character-set (requires 3.0.4 on both sides of the + transfer). + + - When using `--iconv`, if a filename fails to convert on the receiving side, + this no longer makes deletions in the root-dir of the transfer fail silently + (the user now gets a warning about deletions being disabled due to IO error + as long as `--ignore-errors` was not specified). + + - When using `--iconv`, if a server-side receiver can't convert a filename, + the error message sent back to the client no longer mangles the name with + the wrong charset conversion. + + - Fixed a potential alignment issue in the IRIX ACL code when allocating the + initial `struct acl` object. Also, cast mallocs to avoid warnings. + + - Changed some errors that were going to stdout to go to stderr. + + - Made `human_num()` and `human_dnum()` able to output a negative number + (rather than outputting a cryptic string of punctuation). + +### ENHANCEMENTS: + + - Rsync will avoid sending an `-e` option to the server if an older protocol + is requested (and thus the option would not be useful). This lets the user + specify the `--protocol=29` option to access an overly-restrictive server + that is rejecting the protocol-30 use of `-e` to the server. + + - Improved the message output for an `RERR_PARTIAL` exit. + +### DEVELOPER RELATED: + + - The Makefile will not halt for just a timestamp change on the Makefile or + the configure files, only for actual changes in content. + + - Changed some commands in the testsuite's xattrs.test that called `rsync` + instead of `$RSYNC`. + + - Enhanced the release scripts to be able to handle a branch release and to do + even more consistency checks on the files. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.3 (29 Jun 2008) + +Protocol: 30 (unchanged) + +## Changes since 3.0.2: + +### BUG FIXES: + + - Fixed a wildcard matching problem in the daemon when a module has `use + chroot` enabled. + + - Fixed a crash bug in the hard-link code. + + - Fixed the sending of xattr directory information when the code finds a + `--link-dest` or `--copy-dest` directory with unchanged xattrs -- the + destination directory now gets these unchanged xattrs properly applied. + + - Fixed an xattr-sending glitch that could cause an `Internal abbrev` error. + + - Fixed the combination of `--xattrs` and `--backup`. + + - The generator no longer allows a '.' dir to be excluded by a daemon- exclude + rule. + + - Fixed deletion handling when copying a single, empty directory (with no + files) to a differently named, non-existent directory. + + - Fixed the conversion of spaces into dashes in the %M log escape. + + - Fixed several places in the code that were not returning the right errno + when a function failed. + + - Fixed the backing up of a device or special file into a backup dir. + + - Moved the setting of the socket options prior to the connect(). + + - If rsync exits in the middle of a `--progress` output, it now outputs a + newline to help prevent the progress line from being overwritten. + + - Fixed a problem with how a destination path with a trailing slash or a + trailing dot-dir was compared against the daemon excludes. + + - Fixed the sending of large (size > 16GB) files when talking to an older + rsync (protocols < 30): we now use a compatible block size limit. + + - If a file's length is so huge that we overflow a checksum buffer count (i.e. + several hundred TB), warn the user and avoid sending an invalid checksum + struct over the wire. + + - If a source arg is excluded, `--relative` no longer adds the excluded arg's + implied dirs to the transfer. This fix also made the exclude check happen in + the better place in the sending code. + + - Use the `overflow_exit()` function for overflows, not `out_of_memory()`. + + - Improved the code to better handle a system that has only 32-bit file + offsets. + +### ENHANCEMENTS: + + - The rsyncd.conf manpage now consistently refers to the parameters in the + daemon config file as `parameters`. + + - The description of the `--inplace` option was improved. + +### EXTRAS: + + - Added a new script in the support directory, deny-rsync, which allows an + admin to (temporarily) replace the rsync command with a script that sends an + error message to the remote client via the rsync protocol. + +### DEVELOPER RELATED: + + - Fixed a testcase failure if the tests are run as root and made some + compatibility improvements. + + - Improved the daemon tests, including checking module comments, the listing + of files, and the ensuring that daemon excludes can't affect a dot-dir arg. + + - Improved some build rules for those that build in a separate directory from + the source, including better install rules for the man pages, and the fixing + of a proto.h-tstamp rule that could make the binaries get rebuild without + cause. + + - Improved the testsuite to work around a problem with some utilities (e.g. cp + -p & touch -r) rounding sub-second timestamps. + + - Ensure that the early patches don't cause any generated-file hunks to + bleed-over into patches that follow. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.2 (8 Apr 2008) + +Protocol: 30 (unchanged) + +## Changes since 3.0.1: + +### BUG FIXES: + + - Fixed a potential buffer overflow in the xattr code. + +### ENHANCEMENTS: + + - None. + +### DEVELOPER RELATED: + + - The RPM spec file was improved to install more useful files. + + - A few developer-oriented scripts were moved from the support dir to the + packaging dir. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.1 (3 Apr 2008) + +Protocol: 30 (unchanged) + +## Changes since 3.0.0: + +### NOTABLE CHANGES IN BEHAVIOR: + + - Added the 'c'-flag to the itemizing of non-regular files so that the + itemized output doesn't get hidden if there were no attribute changes, and + also so that the itemizing of a `--copy-links` run will distinguish between + copying an identical non-regular file and the creation of a revised version + with a new value (e.g. a changed symlink referent, a new device number, + etc.). + +### BUG FIXES: + + - Fixed a crash bug when a single-use rsync daemon (via remote shell) was run + without specifying a `--config=FILE` option. + + - Fixed a crash when backing up a directory that has a default ACL. + + - Fixed a bug in the handling of xattr values that could cause rsync to not + think that a file's extended attributes are up-to-date. + + - Fixed the working of `--fake-super` with `--link-dest` and `--xattrs`. + + - Fixed a hang when combining `--dry-run` with `--remove-source-files`. + + - Fixed a bug with `--iconv`'s handling of files that cannot be converted: a + failed name can no longer cause a transfer failure. + + - Fixed the building of the rounding.h file on systems that need custom + CPPFLAGS to be used. Also improved the error reporting if the building of + rounding.h fails. + + - Fixed the use of the `--protect-args` (`-s`) option when talking to a + daemon. + + - Fixed the `--ignore-existing` option's protection of files on the receiver + that are non-regular files on the sender (e.g. if a symlink or a dir on the + sender is trying to replace a file on the receiver). The reverse protection + (protecting a dir/symlink/device from being replaced by a file) was already + working. + + - Fixed an assert failure if `--hard-links` is combined with an option that + can skip a file in a set of hard-linked files (i.e. `--ignore-existing`, + `--append`, etc.), without skipping all the files in the set. + + - Avoid setting the modify time on a directory that already has the right + modify time set. This avoids tweaking the dir's ctime. + + - Improved the daemon-exclude handling to do a better job of applying the + exclude rules to path entries. It also sends the user an error just as if + the files were actually missing (instead of silently ignoring the user's + args), and avoids sending the user the filter-action messages for these + non-user-initiated rules. + + - Fixed some glitches with the dry-run code's missing-directory handling, + including a problem when combined with `--fuzzy`. + + - Fixed some glitches with the skipped-directory handling. + + - Fixed the 'T'-flag itemizing of symlinks when `--time` isn't preserved. + + - Fixed a glitch in the itemizing of permissions with the `-E` option. + + - The `--append` option's restricting of transfers to those that add data no + longer prevents the updating of non-content changes to otherwise up-to- date + files (i.e. those with the same content but differing permissions, + ownership, xattrs, etc.). + + - Don't allow `--fake-super` to be specified with `-XX` (double `--xattrs`) + because the options conflict. If a daemon has `fake super` enabled, it + automatically downgrades a `-XX` request to `-X`. + + - Fixed a couple bugs in the parsing of daemon-config excludes that could make + a floating exclude rule get treated as matching an absolute path. + + - A daemon doesn't try to auto-refuse the `iconv` option if iconv-support + wasn't compiled in to the daemon (avoiding a warning in the logs). + + - Fixed the inclusion of per-dir merge files from implied dirs. + + - Fixed the support/rrsync script to work with the latest options that rsync + sends (including its flag-specifying use of `-e` to the server). + +### ENHANCEMENTS: + + - Added the `--old-dirs` (`--old-d`) option to make it easier for a user to + ask for file-listings with older rsync versions (this is easier than having + to type `-r --exclude='/*/*'` manually). + + - When getting an error while asking an older rsync daemon for a file listing, + rsync will try to notice if the error is a rejection of the `--dirs` (`-d`) + option and let the user know how to work around the issue. + + - Added a few more `--no-OPTION` overrides. + + - Improved the documentation of the `--append` option. + + - Improved the documentation of the filter/exclude/include daemon parameters. + +### INTERNAL: + + - Fixed a couple minor bugs in the included popt library (ones which I sent to + the official popt project for inclusion in the 1.14 release). + + - Fixed a stat() call that should have been `do_stat()` so that the proper + normal/64-bit stat() function gets called. (Was in an area that should not + have caused problems, though.) + + - Changed the file-glob code to do a directory scan without using the `glob` + and `glob.h`. This lets us do the globbing with less memory churn, and also + avoid adding daemon-excluded items to the returned args. + +### DEVELOPER RELATED: + + - The configure script tries to get the user's compiler to not warn about + unused function parameters if the build is not including one or more of the + ACL/xattrs/iconv features. + + - The configure script now has better checks for figuring out if the included + popt code should be used or not. + + - Fixed two testsuite glitches: avoid a failure if someone's `cd` command + outputs the current directory when cd-ing to a relative path, and made the + itemized test query how rsync was built to determine if it should expect + hard-linked symlinks or not. + + - Updated the testsuite to verify that various bug fixes remain fixed. + + - The RPM spec file was updated to have: (1) comments for how to use the + rsync-patch tar file, and (2) an /etc/xinetd.d/rsync file. + + - Updated the build scripts to work with a revised FTP directory structure. + +------------------------------------------------------------------------------ + +# NEWS for rsync 3.0.0 (1 Mar 2008) + +Protocol: 30 (changed) + +## Changes since 2.6.9: + +### NOTABLE CHANGES IN BEHAVIOR: + + - The handling of implied directories when using `--relative` has changed to + send them as directories (e.g. no implied dir is ever sent as a symlink). + This avoids unexpected behavior and should not adversely affect most people. + If you're one of those rare individuals who relied upon having an implied + dir be duplicated as a symlink, you should specify the transfer of the + symlink and the transfer of the referent directory as separate args. (See + also `--keep-dirlinks` and `--no-implied-dirs`.) Also, exclude rules no + longer have a partial effect on implied dirs. + + - Requesting a remote file-listing without specifying `-r` (`--recursive`) now + sends the `-d` (`--dirs`) option to the remote rsync rather than sending + `-r` along with an extra exclude of `/*/*`. If the remote rsync does not + understand the `-d` option (i.e. it is 2.6.3 or older), you will need to + either turn off `-d` (`--no-d`), or specify `-r --exclude='/*/*'` manually. + + - In `--dry-run` mode, the last line of the verbose summary text is output + with a "(DRY RUN)" suffix to help remind you that no updates were made. + Similarly, `--only-write-batch` outputs `(BATCH ONLY)`. + + - A writable rsync daemon with `use chroot` disabled now defaults to a + symlink-munging behavior designed to make symlinks safer while also allowing + absolute symlinks to be stored and retrieved. This also has the effect of + making symlinks unusable while they're in the daemon's hierarchy. See the + daemon's `munge symlinks` parameter for details. + + - Starting up an extra copy of an rsync daemon will not clobber the pidfile + for the running daemon -- if the pidfile exists, the new daemon will exit + with an error. This means that your wrapper script that starts the rsync + daemon should be made to handle lock-breaking (if you want any automatic + breaking of locks to be done). + +### BUG FIXES: + + - A daemon with `use chroot = no` and excluded items listed in the daemon + config file now properly checks an absolute-path arg specified for these + options: `--compare-dest`, `--link-dest`, `--copy-dest`, `--partial-dir`, + `--backup-dir`, `--temp-dir`, and `--files-from`. + + - A daemon can now be told to disable all user- and group-name translation on + a per-module basis. This avoids a potential problem with a writable daemon + module that has `use chroot` enabled -- if precautions weren't taken, a user + could try to add a missing library and get rsync to use it. This makes rsync + safer by default, and more configurable when id- translation is not desired. + See the daemon's `numeric ids` parameter for full details. + + - A chroot daemon can now indicate which part of its path should affect the + chroot call, and which part should become an inside-chroot path for the + module. This allows you to have outside-the-transfer paths (such as for + libraries) even when you enable chroot protection. The idiom used in the + rsyncd.conf file is: `path = /chroot/dirs/./dirs/inside` + + - If a file's data arrived successfully on the receiving side but the rename + of the temporary file to the destination file failed AND the + `--remove-source-files` (or the deprecated `--remove-sent-files`) option was + specified, rsync no longer erroneously removes the associated source file. + + - Fixed the output of `-ii` when combined with one of the `--*-dest` options: + it now itemizes all the items, not just the changed ones. + + - Made the output of all file types consistent when using a `--*-dest` option. + Prior versions would output too many creation events for matching items. + + - The code that waits for a child pid now handles being interrupted by a + signal. This fixes a problem with the pre-xfer exec function not being able + to get the exit status from the script. + + - A negated filter rule (i.e. with a '!' modifier) no longer loses the + negation when sending the filter rules to the remote rsync. + + - Fixed a problem with the `--out-format` (aka `--log-format`) option %f: it + no longer outputs superfluous directory info for a non-daemon rsync. + + - Fixed a problem with `-vv` (double `--verbose`) and `--stats` when `pushing` + files (which includes local copies). Version 2.6.9 would complete the copy, + but exit with an error when the receiver output its memory stats. + + - If `--password-file` is used on a non-daemon transfer, rsync now complains + and exits. This should help users figure out that they can't use this option + to control a remote shell's password prompt. + + - Make sure that directory permissions of a newly-created destination + directory are handled right when `--perms` is left off. + + - The itemized output of a newly-created destination directory is now output + as a creation event, not a change event. + + - Improved `--hard-link` so that more corner cases are handled correctly when + combined with options such as `--link-dest` and/or `--ignore-existing`. + + - The `--append` option no longer updates a file that has the same size. + + - Fixed a bug when combining `--backup` and `--backup-dir` with `--inplace`: + any missing backup directories are now created. + + - Fixed a bug when using `--backup` and `--inplace` with `--whole-file` or + `--read-batch`: backup files are actually created now. + + - The daemon pidfile is checked and created sooner in the startup sequence. + + - If a daemon module's `path` value is not an absolute pathname, the code now + makes it absolute internally (making it work properly). + + - Ensure that a temporary file always has owner-write permission while we are + writing to it. This avoids problems with some network filesystems when + transferring read-only files. + + - Any errors output about password-file reading no longer cause an error at + the end of the run about a partial transfer. + + - The `--read-batch` option for protocol 30 now ensures that several more + options are set correctly for the current batch file: `--iconv`, `--acls`, + `--xattrs`, `--inplace`, `--append`, and `--append-verify`. + + - Using `--only-write-batch` to a daemon receiver now works properly (older + versions would update some files while writing the batch). + + - Avoid outputting a "file has vanished" message when the file is a broken + symlink and `--copy-unsafe-links` or `--copy-dirlinks` is used (the code + already handled this for `--copy-links`). + + - Fixed the combination of `--only-write-batch` and `--dry-run`. + + - Fixed rsync's ability to remove files that are not writable by the file's + owner when rsync is running as the same user. + + - When transferring large files, the sender's hashtable of checksums is kept + at a more reasonable state of fullness (no more than 80% full) so that the + scanning of the hashtable will not bog down as the number of blocks + increases. + +### ENHANCEMENTS: + + - A new incremental-recursion algorithm is now used when rsync is talking to + another 3.x version. This starts the transfer going more quickly (before all + the files have been found), and requires much less memory. See the + `--recursive` option in the manpage for some restrictions. + + - Lowered memory use in the non-incremental-recursion algorithm for typical + option values (usually saving from 21-29 bytes per file). + + - The default `--delete` algorithm is now `--delete-during` when talking to a + 3.x rsync. This is a faster scan than using `--delete-before` (which is the + default when talking to older rsync versions), and is compatible with the + new incremental recursion mode. + + - Rsync now allows multiple remote-source args to be specified rather than + having to rely on a special space-splitting side-effect of the remote- + shell. Additional remote args must specify the same host or an empty one + (e.g. empty: `:file1` or `::module/file2`). For example, this means that + local use of brace expansion now works: `rsync -av host:dir/{f1,f2} .` + + - Added the `--protect-args` (`-s`) option, that tells rsync to send most of + the command-line args at the start of the transfer rather than as args to + the remote-shell command. This protects them from space-splitting, and only + interprets basic wildcard special shell characters (`*?[`). + + - Added the `--delete-delay` option, which is a more efficient way to delete + files at the end of the transfer without needing a separate delete pass. + + - Added the `--acls` (`-A`) option to preserve Access Control Lists. This is + an improved version of the prior patch that was available, and it even + supports OS X ACLs. If you need to have backward compatibility with old, + ACL-patched versions of rsync, apply the acls.diff file from the patches + dir. + + - Added the `--xattrs` (`-X`) option to preserve extended attributes. This is + an improved version of the prior patch that was available, and it even + supports OS X xattrs (which includes their resource fork data). If you need + to have backward compatibility with old, xattr-patched versions of rsync, + apply the xattrs.diff file from the patches dir. + + - Added the `--fake-super` option that allows a non-super user to preserve all + attributes of a file by using a special extended-attribute idiom. It even + supports the storing of foreign ACL data on your backup server. There is + also an analogous `fake super` parameter for an rsync daemon. + + - Added the `--iconv` option, which allows rsync to convert filenames from one + character-set to another during the transfer. The default is to make this + feature available as long as your system has `iconv_open()`. If compilation + fails, specify `--disable-iconv` to configure, and then rebuild. If you want + rsync to perform character-set conversions by default, you can specify + `--enable-iconv=CONVERT_STRING` with the default value for the `--iconv` + option that you wish to use. For example, `--enable-iconv=.` is a good + choice. See the rsync manpage for an explanation of the `--iconv` option's + settings. + + - A new daemon config parameter, `charset`, lets you control the character- + set that is used during an `--iconv` transfer to/from a daemon module. You + can also set your daemon to refuse `no-iconv` if you want to force the + client to use an `--iconv` transfer (requiring an rsync 3.x client). + + - Added the `--skip-compress=LIST` option to override the default list of file + suffixes that will not be compressed when using `--compress` (`-z`). + + - The daemon's default for `dont compress` was extended to include: `*.7z` + `*.mp[34]` `*.mov` `*.avi` `*.ogg` `*.jpg` `*.jpeg` and the name-matching routine was also + optimized to run more quickly. + + - The `--max-delete` option now outputs a warning if it skipped any file + deletions, including a count of how many deletions were skipped. (Older + versions just silently stopped deleting things.) + + - You may specify `--max-delete=0` to a 3.0.0 client to request that it warn + about extraneous files without deleting anything. If you're not sure what + version the client is, you can use the less-obvious `--max-delete=-1`, as + both old and new versions will treat that as the same request (though older + versions don't warn). + + - The `--hard-link` option now uses less memory on both the sending and + receiving side for all protocol versions. For protocol 30, the use of a + hashtable on the sending side allows us to more efficiently convey to the + receiver what files are linked together. This reduces the amount of data + sent over the socket by a considerable margin (rather than adding more + data), and limits the in-memory storage of the device+inode information to + just the sending side for the new protocol 30, or to the receiving side when + speaking an older protocol (note that older rsync versions kept the + device+inode information on both sides). + + - The filter rules now support a perishable (`p`) modifier that marks rules + that should not have an effect in a directory that is being deleted. e.g. + `-f '-p .svn/'` would only affect `live` .svn directories. + + - Rsync checks all the alternate-destination args for validity (e.g. + `--link-dest`). This lets the user know when they specified a directory that + does not exist. + + - If we get an ENOSYS error setting the time on a symlink, we don't complain + about it anymore (for those systems that even support the setting of the + modify-time on a symlink). + + - Protocol 30 now uses MD5 checksums instead of MD4. + + - Changed the `--append` option to not checksum the existing data in the + destination file, which speeds up file appending. + + - Added the `--append-verify` option, which works like the older `--append` + option (verifying the existing data in the destination file). For + compatibility with older rsync versions, any use of `--append` that is + talking protocol 29 or older will revert to the `--append-verify` method. + + - Added the `--contimeout=SECONDS` option that lets the user specify a + connection timeout for rsync daemon access. + + - Documented and extended the support for the `RSYNC_CONNECT_PROG` variable + that can be used to enhance the client side of a daemon connection. + + - Improved the dashes and double-quotes in the nroff manpage output. + + - Rsync now supports a lot more `--no-OPTION` override options. + +### INTERNAL: + + - The file-list sorting algorithm now uses a sort that keeps any same- named + items in the same order as they were specified. This allows rsync to always + ensure that the first of the duplicates is the one that will be included in + the copy. The new sort is also faster than the glibc version of qsort() and + mergesort(). + + - Rsync now supports the transfer of 64-bit timestamps (`time_t` values). + + - Made the file-deletion code use a little less stack when recursing through a + directory hierarchy of extraneous files. + + - Fixed a build problem with older (2.x) versions of gcc. + + - Added some isType() functions that make dealing with signed characters + easier without forcing variables via casts. + + - Changed strcat/strcpy/sprintf function calls to use safer versions. + + - Upgraded the included popt version to 1.10.2 and improved its use of + string-handling functions. + + - Added missing prototypes for compatibility functions from the lib dir. + + - Configure determines if iconv() has a const arg, allowing us to avoid a + compiler warning. + + - Made the sending of some numbers more efficient for protocol 30. + + - Make sure that a daemon process doesn't mind if the client was weird and + omitted the `--server` option. + + - There are more internal logging categories available in protocol 30 than the + age-old FINFO and FERROR, including `FERROR_XFER` and FWARN. These new + categories allow some errors and warnings to go to stderr without causing an + erroneous end-of-run warning about some files not being able to be + transferred. + + - Improved the use of `const` on pointers. + + - Improved J.W.'s `pool_alloc` routines to add a way of incrementally freeing + older sections of a pool's memory. + + - The getaddrinfo.c compatibility code in the `lib` dir was replaced with some + new code (derived from samba, derived from PostgreSQL) that has a better + license than the old code. + +### DEVELOPER RELATED: + + - Rsync is now licensed under the GPLv3 or later. + + - Rsync is now being maintained in a `git` repository instead of CVS (though + the old CVS repository still exists for historical access). Several + maintenance scripts were updated to work with git. + + - Generated files are no longer committed into the source repository. The + autoconf and autoheader commands are now automatically run during the normal + use of `configure` and `make`. The latest dev versions of all generated + files can also be copied from the samba.org web site (see the prepare-source + script's fetch option). + + - The `patches` directory of diff files is now built from branches in the + rsync git repository (branch patch/FOO creates file patches/FOO.diff). This + directory is now distributed in a separate separate tar file named + rsync-patches-VERSION.tar.gz instead of the main rsync-VERSION.tar.gz. + + - The proto.h file is now built using a simple perl script rather than a + complex awk script, which proved to be more widely compatible. + + - When running the tests, we now put our per-test temp dirs into a sub- + directory named testtmp (which is created, if missing). This allows someone + to symlink the testtmp directory to another filesystem (which is useful if + the build dir's filesystem does not support ACLs and xattrs, but another + filesystem does). + + - Rsync now has a way of handling protocol-version changes during the + development of a new protocol version. This causes any out-of-sync versions + to speak an older protocol rather than fail in a cryptic manner. This + addition makes it safer to deploy a pre-release version that may interact + with the public. This new exchange of sub-version info does not interfere + with the `{MIN,MAX}_PROTOCOL_VERSION` checking algorithm (which does not + have enough range to allow the main protocol number to be incremented for + every minor tweak in that happens during development). + + - The csprotocol.txt file was updated to mention the daemon protocol change in + the 3.0.0 release. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.9 (6 Nov 2006) + +Protocol: 29 (unchanged) + +## Changes since 2.6.8: + +### BUG FIXES: + + - If rsync is interrupted via a handled signal (such as SIGINT), it will once + again clean-up its temp file from the destination dir. + + - Fixed an overzealous sanitizing bug in the handling of the `--link-dest`, + `--copy-dest`, and `--compare-dest` options to a daemon without chroot: if + the copy's destination dir is deeper than the top of the module's path, + these options now accept a safe number of parent-dir (../) references (since + these options are relative to the destination dir). The old code incorrectly + chopped off all `../` prefixes for these options, no matter how deep the + destination directory was in the module's hierarchy. + + - Fixed a bug where a deferred info/error/log message could get sent directly + to the sender instead of being handled by rwrite() in the generator. This + fixes an `unexpected tag 3` fatal error, and should also fix a potential + problem where a deferred info/error message from the receiver might bypass + the log file and get sent only to the client process. (These problems could + only affect an rsync daemon that was receiving files.) + + - Fixed a bug when `--inplace` was combined with a `--*-dest` option and we + update a file's data using an alternate basis file. The code now notices + that it needs to copy the matching data from the basis file instead of + (wrongly) assuming that it was already present in the file. + + - Fixed a bug where using `--dry-run` with a `--*-dest` option with a path + relative to a directory that does not yet exist: the affected option gets + its proper path value so that the output of the dry-run is right. + + - Fixed a bug in the %f logfile escape when receiving files: the destination + path is now included in the output (e.g. you can now tell when a user + specifies a subdir inside a module). + + - If the receiving side fails to create a directory, it will now skip trying + to update everything that is inside that directory. + + - If `--link-dest` is specified with `--checksum` but without `--times`, rsync + will now allow a hard-link to be created to a matching link-dest file even + when the file's modify-time doesn't match the server's file. + + - The daemon now calls more timezone-using functions prior to doing a chroot. + This should help some C libraries to generate proper timestamps from inside + a chrooted daemon (and to not try to access /etc/timezone over and over + again). + + - Fixed a bug in the handling of an absolute `--partial-dir=ABS_PATH` option: + it now deletes an alternate basis file from the partial-dir that was used to + successfully update a destination file. + + - Fixed a bug in the handling of `--delete-excluded` when using a per-dir + merge file: the merge file is now honored on the receiving side, and only + its unqualified include/exclude commands are ignored (just as is done for + global include/excludes). + + - Fixed a recent bug where `--delete` was not working when transferring from + the root (/) of the filesystem with `--relative` enabled. + + - Fixed a recent bug where an `--exclude='*'` could affect the root (/) of the + filesystem with `--relative` enabled. + + - When `--inplace` creates a file, it is now created with owner read/write + permissions (0600) instead of no permissions at all. This avoids a problem + continuing a transfer that was interrupted (since `--inplace` will not + update a file that has no write permissions). + + - If either `--remove-source-files` or `--remove-sent-files` is enabled and we + are unable to remove the source file, rsync now outputs an error. + + - Fixed a bug in the daemon's `incoming chmod` rule: newly-created directories + no longer get the 'F' (file) rules applied to them. + + - Fixed an infinite loop bug when a filter rule was rejected due to being + overly long. + + - When the server receives a `--partial-dir` option from the client, it no + longer runs the client-side code that adds an assumed filter rule (since the + client will be sending us the rules in the usual manner, and they may have + chosen to override the auto-added rule). + +### ENHANCEMENTS: + + - Added the `--log-file=FILE` and `--log-file-format=FORMAT` options. These + can be used to tell any rsync to output what it is doing to a log file. + They work with a client rsync, a non-daemon server rsync (see the man page + for instructions), and also allows the overriding of rsyncd.conf settings + when starting a daemon. + + - The `--log-format` option was renamed to be `--out-format` to avoid + confusing it with affecting the log-file output. (The old option remains as + an alias for the new to preserve backward compatibility.) + + - Made `log file` and `syslog facility` settable on a per-module basis in the + daemon's config file. + + - Added the `--remove-source-files` option as a replacement for the (now + deprecated) `--remove-sent-files` option. This new option removes all + non-dirs from the source directories, even if the file was already + up-to-date. This fixes a problem where interrupting an rsync that was using + `--remove-sent-files` and restarting it could leave behind a file that the + earlier rsync synchronized, but didn't get to remove. (The deprecated + `--remove-sent-files` is still understood for now, and still behaves in the + same way as before.) + + - Added the option `--no-motd` to suppress the message-of-the-day output from + a daemon when doing a copy. (See the manpage for a caveat.) + + - Added a new environment variable to the pre-/post-xfer exec commands (in the + daemon's config file): `RSYNC_PID`. This value will be the same in both the + pre- and post-xfer commands, so it can be used as a unique ID if the + pre-xfer command wants to cache some arg/request info for the post-xfer + command. + +### INTERNAL: + + - Did a code audit using IBM's code-checker program and made several changes, + including: replacing most of the strcpy() and sprintf() calls with + strlcpy(), snprintf(), and memcpy(), adding a 0-value to an enum that had + been intermingling a literal 0 with the defined enum values, silencing some + uninitialized memory checks, marking some functions with a `noreturn` + attribute, and changing an `if` that could never succeed on some platforms + into a pre-processor directive that conditionally compiles the code. + + - Fixed a potential bug in `f_name_cmp()` when both the args are a top-level + `.` dir (which doesn't happen in normal operations). + + - Changed `exit_cleanup()` so that it can never return instead of exit. The + old code might return if it found the `exit_cleanup()` function was being + called recursively. The new code is segmented so that any recursive calls + move on to the next step of the exit-processing. + + - The macro WIFEXITED(stat) will now be defined if the OS didn't already + define it. + +### DEVELOPER RELATED: + + - The acls.diff and xattrs.diff patches have received a bunch of work to make + them much closer to being acceptable in the main distribution. The xattrs + patch also has some preliminary Mac OS X and FreeBSD compatibility code that + various system types to exchange extended file-attributes. + + - A new diff in the patches dir, fake-root.diff, allows rsync to maintain a + backup hierarchy with full owner, group, and device info without actually + running as root. It does this using a special extended attribute, so it + depends on xattrs.diff (which depends on acls.diff). + + - The rsync.yo and rsyncd.conf.yo files have been updated to work better with + the latest yodl 2.x releases. + + - Updated config.guess and config.sub to their 2006-07-02 versions. + + - Updated various files to include the latest FSF address and to have + consistent opening comments. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.8 (22 Apr 2006) + +Protocol: 29 (unchanged) + +## Changes since 2.6.7: + +### BUG FIXES: + + - Fixed a bug in the exclude code where an anchored exclude without any + wildcards fails to match an absolute source arg, but only when `--relative` + is in effect. + + - Improved the I/O code for the generator to fix a potential hang when the + receiver gets an EOF on the socket but the generator's select() call never + indicates that the socket is writable for it to be notified about the EOF. + (This can happen when using stunnel). + + - Fixed a problem with the file-reading code where a failed read (such as that + caused by a bad sector) would not advance the file's read-position beyond + the failed read's data. + + - Fixed a logging bug where the `log file` directive was not being honored in + a single-use daemon (one spawned by a remote-shell connection or by init). + + - If rsync cannot honor the `--delete` option, we output an error and exit + instead of silently ignoring the option. + + - Fixed a bug in the `--link-dest` code that prevented special files (such as + fifos) from being linked. + + - The ability to hard-link symlinks and special files is now determined at + configure time instead of at runtime. This fixes a bug with `--link-dest` + creating a hard-link to a symlink's referent on a BSD system. + +### ENHANCEMENTS: + + - In daemon mode, if rsync fails to bind to the requested port, the error(s) + returned by socket() and/or bind() are now logged. + + - When we output a fatal error, we now output the version of rsync in the + message. + + - Improved the documentation for the `--owner` and `--group` options. + + - The rsyncstats script in `support` has an improved line-parsing regex that + is easier to read and also makes it to parse syslog-generated lines. + + - A new script in `support`: file-attr-restore, can be used to restore the + attributes of a file-set (the permissions, ownership, and group info) taken + from the cached output of a `find ARG... -ls` command. + +### DEVELOPER RELATED: + + - Removed the unused function `write_int_named()`, the unused variable + `io_read_phase`, and the rarely used variable `io_write_phase`. This also + elides the confusing 'phase "unknown"' part of one error message. + + - Removed two unused configure checks and two related (also unused) + compatibility functions. + + - The xattrs.diff patch received a security fix that prevents a potential + buffer overflow in the `receive_xattr()` code. + + - The acls.diff patch has been improved quite a bit, with more to come. + + - A new patch was added: log-file.diff. This contains an early version of a + future option, `--log-file=FILE`, that will allow any rsync to log its + actions to a file (something that only a daemon supports at present). + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.7 (11 Mar 2006) + +Protocol: 29 (unchanged) + +## Changes since 2.6.6: + +### OUTPUT CHANGES: + + - The letter 'D' in the itemized output was being used for both devices + (character or block) as well as other special files (such as fifos and named + sockets). This has changed to separate non-device special files under the + 'S' designation (e.g. `cS+++++++ path/fifo`). See also the `--specials` + option, below. + + - The way rsync escapes unreadable characters has changed. First, rsync now + has support for recognizing valid multibyte character sequences in your + current locale, allowing it to escape fewer characters than before for a + locale such as UTF-8. Second, it now uses an escape idiom of `\#123`, which + is the literal string `\#` followed by exactly 3 octal digits. Rsync no + longer doubles a backslash character in a filename (e.g. it used to output + `foo\\bar` when copying `foo\bar`) -- now it only escapes a backslash that + is followed by a hash-sign and 3 digits (0-9) (e.g. it will output + `foo\#134#789` when copying `foo\#789`). See also the `--8-bit-output` + (`-8`) option, mentioned below. + + Script writers: the local rsync is the one that outputs escaped names, so if + you need to support unescaping of filenames for older rsyncs, I'd suggest + that you parse the output of `rsync --version` and only use the old + unescaping rules for 2.6.5 and 2.6.6. + +### BUG FIXES: + + - Fixed a really old bug that caused `--checksum` (`-c`) to checksum all the + files encountered during the delete scan (ouch). + + - Fixed a potential hang in a remote generator: when the receiver gets a + read-error on the socket, it now signals the generator about this so that + the generator does not try to send any of the terminating error messages to + the client (avoiding a potential hang in some setups). + + - Made hard-links work with symlinks and devices again. + + - If the sender gets an early EOF reading a source file, we propagate this + error to the receiver so that it can discard the file and try requesting it + again (which is the existing behavior for other kinds of read errors). + + - If a device-file/special-file changes permissions, rsync now updates the + permissions without recreating the file. + + - If the user specifies a remote-host for both the source and destination, we + now output a syntax error rather than trying to open the destination + hostspec as a filename. + + - When `--inplace` creates a new destination file, rsync now creates it with + permissions 0600 instead of 0000 -- this makes restarting possible when the + transfer gets interrupted in the middle of sending a new file. + + - Reject the combination of `--inplace` and `--sparse` since the sparse-output + algorithm doesn't work when overwriting existing data. + + - Fixed the directory name in the error that is output when `pop_dir()` fails. + + - Really fixed the parsing of a `!` entry in .cvsignore files this time. + + - If the generator gets a stat() error on a file, output it (this used to + require at least `-vv` for the error to be seen). + + - If waitpid() fails or the child rsync didn't exit cleanly, we now handle the + exit status properly and generate a better error. + + - Fixed some glitches in the double-verbose output when using `--copy-dest`, + `--link-dest`, or `--compare-dest`. Also improved how the verbose output + handles hard-links (within the transfer) that had an up-to-date alternate + `dest` file, and copied files (via `--copy-dest`). + + - Fixed the matching of the dont-compress items (e.g. `*.gz`) against files + that have a path component containing a slash. + + - If the code reading a filter/exclude file gets an EINTR error, rsync now + clears the error flag on the file handle so it can keep on reading. + + - If `--relative` is active, the sending side cleans up trailing `/` or `/.` + suffixes to avoid triggering a bug in older rsync versions. Also, we now + reject a `..` dir if it would be sent as a relative dir. + + - If a non-directory is in the way of a directory and rsync is run with + `--dry-run` and `--delete`, rsync no longer complains about not being able + to opendir() the not-yet present directory. + + - When `--list-only` is used and a non-existent local destination dir was also + specified as a destination, rsync no longer generates a warning about being + unable to create the missing directory. + + - Fixed some problems with `--relative --no-implied-dirs` when the destination + directory did not yet exist: we can now create a symlink or device when it + is the first thing in the missing dir, and `--fuzzy` no longer complains + about being unable to open the missing dir. + + - Fixed a bug where the `--copy-links` option would not affect implied + directories without `--copy-unsafe-links` (see `--relative`). + + - Got rid of the need for `--force` to be used in some circumstances with + `--delete-after` (making it consistent with + `--delete-before`/`--delete-during`). + + - Rsync now ignores the SIGXFSZ signal, just in case your OS sends this when a + file is too large (rsync handles the write error). + + - Fixed a bug in the Proxy-Authorization header's base64-encoded value: it was + not properly padded with trailing '=' chars. This only affects a user that + need to use a password-authenticated proxy for an outgoing daemon-rsync + connection. + + - If we're transferring an empty directory to a new name, rsync no longer + forces `S_IWUSR` if it wasn't already set, nor does it accidentally leave it + set. + + - Fixed a bug in the debug output (`-vvvvv`) that could mention the wrong + checksum for the current file offset. + + - Rsync no longer allows a single directory to be copied over a non- directory + destination arg. + +### ENHANCEMENTS: + + - Added the `--append` option that makes rsync append data onto files that are + longer on the source than the destination (this includes new files). + + - Added the `--min-size=SIZE` option to exclude small files from the transfer. + + - Added the `--compress-level` option to allow you to set how aggressive + rsync's compression should be (this option implies `--compress`). + + - Enhanced the parsing of the SIZE value for `--min-size` and `--max-size` to + allow easy entry of multiples of 1000 (instead of just multiples of 1024) + and off-by-one values too (e.g. `--max-size=8mb-1`). + + - Added the `--8-bit-output` (`-8`) option, which tells rsync to avoid + escaping high-bit characters that it thinks are unreadable in the current + locale. + + - The new option `--human-readable` (`-h`) changes the output of `--progress`, + `--stats`, and the end-of-run summary to be easier to read. If repeated, the + units become powers of 1024 instead of powers of 1000. (The old meaning of + `-h`, as a shorthand for `--help`, still works as long as you just use it on + its own, as in `rsync -h`.) + + - If lutimes() and/or lchmod() are around, use them to allow the preservation + of attributes on symlinks. + + - The `--link-dest` option now affects symlinks and devices (when possible). + + - Added two config items to the rsyncd.conf parsing: `pre-xfer exec` and + `post-xfer exec`. These allow a command to be specified on a per-module + basis that will be run before and/or after a daemon-mode transfer. (See the + man page for a list of the environment variables that are set with + information about the transfer.) + + - When using the `--relative` option, you can now insert a dot dir in the + source path to indicate where the replication of the source dirs should + start. For example, if you specify a source path of + rsync://host/module/foo/bar/./baz/dir with `-R`, rsync will now only + replicate the `baz/dir` part of the source path (note: a trailing dot dir is + unaffected unless it also has a trailing slash). + + - Added some new `--no-FOO` options that make it easier to override unwanted + implied or default options. For example, `-a --no-o` (aka `--archive + --no-owner`) can be used to turn off the preservation of file ownership that + is implied by `-a`. + + - Added the `--chmod=MODE` option that allows the destination permissions to + be changed from the source permissions. E.g. `--chmod=g+w,o-rwx` + + - Added the `incoming chmod` and `outgoing chmod` daemon options that allow a + module to specify what permissions changes should be applied to all files + copied to and from the daemon. + + - Allow the `--temp-dir` option to be specified when starting a daemon, which + sets the default temporary directory for incoming files. + + - If `--delete` is combined with `--dirs` without `--recursive`, rsync will + now delete in any directory whose content is being synchronized. + + - If `--backup` is combined with `--delete` without `--backup-dir` (and + without `--delete-excluded`), we add a `protect` filter-rule to ensure that + files with the backup suffix are not deleted. + + - The file-count stats that are output by `--progress` were improved to better + indicate what the numbers mean. For instance, the output: `(xfer#5, + to-check=8383/9999)` indicates that this was the fifth file to be + transferred, and we still need to check 8383 more files out of a total of + 9999. + + - The include/exclude code now allows a `dir/***` directive (with 3 trailing + stars) to match both the dir itself as well as all the content below the dir + (`dir/**` would not match the dir). + + - Added the `--prune-empty-dirs` (`-m`) option that makes the receiving rsync + discard empty chains of directories from the file-list. This makes it easier + to selectively copy files from a source hierarchy and end up with just the + directories needed to hold the resulting files. + + - If the `--itemize-changes` (`-i`) option is repeated, rsync now includes + unchanged files in the itemized output (similar to `-vv`, but without all + the other verbose messages that can get in the way). Of course, the client + must be version 2.6.7 for this to work, but the remote rsync only needs to + be 2.6.7 if you're pushing files. + + - Added the `--specials` option to tell rsync to copy non-device special files + (which rsync now attempts even as a normal user). The `--devices` option now + requests the copying of just devices (character and block). The `-D` option + still requests both (e.g. `--devices` and `--specials`), `-a` still implies + `-D`, and non-root users still get a silent downgrade that omits device + copying. + + - Added the `--super` option to make the receiver always attempt super-user + activities. This is useful for systems that allow things such as devices to + be created or ownership to be set without being UID 0, and is also useful + for someone who wants to ensure that errors will be output if the receiving + rsync isn't being run as root. + + - Added the `--sockopts` option for those few who want to customize the TCP + options used to contact a daemon rsync. + + - Added a way for the `--temp-dir` option to be combined with a partial-dir + setting that lets rsync avoid non-atomic updates (for those times when + `--temp-dir` is not being used because space is tight). + + - A new support script, files-to-excludes, will transform a list of files into + a set of include/exclude directives that will copy those files. + + - A new option, `--executability` (`-E`) can be used to preserve just the + execute bit on files, for those times when using the `--perms` option is not + desired. + + - The daemon now logs each connection and also each module-list request that + it receives. + + - New log-format options: %M (modtime), %U (uid), %G (gid), and %B (permission + bits, e.g. `rwxr-xrwt`). + + - The `--dry-run` option no longer forces the enabling of `--verbose`. + + - The `--remove-sent-files` option now does a better job of incrementally + removing the sent files on the sending side (older versions tended to clump + up all the removals at the end). + + - A daemon now supersedes its minimal SIGCHLD handler with the standard + PID-remembering version after forking. This ensures that the generator can + get the child-exit status from the receiver. + + - Use of the `--bwlimit` option no longer interferes with the remote rsync + sending error messages about invalid/refused options. + + - Rsync no longer returns a usage error when used with one local source arg + and no destination: this now implies the `--list-only` option, just like the + comparable situation with a remote source arg. + + - Added the `--copy-dirlinks` option, a more limited version of + `--copy-links`. + + - Various documentation improvements, including: a better synopsis, some + improved examples, a better discussion of the presence and absence of + `--perms` (including how it interacts with the new `--executability` and + `--chmod` options), an extended discussion of `--temp-dir`, an improved + discussion of `--partial-dir`, a better description of rsync's pattern + matching characters, an improved `--no-implied-dirs` section, and the + documenting of what the `--stats` option outputs. + + - Various new and updated diffs in the patches dir, including: acls.diff, + xattrs.diff, atimes.diff, detect-renamed.diff, and slp.diff. + +### INTERNAL: + + - We now use sigaction() and sigprocmask() if possible, and fall back on + signal() if not. Using sigprocmask() ensures that rsync enables all the + signals that it needs, just in case it was started in a masked state. + + - Some buffer sizes were expanded a bit, particularly on systems where + MAXPATHLEN is overly small (e.g. cygwin). + + - If `io_printf()` tries to format more data than fits in the buffer, exit + with an error instead of transmitting a truncated buffer. + + - If a `va_copy` macro is defined, lib/snprintf.c will use it when defining + the `VA_COPY` macro. + + - Reduced the amount of stack memory needed for each level of directory + recursion by nearly MAXPATHLEN bytes. + + - The wildmatch function was extended to allow an array of strings to be + supplied as the string to match. This allows the exclude code to do less + string copying. + + - Got rid of the `safe_fname()` function (and all the myriad calls) and + replaced it with a new function in the log.c code that filters all the + output going to the terminal. + + - Unified the `f_name()` and the `f_name_to()` functions. + + - Improved the hash-table code the sender uses to handle checksums to make it + use slightly less memory and run just a little faster. + +### DEVELOPER RELATED: + + - The diffs in the patches dir now require `patch -p1 <DIFF` instead of the + previous `-p0`. Also, the version included in the release tar now affect + generated files (e.g. configure, rsync.1, proto.h, etc.), so it is no longer + necessary to run autoconf and/or yodl unless you're applying a patch that + was checked out from CVS. + + - Several diffs in the patches dir now use the proper `--enable-FOO` configure + option instead of `--with-FOO` to turn on the inclusion of the newly patched + feature. + + - There is a new script, `prepare-source` than can be used to update the + various generated files (proto.h, configure, etc.) even before configure has + created the Makefile (this is mainly useful when patching the source with a + patch that doesn't affect generated files). + + - The testsuite now sets HOME so that it won't be affected by a file such as + ~/.popt. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.6 (28 Jul 2005) + +Protocol: 29 (unchanged) + +## Changes since 2.6.5: + +### SECURITY FIXES: + + - The zlib code was upgraded to version 1.2.3 in order to make it more secure. + While the widely-publicized security problem in zlib 1.2.2 did not affect + rsync, another security problem surfaced that affects rsync's zlib 1.1.4. + +### BUG FIXES: + + - The setting of `flist->high` in `clean_flist()` was wrong for an empty list. + This could cause `flist_find()` to crash in certain rare circumstances (e.g. + if just the right directory setup was around when `--fuzzy` was combined + with `--link-dest`). + + - The outputting of hard-linked files when verbosity was > 1 was not right: + (1) Without `-i` it would output the name of each hard-linked file as though + it had been changed; it now outputs a `is hard linked` message for the file. + (2) With `-i` it would output all dots for the unchanged attributes of a + hard-link; it now changes those dots to spaces, as is done for other totally + unchanged items. + + - When backing up a changed symlink or device, get rid of any old backup item + so that we don't get an `already exists` error. + + - A couple places that were comparing a local and a remote modification- time + were not honoring the `--modify-window` option. + + - Fixed a bug where the 'p' (permissions) itemized-changes flag might get set + too often (if some non-significant mode bits differed). + + - Fixed a really old, minor bug that could cause rsync to warn about being + unable to mkdir() a path that ends in `/.` because it just created the + directory (required `--relative`, `--no-implied-dirs`, a source path that + ended in either a trailing slash or a trailing `/.`, and a non-existing + destination dir to tickle the bug in a recent version). + +### ENHANCEMENTS: + + - Made the `max verbosity` setting in the rsyncd.conf file settable on a + per-module basis (which now matches the documentation). + + - The support/rrsync script has been upgraded to verify the args of options + that take args (instead of rejecting any such options). The script was also + changed to try to be more secure and to fix a problem in the parsing of a + pull operation that has multiple sources. + + - Improved the documentation that explains the difference between a normal + daemon transfer and a daemon-over remote-shell transfer. + + - Some of the diffs supplied in the patches dir were fixed and/or improved. + +### BUILD CHANGES: + + - Made configure define `NOBODY_USER` (currently hard-wired to `nobody`) and + `NOBODY_GROUP` (set to either `nobody` or `nogroup` depending on what we + find in the /etc/group file). + + - Added a test to the test suite, itemized.test, that tests the output of `-i` + (log-format w/%i) and some double-verbose messages. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.5 (1 Jun 2005) + +Protocol: 29 (unchanged) + +## Changes since 2.6.4: + +### OUTPUT CHANGES: + + - Non-printable chars in filenames are now output using backslash- escaped + characters rather than '?'s. Any non-printable character is output using 3 + digits of octal (e.g. `\n` -> `\012`), and a backslash is now output as + `\\`. Rsync also uses your locale setting, which can make it treat fewer + high-bit characters as non-printable. + + - If rsync received an empty file-list when pulling files, it would output a + `nothing to do` message and exit with a 0 (success) exit status, even if the + remote rsync returned an error (it did not do this under the same conditions + when pushing files). This was changed to make the pulling behavior the same + as the pushing behavior: we now do the normal end-of-run outputting + (depending on options) and exit with the appropriate exit status. + +### BUG FIXES: + + - A crash bug was fixed when a daemon had its `path` set to `/`, did not have + chroot enabled, and used some anchored excludes in the rsyncd.conf file. + + - Fixed a bug in the transfer of a single file when `-H` is specified (rsync + would either infinite loop or perhaps crash). + + - Fixed a case where the generator might try (and fail) to tweak the + write-permissions of a read-only directory in list-only mode (this only + caused an annoying warning message). + + - If `--compare-dest` or `--link-dest` uses a locally-copied file as the basis + for an updated version, log this better when `--verbose` or `-i` is in + effect. + + - Fixed the accidental disabling of `--backup` during the `--delete-after` + processing. + + - Restored the ability to use the `--address` option in client mode (in + addition to its use in daemon mode). + + - Make sure that some temporary progress information from the delete + processing does not get left on the screen when it is followed by a newline. + + - When `--existing` skips a directory with extra verbosity, refer to it as a + `directory`, not a `file`. + + - When transferring a single file to a different-named file, any generator + messages that are source-file related no longer refer to the file by the + destination filename. + + - Fixed a bug where hard-linking a group of files might fail if the generator + hasn't created a needed destination directory yet. + + - Fixed a bug where a hard-linked group of files that is newly-linked to a + file in a `--link-dest` dir doesn't link the files from the rest of the + cluster. + + - When deleting files with the `--one-file-system` (`-x`) option set, rsync no + longer tries to remove files from inside a mount-point on the receiving + side. Also, we don't complain about being unable to remove the mount-point + dir. + + - Fixed a compatibility problem when using `--cvs-ignore` (`-C`) and sending + files to an older rsync without using `--delete`. + + - Make sure that a `- !` or `+ !` include/exclude pattern does not trigger the + list-clearing action that is reserved for `!`. + + - Avoid a timeout in the generator when the sender/receiver aren't handling + the generator's checksum output quickly enough. + + - Fixed the omission of some directories in the delete processing when + `--relative` (`-R`) was combined with a source path that had a trailing + slash. + + - Fixed a case where rsync would erroneously delete some files and then + re-transfer them when the options `--relative` (`-R`) and `--recursive` + (`-r`) were both enabled (along with `--delete`) and a source path had a + trailing slash. + + - Make sure that `--max-size` doesn't affect a device or a symlink. + + - Make sure that a system with a really small MAXPATHLEN does not cause the + buffers in `readfd_unbuffered()` to be too small to receive normal messages. + (This mainly affected Cygwin.) + + - If a source pathname ends with a filename of `..`, treat it as if `../` had + been specified (so that we don't copy files to the parent dir of the + destination). + + - If `--delete` is combined with a file-listing rsync command (i.e. no + transfer is happening), avoid outputting a warning that we couldn't delete + anything. + + - If `--stats` is specified with `--delete-after`, ensure that all the + `deleting` messages are output before the statistics. + + - Improved one `if` in the deletion code that was only checking errno for + ENOTEMPTY when it should have also been checking for EEXIST (for + compatibility with OS variations). + +### ENHANCEMENTS: + + - Added the `--only-write-batch=FILE` option that may be used (instead of + `--write-batch=FILE`) to create a batch file without doing any actual + updating of the destination. This allows you to divert all the file-updating + data away from a slow data link (as long as you are pushing the data to the + remote server when creating the batch). + + - When the generator is taking a long time to fill up its output buffer (e.g. + if the transferred files are few, small, or missing), it now periodically + flushes the output buffer so that the sender/receiver can get started on the + files sooner rather than later. + + - Improved the keep-alive code to handle a long silence between the sender and + the receiver that can occur when the sender is receiving the checksum data + for a large file. + + - Improved the auth-errors that are logged by the daemon to include some + information on why the authorization failed: wrong user, password mismatch, + etc. (The client-visible message is unchanged!) + + - Improved the client's handling of an `@ERROR` from a daemon so that it does + not complain about an unexpectedly closed socket (since we really did expect + the socket to close). + + - If the daemon can't open the log-file specified in rsyncd.conf, fall back to + using syslog and log an appropriate warning. This is better than what was + typically a totally silent (and fatal) failure (since a daemon is not + usually run with the `--no-detach` option that was necessary to see the + error on stderr). + + - The man pages now consistently refer to an rsync daemon as a `daemon` + instead of a `server` (to distinguish it from the server process in a + non-daemon transfer). + + - Made a small change to the rrsync script (restricted rsync -- in the support + dir) to make a read-only server reject all `--remove-*` options when sending + files (to future-proof it against the possibility of other similar options + being added at some point). + +### INTERNAL: + + - Rsync now calls `setlocale(LC_CTYPE, "")`. This enables isprint() to better + discern which filename characters need to be escaped in messages (which + should result in fewer escaped characters in some locales). + + - Improved the naming of the log-file open/reopen/close functions. + + - Removed some protocol-compatibility code that was only needed to help + someone running a pre-release of 2.6.4. + +### BUILD CHANGES: + + - Added configure option `--disable-locale` to disable any use of setlocale() + in the binary. + + - Fixed a bug in the `SUPPORT{,_HARD}_LINKS` #defines which prevented rsync + from being built without symlink or hard-link support. + + - Only #define `HAVE_REMSH` if it is going to be set to 1. + + - Configure now disables the use of mkstemp() under HP-UX (since they refuse + to fix its broken handling of large files). + + - Configure now explicitly checks for the lseek64() function so that the code + can use `HAVE_LSEEK64` instead of inferring lseek64()'s presence based on + the presence of the `off64_t` type. + + - Configure no longer mentions the change in the default remote-shell (from + rsh to ssh) that occurred for the 2.6.0 release. + + - Some minor enhancements to the test scripts. + + - Added a few new `*.diff` files to the patches dir, including a patch that + enables the optional copying of extended attributes. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.4 (30 March 2005) + +Protocol: 29 (changed) + +## Changes since 2.6.3: + +### OUTPUT CHANGES: + + - When rsync deletes a directory and outputs a verbose message about it, it + now appends a trailing slash to the name instead of (only sometimes) + outputting a preceding "directory " string. + + - The `--stats` output will contain file-list time-statistics if both sides + are 2.6.4, or if the local side is 2.6.4 and the files are being pushed + (since the stats come from the sending side). (Requires protocol 29 for a + pull.) + + - The `%o` (operation) log-format escape now has a third value (besides `send` + and `recv`): `del.` (with trailing dot to make it 4 chars). This changes + the way deletions are logged in the daemon's log file. + + - When the `--log-format` option is combined with `--verbose`, rsync now + avoids outputting the name of the file twice in most circumstances. As long + as the `--log-format` item does not refer to any post-transfer items (such + as %b or %c), the `--log-format` message is output prior to the transfer, so + `--verbose` is now the equivalent of a `--log-format` of '%n%L' (which + outputs the name and any link info). If the log output must occur after the + transfer to be complete, the only time the name is also output prior to the + transfer is when `--progress` was specified (so that the name will precede + the progress stats, and the full `--log-format` output will come after). + + - Non-printable characters in filenames are replaced with a '?' to avoid + corrupting the screen or generating empty lines in the output. + +### BUG FIXES: + + - Restore the list-clearing behavior of `!` in a .cvsignore file (2.6.3 was + only treating it as a special token in an rsync include/exclude file). + + - The combination of `--verbose` and `--dry-run` now mentions the full list of + changes that would be output without `--dry-run`. + + - Avoid a mkdir warning when removing a directory in the destination that + already exists in the `--backup-dir`. + + - An OS that has a binary mode for its files (such as cygwin) needed + `setmode(fd, O_BINARY)` called on the temp-file we opened with mkstemp(). + (Fix derived from cygwin's 2.6.3 rsync package.) + + - Fixed a potential hang when verbosity is high, the client side is the + sender, and the file-list is large. + + - Fixed a potential protocol-corrupting bug where the generator could merge a + message from the receiver into the middle of a multiplexed packet of data if + only part of that data had been written out to the socket when the message + from the generator arrived. + + - We now check if the OS doesn't support using mknod() for creating FIFOs and + sockets, and compile-in some compatibility code using mkfifo() and socket() + when necessary. + + - Fixed an off-by-one error in the handling of `--max-delete=N`. Also, if the + `--max-delete` limit is exceeded during a run, we now output a warning about + this at the end of the run and exit with a new error code (25). + + - One place in the code wasn't checking if fork() failed. + + - The `ignore nonreadable` daemon parameter used to erroneously affect + readable symlinks that pointed to a non-existent file. + + - If the OS does not have lchown() and a chown() of a symlink will affect the + referent of a symlink (as it should), we no longer try to set the user and + group of a symlink. + + - The generator now properly runs the hard-link loop and the dir-time + rewriting loop after we're sure that the redo phase is complete. + + - When `--backup` was specified with `--partial-dir=DIR`, where DIR is a + relative path, the backup code was erroneously trying to backup a file that + was put into the partial-dir. + + - If a file gets resent in a single transfer and the `--backup` option is + enabled along with `--inplace`, rsync no longer performs a duplicate backup + (it used to overwrite the first backup with the failed file). + + - One call to `flush_write_file()` was not being checked for an error. + + - The `--no-relative` option was not being sent from the client to a server + sender. + + - If an rsync daemon specified `dont compress = ...` for a file and the client + tried to specify `--compress`, the libz code was not handling a compression + level of 0 properly. This could cause a transfer failure if the block-size + for a file was large enough (e.g. rsync might have exited with an error for + large files). + + - Fixed a bug that would sometimes surface when using `--compress` and sending + a file with a block-size larger than 64K (either manually specified, or + computed due to the file being really large). Prior versions of rsync would + sometimes fail to decompress the data properly, and thus the transferred + file would fail its verification. + + - If a daemon can't open the specified log file (i.e. syslog is not being + used), die without crashing. We also output an error about the failure on + stderr (which will only be seen if `--no-detach` was specified) and exit + with a new error code (6). + + - A local transfer no longer duplicates all its include/exclude options (since + the forked process already has a copy of the exclude list, there's no need + to send them a set of duplicates). + + - The output of the items that are being updated by the generator (dirs, + symlinks, devices) is now intermingled in the proper order with the output + from the items that the receiver is updating (regular files) when pulling. + This misordering was particularly bad when `--progress` was specified. + (Requires protocol 29.) + + - When `--timeout` is specified, lulls that occur in the transfer while the + generator is doing work that does not generate socket traffic (looking for + changed files, deleting files, doing directory-time touch-ups, etc.) will + cause a new keep-alive packet to be sent that should keep the transfer going + as long as the generator continues to make progress. (Requires protocol 29.) + + - The stat size of a device is not added to the total file size of the items + in the transfer (the size might be undefined on some OSes). + + - Fixed a problem with refused-option messages sometimes not making it back to + the client side when a remote `--files-from` was in effect and the daemon + was the receiver. + + - The `--compare-dest` option was not updating a file that differed in (the + preserved) attributes from the version in the compare-dest DIR. + + - When rsync is copying files into a write-protected directory, fixed the + change-report output for the directory so that we don't report an identical + directory as changed. + +### ENHANCEMENTS: + + - Rsync now supports popt's option aliases, which means that you can use + /etc/popt and/or ~/.popt to create your own option aliases. + + - Added the `--delete-during` (`--del`) option which will delete files from + the receiving side incrementally as each directory in the transfer is being + processed. This makes it more efficient than the default, + before-the-transfer behavior, which is now also available as + `--delete-before` (and is still the default `--delete-WHEN` option that will + be chosen if `--delete` or `--delete-excluded` is specified without a + `--delete-WHEN` choice). All the `--del*` options infer `--delete`, so an + rsync daemon that refuses `delete` will still refuse to allow any + file-deleting options (including the new `--remove-sent-files` option). + + - All the `--delete-WHEN` options are now more memory efficient: Previously an + duplicate set of file-list objects was created on the receiving side for the + entire destination hierarchy. The new algorithm only creates one directory + of objects at a time (for files inside the transfer). + + - Added the `--copy-dest` option, which works like `--link-dest` except that + it locally copies identical files instead of hard-linking them. + + - Added support for specifying multiple `--compare-dest`, `--copy-dest`, or + `--link-dest` options, but only of a single type. (Promoted from the patches + dir and enhanced.) (Requires protocol 29.) + + - Added the `--max-size` option. (Promoted from the patches dir.) + + - The daemon-mode options are now separated from the normal rsync options so + that they can't be mixed together. This makes it impossible to start a + daemon that has improper default option values (which could cause problems + when a client connects, such as hanging or crashing). + + - The `--bwlimit` option may now be used in combination with `--daemon` to + specify both a default value for the daemon side and a value that cannot be + exceeded by a user-specified `--bwlimit` option. + + - Added the `port` parameter to the rsyncd.conf file. (Promoted from the + patches dir.) Also added `address`. The command-line options take precedence + over a config-file option, as expected. + + - In `_exit_cleanup()`: when we are exiting with a partially-received file, we + now flush any data in the write-cache before closing the partial file. + + - The `--inplace` support was enhanced to work with `--compare-dest`, + `--link-dest`, and (the new) `--copy-dest` options. (Requires protocol 29.) + + - Added the `--dirs` (`-d`) option for an easier way to copy directories + without recursion. Any directories that are encountered are created on the + destination. Specifying a directory with a trailing slash copies its + immediate contents to the destination. + + - The `--files-from` option now implies `--dirs` (`-d`). + + - Added the `--list-only` option, which is mainly a way for the client to put + the server into listing mode without needing to resort to any internal + option kluges (e.g. the age-old use of `-r --exclude='/*/*'` for a + non-recursive listing). This option is used automatically (behind the + scenes) when a modern rsync speaks to a modern daemon, but may also be + specified manually if you want to force the use of the `--list-only` option + over a remote-shell connection. + + - Added the `--omit-dir-times` (`-O`) option, which will avoid updating the + modified time for directories when `--times` was specified. This option will + avoid an extra pass through the file-list at the end of the transfer (to + tweak all the directory times), which may provide an appreciable speedup for + a really large transfer. (Promoted from the patches dir.) + + - Added the `--filter` (`-f`) option and its helper option, `-F`. Filter rules + are an extension to the existing include/exclude handling that also supports + nested filter files as well as per-directory filter files (like .cvsignore, + but with full filter-rule parsing). This new option was chosen in order to + ensure that all existing include/exclude processing remained 100% compatible + with older versions. Protocol 29 is needed for full filter-rule support, but + backward-compatible rules work with earlier protocol versions. (Promoted + from the patches dir and enhanced.) + + - Added the `--delay-updates` option that puts all updated files into a + temporary directory (by default `.~tmp~`, but settable via the + `--partial-dir=DIR` option) until the end of the transfer. This makes the + updates a little more atomic for a large transfer. + + - If rsync is put into the background, any output from `--progress` is + reduced. + + - Documented the `max verbosity` setting for rsyncd.conf. (This setting was + added a couple releases ago, but left undocumented.) + + - The sender and the generator now double-check the file-list index they are + given, and refuse to try to do a file transfer on a non-file index (since + that would indicate that something had gone very wrong). + + - Added the `--itemize-changes` (`-i`) option, which is a way to output a more + detailed list of what files changed and in what way. The effect is the same + as specifying a `--log-format` of `%i %n%L` (see both the rsync and + rsyncd.conf manpages). Works with `--dry-run` too. + + - Added the `--fuzzy` (`-y`) option, which attempts to find a basis file for a + file that is being created from scratch. The current algorithm only looks in + the destination directory for the created file, but it does attempt to find + a match based on size/mod-time (in case the file was renamed with no other + changes) as well as based on a fuzzy name-matching algorithm. This option + requires protocol 29 because it needs the new file-sorting order. (Promoted + from patches dir and enhanced.) (Requires protocol 29.) + + - Added the `--remove-sent-files` option, which lets you move files between + systems. + + - The hostname in HOST:PATH or HOST::PATH may now be an IPv6 literal enclosed + in '[' and ']' (e.g. `[::1]`). (We already allowed IPv6 literals in the + rsync://HOST:PORT/PATH format.) + + - When rsync recurses to build the file list, it no longer keeps open one or + more directory handles from the dir's parent dirs. + + - When building under windows, the default for `--daemon` is now to avoid + detaching, requiring the new `--detach` option to force rsync to detach. + + - The `--dry-run` option can now be combined with either `--write-batch` or + `--read-batch`, allowing you to run a do-nothing test command to see what + would happen without `--dry-run`. + + - The daemon's `read only` config item now sets an internal `read_only` + variable that makes extra sure that no write/delete calls on the read-only + side can succeed. + + - The log-format % escapes can now have a numeric field width in between the % + and the escape letter (e.g. `%-40n %08p`). + + - Improved the option descriptions in the `--help` text. + +### SUPPORT FILES: + + - Added atomic-rsync to the support dir: a perl script that will transfer some + files using rsync, and then move the updated files into place all at once at + the end of the transfer. Only works when pulling, and uses `--link-dest` and + a parallel hierarchy of files to effect its update. + + - Added mnt-excl to the support dir: a perl script that takes the /proc/mounts + file and translates it into a set of excludes that will exclude all mount + points (even mapped mounts to the same disk). The excludes are made relative + to the specified source dir and properly anchored. + + - Added savetransfer.c to the support dir: a C program that can make a copy of + all the data that flows over the wire. This lets you test for data + corruption (by saving the data on both the sending side and the receiving + side) and provides one way to debug a protocol error. + + - Added rrsync to the support dir: this is an updated version of Joe Smith's + restricted rsync perl script. This helps to ensure that only certain rsync + commands can be run by an ssh invocation. + +### INTERNAL: + + - Added better checking of the checksum-header values that come over the + socket. + + - Merged a variety of file-deleting functions into a single function so that + it is easier to maintain. + + - Improved the type of some variables (particularly blocksize vars) for + consistency and proper size. + + - Got rid of the uint64 type (which we didn't need). + + - Use a slightly more compatible set of core #include directives. + + - Defined int32 in a way that ensures that the build dies if we can't find a + variable with at least 32 bits. + +### PROTOCOL DIFFERENCES FOR VERSION 29: + + - A 16-bit flag-word is transmitted after every file-list index. This + indicates what is changing between the sender and the receiver. The + generator now transmits an index and a flag-word to indicate when dirs and + symlinks have changed (instead of producing a message), which makes the + outputting of the information more consistent and less prone to screen + corruption (because the local receiver/sender is now outputting all the + file-change info messages). + + - If a file is being hard-linked, the `ITEM_XNAME_FOLLOWS` bit is enabled in + the flag-word and the name of the file that was linked immediately follows + in vstring format (see below). + + - If a file is being transferred with an alternate-basis file, the + `ITEM_BASIS_TYPE_FOLLOWS` bit is enabled in the flag-word and a single byte + follows, indicating what type of basis file was chosen. If that indicates + that a fuzzy-match was selected, the `ITEM_XNAME_FOLLOWS` bit is set in the + flag-word and the name of the match in vstring format follows the basis + byte. A vstring is a variable length string that has its size written prior + to the string, and no terminating null. If the string is from 1-127 bytes, + the length is a single byte. If it is from 128-32767 bytes, the length is + written as ((len >> 8) | 0x80) followed by (len % 0x100). + + - The sending of exclude names is done using filter-rule syntax. This means + that all names have a prefixed rule indicator, even excludes (which used to + be sent as a bare pattern, when possible). The `-C` option will include the + per-dir .cvsignore merge file in the list of filter rules so it is + positioned correctly (unlike in some older transfer scenarios). + + - Rsync sorts the filename list in a different way: it sorts the subdir names + after the non-subdir names for each dir's contents, and it always puts a + dir's contents immediately after the dir's name in the list. (Previously an + item named `foo.txt` would sort in between directory `foo/` and `foo/bar`.) + + - When talking to a protocol 29 rsync daemon, a list-only request is able to + note this before the options are sent over the wire and the new + `--list-only` option is included in the options. + + - When the `--stats` bytes are sent over the wire (or stored in a batch), they + now include two elapsed-time values: one for how long it took to build the + file-list, and one for how long it took to send it over the wire (each + expressed in thousandths of a second). + + - When `--delete-excluded` is specified with some filter rules (AKA excludes), + a client sender will now initiate a send of the rules to the receiver (older + protocols used to omit the sending of excludes in this situation since there + were no receiver-specific rules that survived `--delete-excluded` back + then). Note that, as with all the filter-list sending, only items that are + significant to the other side will actually be sent over the wire, so the + filter-rule list that is sent in this scenario is often empty. + + - An index equal to the file-list count is sent as a keep-alive packet from + the generator to the sender, which then forwards it on to the receiver. This + normally invalid index is only a valid keep-alive packet if the 16-bit + flag-word that follows it contains a single bit (`ITEM_IS_NEW`, which is + normally an illegal flag to appear alone). + + - A protocol-29 batch file includes a bit for the setting of the `--dirs` + option and for the setting of the `--compress` option. Also, the shell + script created by `--write-batch` will use the `--filter` option instead of + `--exclude-from` to capture any filter rules. + +### BUILD CHANGES: + + - Handle an operating system that use mkdev() in place of makedev(). + + - Improved configure to better handle cross-compiling. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.3 (30 Sep 2004) + +Protocol: 28 (unchanged) + +## Changes since 2.6.2: + +### SECURITY FIXES: + + - A bug in the `sanitize_path` routine (which affects a non-chrooted rsync + daemon) could allow a user to craft a pathname that would get transformed + into an absolute path for certain options (but not for file-transfer names). + If you're running an rsync daemon with chroot disabled, **please upgrade**, + ESPECIALLY if the user privs you run rsync under is anything above `nobody`. + + OUTPUT CHANGES (ATTN: those using a script to parse the verbose output): + + - Please note that the 2-line footer (output when verbose) now uses the term + `sent` instead of `wrote` and `received` instead of `read`. If you are not + parsing the numeric values out of this footer, a script would be better off + using the empty line prior to the footer as the indicator that the verbose + output is over. + + - The output from the `--stats` option was similarly affected to change + `written` to `sent` and `read` to `received`. + + - Rsync ensures that a filename that contains a newline gets mentioned with + each newline transformed into a question mark (which prevents a filename + from causing an empty line to be output). + + - The `backed up ...` message that is output when at least 2 `--verbose` + options are specified is now the same both with and without the + `--backup-dir` option. + +### BUG FIXES: + + - Fixed a crash bug that might appear when `--delete` was used and multiple + source directories were specified. + + - Fixed a 32-bit truncation of the file length when generating the checksums. + + - The `--backup` code no longer attempts to create some directories over and + over again (generating warnings along the way). + + - Fixed a bug in the reading of the secrets file (by the daemon) and the + password file (by the client): the files no longer need to be terminated by + a newline for their content to be read in. + + - If a file has a read error on the sending side or the reconstructed data + doesn't match the expected checksum (perhaps due to the basis file changing + during the transfer), the receiver will no longer retain the resulting file + unless the `--partial` option was specified. (Note: for the read-error + detection to work, neither side can be older than 2.6.3 -- older receivers + will always retain the file, and older senders don't tell the receiver that + the file had a read error.) + + - If a file gets resent in a single transfer and the `--backup` option is + enabled, rsync no longer performs a duplicate backup (it used to overwrite + the original file in the backup area). + + - Files specified in the daemon's `exclude` or `exclude from` config items are + now excluded from being uploaded (assuming that the module allows uploading + at all) in addition to the old download exclusion. + + - Got rid of a potential hang in the receiver when near the end of a phase. + + - When using `--backup` without a `--backup-dir`, rsync no longer preserves + the modify time on directories. This avoids confusing NFS. + + - When `--copy-links` (`-L`) is specified, we now output a separate error for + a symlink that has no referent instead of claiming that a file `vanished`. + + - The `--copy-links` (`-L`) option no longer has the side-effect of telling + the receiving side to follow symlinks. See the `--keep-dirlinks` option + (mentioned below) for a way to specify that behavior. + + - Error messages from the daemon server's option-parsing (such as refused + options) are now successfully transferred back to the client (the server + used to fail to send the message because the socket wasn't in the right + state for the message to get through). + + - Most transfer errors that occur during a daemon transfer are now returned to + the user in addition to being logged (some messages are intended to be + daemon-only and are not affected by this). + + - Fixed a bug in the daemon authentication code when using one of the + batch-processing options. + + - We try to work around some buggy IPv6 implementations that fail to implement + `IPV6_V6ONLY`. This should fix the `address in use` error that some daemons + get when running on an OS with a buggy IPv6 implementation. Also, if the new + code gets this error, we might suggest that the user specify `--ipv4` or + `--ipv6` (if we think it will help). + + - When the remote rsync dies, make a better effort to recover any error + messages it may have sent before dying (the local rsync used to just die + with a socket-write error). + + - When using `--delete` and a `--backup-dir` that contains files that are + hard-linked to their destination equivalents, rsync now makes sure that + removed files really get removed (avoids a really weird rename() behavior). + + - Avoid a bogus run-time complaint about a lack of 64-bit integers when the + int64 type is defined as an `off_t` and it actually has 64-bits. + + - Added a configure check for open64() without mkstemp64() so that we can + avoid using mkstemp() when such a combination is encountered. This bypasses + a problem writing out large temp files on OSes such as AIX and HP-UX. + + - Fixed an age-old crash problem with `--read-batch` on a local copy (rsync + was improperly assuming `--whole-file` for the local copy). + + - When `--dry-run` (`-n`) is used and the destination directory does not + exist, rsync now produces a correct report of files that would be sent + instead of dying with a chdir() error. + + - Fixed a bug that could cause a slow-to-connect rsync daemon to die with an + error instead of waiting for the connection to finish. + + - Fixed an ssh interaction that could cause output to be lost when the user + chose to combine the output of rsync's stdout and stderr (e.g. using the + `2>&1`). + + - Fixed an option-parsing bug when `--files-from` got passed to a daemon. + +### ENHANCEMENTS: + + - Added the `--partial-dir=DIR` option that lets you specify where to + (temporarily) put a partially transferred file (instead of over- writing the + destination file). E.g. `--partial-dir=.rsync-partial` Also added support + for the `RSYNC_PARTIAL_DIR` environment variable that, when found, + transforms a regular `--partial` option (such as the convenient `-P` option) + into one that also specifies a directory. + + - Added `--keep-dirlinks` (`-K`), which allows you to symlink a directory onto + another partition on the receiving side and have rsync treat it as matching + a normal directory from the sender. + + - Added the `--inplace` option that tells rsync to write each destination file + without using a temporary file. The matching of existing data in the + destination file can be severely limited by this, but there are also cases + where this is more efficient (such as appending data). Use only when needed + (see the man page for more details). + + - Added the `write only` option for the daemon's config file. + + - Added long-option names for `-4` and `-6` (namely `--ipv4` and `--ipv6`) and + documented all these options in the man page. + + - Improved the handling of the `--bwlimit` option so that it's less bursty, + more accurate, and works properly over a larger range of values. + + - The rsync daemon-over-ssh code now looks for `SSH_CONNECTION` and + `SSH2_CLIENT` in addition to `SSH_CLIENT` to figure out the IP address. + + - Added the `--checksum-seed=N` option for advanced users. + + - Batch writing/reading has a brand-new implementation that is simpler, fixes + a few weird problems with the old code (such as no longer sprinkling the + batch files into different dirs or even onto different systems), and is much + less intrusive into the code (making it easier to maintain for the future). + The new code generates just one data file instead of three, which makes it + possible to read the batch on stdin via a remote shell. Also, the old + requirement of forcing the same fixed checksum-seed for all batch processing + has been removed. + + - If an rsync daemon has a module set with `list = no` (which hides its + presence in the list of available modules), a user that fails to + authenticate gets the same `unknown module` error that they would get if the + module were actually unknown (while still logging the real error to the + daemon's log file). This prevents fishing for module names. + + - The daemon's `refuse options` config item now allows you to match option + names using wildcards and/or the single-letter option names. + + - Each transferred file now gets its permissions and modified-time updated + before the temp-file gets moved into place. Previously, the finished file + would have a very brief window where its permissions disallowed all group + and world access. + + - Added the ability to parse a literal IPv6 address in an `rsync:` URL (e.g. + rsync://[2001:638:500:101::21]:873/module/dir). + + - The daemon's wildcard expanding code can now handle more than 1000 filenames + (it's now limited by memory instead of having a hard-wired limit). + +### INTERNAL: + + - Some cleanup in the exclude code has saved some per-exclude memory and made + the code easier to maintain. + + - Improved the argv-overflow checking for a remote command that has a lot of + args. + + - Use rsyserr() in the various places that were still calling rprintf() with + strerror() as an arg. + + - If an rsync daemon is listening on multiple sockets (to handle both IPv4 and + IPv6 to a single port), we now close all the unneeded file handles after we + accept a connection (we used to close just one of them). + + - Optimized the handling of larger block sizes (rsync used to slow to a crawl + if the block size got too large). + + - Optimized away a loop in `hash_search()`. + + - Some improvements to the `sanitize_path()` and `clean_fname()` functions + makes them more efficient and produce better results (while still being + compatible with the file-name cleaning that gets done on both sides when + sending the file-list). + + - Got rid of `alloc_sanitize_path()` after adding a destination-buffer arg to + `sanitize_path()` made it possible to put all the former's functionality + into the latter. + + - The file-list that is output when at least 4 verbose options are specified + reports the uid value on the sender even when rsync is not running as root + (since we might be sending to a root receiver). + +### BUILD CHANGES: + + - Added a `gen` target to rebuild most of the generated files, including + configure, config.h.in, the man pages, and proto.h. + + - If `make proto` doesn't find some changes in the prototypes, the proto.h + file is left untouched (its time-stamp used to always be updated). + + - The variable $STRIP (that is optionally set by the install-strip target's + rule) was changed to `$INSTALL_STRIP` because some systems have $STRIP + already set in the environment. + + - Fixed a build problem when `SUPPORT_HARD_LINKS` isn't defined. + + - When cross-compiling, the gettimeofday() function is now assumed to be a + modern version that takes two-args (since we can't test it). + +### DEVELOPER RELATED: + + - The scripts in the testsuite dir were cleaned up a bit and a few new tests + added. + + - Some new diffs were added to the patches dir, and some accepted ones were + removed. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.2 (30 Apr 2004) + +Protocol: 28 (unchanged) + +## Changes since 2.6.1: + +### BUG FIXES: + + - Fixed a major bug in the sorting of the filenames when `--relative` is used + for some sources (just sources such as `/` and `/*` were affected). This fix + ensures that we ask for the right file-list item when requesting changes + from the sender. + + - Rsync now checks the return value of the close() function to better report + disk-full problems on an NFS file system. + + - Restored the old daemon-server behavior of logging error messages rather + than returning them to the user. (A better long-term fix will be sought in + the future.) + + - An obscure uninitialized-variable bug was fixed in the uid/gid code. (This + bug probably had no ill effects.) + +### BUILD CHANGES: + + - Got rid of the configure check for sys/sysctl.h (it wasn't used and was + causing a problem on some systems). Also improved the + broken-largefile-locking test to try to avoid failure due to an NFS + build-dir. + + - Fixed a compile problem on systems that don't define `AI_NUMERICHOST`. + + - Fixed a compile problem in the popt source for compilers that don't support + `__attribute__`. + +### DEVELOPER RELATED: + + - Improved the testsuite's `merge` test to work on OSF1. + + - Two new diffs were added to the patches dir. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.1 (26 Apr 2004) + +Protocol: 28 (changed) + +## Changes since 2.6.0: + +### SECURITY FIXES: + + - Paths sent to an rsync daemon are more thoroughly sanitized when chroot is + not used. If you're running a non-read-only rsync daemon with chroot + disabled, **please upgrade**, ESPECIALLY if the user privs you run rsync + under is anything above `nobody`. + +### ENHANCEMENTS: + + - Lower memory use, more optimal transfer of data over the socket, and lower + CPU usage (see the INTERNAL section for details). + + - The `RSYNC_PROXY` environment variable can now contain a `USER:PASS@` prefix + before the `HOST:PORT` information. (Bardur Arantsson) + + - The `--progress` output now mentions how far along in the transfer we are, + including both a count of files transferred and a percentage of the total + file-count that we've processed. It also shows better + current-rate-of-transfer and remaining-transfer-time values. + + - Documentation changes now attempt to describe some often mis- understood + features more clearly. + +### BUG FIXES: + + - When `-x` (`--one-file-system`) is combined with `-L` (`--copy-links`) or + `--copy-unsafe-links,` no symlinked files are skipped, even if the referent + file is on a different filesystem. + + - The `--link-dest` code now works properly for a non-root user when (1) the + UIDs of the source and destination differ and `-o` was specified, or (2) + when the group of the source can't be used on the destination and `-g` was + specified. + + - Fixed a bug in the handling of `-H` (hard-links) that might cause the + expanded PATH/NAME value of the current item to get overwritten (due to an + expanded-name caching bug). + + - We now reset the `new data has been sent` flag at the start of each file we + send. This makes sure that an interrupted transfer with the `--partial` + option set doesn't keep a shorter temp file than the current basis file when + no new data has been transferred over the wire for that file. + + - Fixed a byte-order problem in `--batch-mode` on big-endian machines. (Jay + Fenlason) + + - When using `--cvs-exclude`, the exclude items we get from a per-directory's + .cvsignore file once again only affect that one directory (not all following + directories too). The items are also now properly word-split and parsed + without any +/- prefix parsing. + + - When specifying the USER@HOST: prefix for a file, the USER part can now + contain an '@', if needed (i.e. the last '@' is used to find the HOST, not + the first). + + - Fixed some bugs in the handling of group IDs for non-root users: (1) It + properly handles a group that the sender didn't have a name for (it would + previously skip changing the group on any files in that group). (2) If + `--numeric-ids` is used, rsync no longer attempts to set groups that the + user doesn't have the permission to set. + + - Fixed the `refuse options` setting in the rsyncd.conf file. + + - Improved the `-x` (`--one-file-system`) flag's handling of any mount- point + directories we encounter. It is both more optimal (in that it no longer does + a useless scan of the contents of the mount- point dirs) and also fixes a + bug where a remapped mount of the original filesystem could get discovered + in a subdir we should be ignoring. + + - Rsync no longer discards a double-slash at the start of a filename when + trying to open the file. It also no longer constructs names that start with + a double slash (unless the user supplied them). + + - Path-specifying options to a daemon should now work the same with or without + chroot turned on. Previously, such a option (such as `--link-dest`) would + get its absolute path munged into a relative one if chroot was not on, + making that setting fairly useless. Rsync now transforms the path into one + that is based on the module's base dir when chroot is not enabled. + + - Fixed a compatibility problem interacting with older rsync versions that + might send us an empty `--suffix` value without telling us that + `--backup-dir` was specified. + + - The `hosts allow` option for a daemon-over-remote-shell process now has + improved support for IPv6 addresses and a fix for systems that have a length + field in their socket structs. + + - Fixed the ability to request an empty backup `--suffix` when sending files + to an rsync daemon. + + - Fixed an option-parsing bug when `--files-from` was sent to a server sender. + +### INTERNAL: + + - Most of the I/O is now buffered, which results in a pretty large speedup + when running under MS Windows. (Craig Barratt) + + - Optimizations to the name-handling/comparing code have made some significant + reductions in user-CPU time for large file sets. + + - Some cleanup of the variable types make the code more consistent. + + - Reduced memory requirements of hard link preservation. (J.W. Schultz) + + - Implemented a new algorithm for hard-link handling that speeds up the code + significantly. (J.W. Schultz and Wayne Davison) + + - The `--hard-link` option now uses the first existing file in the group of + linked files as the basis for the transfer. This prevents the sub-optimal + transfer of a file's data when a new hardlink is added on the sending side + and it sorts alphabetically earlier in the list than the files that are + already present on the receiving side. + + - Dropped support for protocol versions less than 20 (2.3.0 released 15 Mar + 1999) and activated warnings for protocols less than 25 (2.5.0 released 23 + Aug 2001). (Wayne Davison and J.W. Schultz, severally) + + - More optimal data transmission for `--hard-links` (protocol 28). + + - More optimal data transmission for `--checksum` (protocol 28). + + - Less memory is used when `--checksum` is specified. + + - Less memory is used in the file list (a per-file savings). + + - The generator is now better about not modifying the file list during the + transfer in order to avoid a copy-on-write memory bifurcation (on systems + where fork() uses shared memory). Previously, rsync's shared memory would + slowly become unshared, resulting in real memory usage nearly doubling on + the receiving side by the end of the transfer. Now, as long as permissions + are being preserved, the shared memory should remain that way for the entire + transfer. + + - Changed hardlink info and `file_struct` + strings to use allocation pools. + This reduces memory use for large file-sets and permits freeing memory to + the OS. (J.W. Schultz) + + - The 2 pipes used between the receiver and generator processes (which are + forked on the same machine) were reduced to 1 pipe and the protocol improved + so that (1) it is now impossible to have the `redo` pipe fill up and hang + rsync, and (2) trailing messages from the receiver don't get lost on their + way through the generator over to the sender (which mainly affected + hard-link messages and verbose `--stats` output). + + - Improved the internal uid/gid code to be more portable and a little more + optimized. + + - The device numbers sent when using `--devices` are now sent as separate + major/minor values with 32-bit accuracy (protocol 28). Previously, the + copied devices were sent as a single 32-bit number. This will make + inter-operation of 64-bit binaries more compatible with their 32-bit + brethren (with both ends of the connection are using protocol 28). Note that + optimizations in the binary protocol for sending the device numbers often + results in fewer bytes being used than before, even though more precision is + now available. + + - Some cleanup of the exclude/include structures and its code made things + clearer (internally), simpler, and more efficient. + + - The reading & writing of the file-list in batch-mode is now handled by the + same code that sends & receives the list over the wire. This makes it much + easier to maintain. (Note that the batch code is still considered to be + experimental.) + +### BUILD CHANGES: + + - The configure script now accepts `--with-rsyncd-conf=PATH` to override the + default value of the /etc/rsyncd.conf file. + + - Fixed configure bug when running `./configure --disable-ipv6`. + + - Fixed compilation problem on Tru64 Unix (having to do with `sockaddr.sa_len` + and `sockaddr.sin_len`). + +### DEVELOPER RELATED: + + - Fixed `make test` bug when build dir is not the source dir. + + - Added a couple extra diffs in the `patches` dir, removed the ones that got + applied, and rebuilt the rest. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.6.0 (1 Jan 2004) + +Protocol: 27 (changed) + +## Changes since 2.5.7: + +### ENHANCEMENTS: + + - `ssh` is now the default remote shell for rsync. If you want to change this, + configure like this: `./configure --with-rsh=rsh`. + + - Added `--files-from`, `--no-relative`, `--no-implied-dirs`, and `--from0`. + Note that `--from0` affects the line-ending character for all the files read + by the `--*-from` options. (Wayne Davison) + + - Length of csum2 is now per-file starting with protocol version + 27. (J.W. Schultz) + + - Per-file dynamic block size is now sqrt(file length). The per-file checksum + size is determined according to an algorithm provided by Donovan Baarda + which reduces the probability of rsync algorithm corrupting data and falling + back using the whole md4 checksums. (J.W. Schultz, Donovan Baarda) + + - The `--stats` option no longer includes the (debug) malloc summary unless + the verbose option was specified at least twice. + + - Added a new error/warning code for when files vanish from the sending side. + Made vanished source files not interfere with the file-deletion pass when + `--delete-after` was specified. + + - Various trailing-info sections are now preceded by a newline. + +### BUG FIXES: + + - Fixed several exclude/include matching bugs when using wild-cards. This has + a several user-visible effects, all of which make the matching more + consistent and intuitive. This should hopefully not cause anyone problems + since it makes the matching work more like what people are expecting. (Wayne + Davison) + + - A pattern with a `**` no longer causes a `*` to match slashes. For example, + with `/*/foo/**`, `foo` must be 2 levels deep. [If your string has BOTH `*` + and `**` wildcards, changing the `*` wildcards to `**` will provide the old + behavior in all versions.] + + - `**/foo` now matches at the base of the transfer (like /foo does). [Use + `/**/foo` to get the old behavior in all versions.] + + - A non-anchored wildcard term floats to match beyond the base of the + transfer. E.g. `CVS/R*` matches at the end of the path, just like the + non-wildcard term `CVS/Root` does. [Use `/CVS/R*` to get the old behavior in + all versions.] + + - Including a `**` in the match term causes it to be matched against the + entire path, not just the name portion, even if there aren't any interior + slashes in the term. E.g. `foo**bar` would exclude `/path/foo-bar` (just + like before) as well as `/foo-path/baz-bar` (unlike before). [Use `foo*bar` + to get the old behavior in all versions.] + + - The exclude list specified in the daemon's config file is now properly + applied to the pulled items no matter how deep the user's file-args are in + the source tree. (Wayne Davison) + + - For protocol version >= 27, `mdfour_tail()` is called when the block size + (including `checksum_seed`) is a multiple of 64. Previously it was not + called, giving the wrong MD4 checksum. (Craig Barratt) + + - For protocol version >= 27, a 64 bit bit counter is used in mdfour.c as + required by the RFC. Previously only a 32 bit bit counter was used, causing + incorrect MD4 file checksums for file sizes >= 512MB - 4. (Craig Barratt) + + - Fixed a crash bug when interacting with older rsync versions and multiple + files of the same name are destined for the same dir. (Wayne Davison) + + - Keep tmp names from overflowing MAXPATHLEN. + + - Make `--link-dest` honor the absence of `-p`, `-o`, and `-g`. + + - Made rsync treat a trailing slash in the destination in a more consistent + manner. + + - Fixed file I/O error detection. (John Van Essen) + + - Fixed bogus `malformed address {hostname}` message in rsyncd log when + checking IP address against hostnames from `hosts allow` and `hosts deny` + parameters in config file. + + - Print heap statistics when verbose >= 2 instead of when >= 1. + + - Fixed a compression (`-z`) bug when syncing a mostly-matching file that + contains already-compressed data. (Yasuoka Masahiko and Wayne Davison) + + - Fixed a bug in the `--backup` code that could cause deleted files to not get + backed up. + + - When the backup code makes new directories, create them with mode 0700 + instead of 0755 (since the directory permissions in the backup tree are not + yet copied from the main tree). + + - Call setgroups() in a more portable manner. + + - Improved file-related error messages to better indicate exactly what + pathname failed. (Wayne Davison) + + - Fixed some bugs in the handling of `--delete` and `--exclude` when using the + `--relative` (`-R`) option. (Wayne Davison) + + - Fixed bug that prevented regular files from replacing special files and + caused a directory in `--link-dest` or `--compare-dest` to block the + creation of a file with the same path. A directory still cannot be replaced + by a regular file unless `--delete` specified. (J.W. Schultz) + + - Detect and report when open or opendir succeed but read and readdir fail + caused by network filesystem issues and truncated files. (David Norwood, + Michael Brown, J.W. Schultz) + + - Added a fix that should give ssh time to restore the tty settings if the + user presses Ctrl-C at an ssh password prompt. + +### INTERNAL: + + - Eliminated vestigial support for old versions that we stopped supporting. + (J.W. Schultz) + + - Simplified some of the option-parsing code. (Wayne Davison) + + - Some cleanup made to the exclude code, as well as some new defines added to + enhance readability. (Wayne Davison) + + - Changed the protocol-version code so that it can interact at a lower + protocol level than the maximum supported by both sides. Added an + undocumented option, `--protocol=N`, to force the value we advertise to the + other side (primarily for testing purposes). (Wayne Davison) + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.7 (4 Dec 2003) + +Protocol: 26 (unchanged) + +## Changes since 2.5.6: + +### SECURITY FIXES: + + - Fix buffer handling bugs. (Andrew Tridgell, Martin Pool, Paul Russell, + Andrea Barisani) + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.6, aka "the dwd-between-jobs release" (26 Jan 2003) + +Protocol: 26 (unchanged) + +## Changes since 2.5.5: + +### ENHANCEMENTS: + + - The `--delete-after` option now implies `--delete`. (Wayne Davison) + + - The `--suffix` option can now be used with `--backup-dir`. (Michael + Zimmerman) + + - Combining `::` syntax with the `--rsh`/`-e` option now uses the specified + remote-shell as a transport to talk to a (newly-spawned) server-daemon. This + allows someone to use daemon features, such as modules, over a secure + protocol, such as ssh. (JD Paul) + + - The rsync:// syntax for daemon connections is now accepted in the + destination field. + + - If the file name given to `--include-from` or `--exclude-from` is `-`, rsync + will read from standard input. (J.W. Schultz) + + - New option `--link-dest` which is like `--compare-dest` except that + unchanged files are hard-linked in to the destination directory. (J.W. + Schultz) + + - Don't report an error if an excluded file disappears during an rsync run. + (Eugene Chupriyanov and Bo Kersey) + + - Added .svn to `--cvs-exclude` list to support subversion. (Jon Middleton) + + - Properly support IPv6 addresses in the rsyncd.conf `hosts allow` and `hosts + deny` fields. (Hideaki Yoshifuji) + + - Changed exclude file handling to permit DOS or MAC style line terminations. + (J.W. Schultz) + + - Ignore errors from chmod when `-p`/`-a`/`--preserve-perms` is not set. + (Dave Dykstra) + +### BUG FIXES: + + - Fix `forward name lookup failed` errors on AIX 4.3.3. (John L. Allen, Martin + Pool) + + - Generate each file's rolling-checksum data as we send it, not in a separate + (memory-eating) pass before hand. This prevents timeout errors on really + large files. (Stefan Nehlsen) + + - Fix compilation on Tru64. (Albert Chin, Zoong Pham) + + - Better handling of some client-server errors. (Martin Pool) + + - Fixed a crash that would occur when sending a list of files that contains a + duplicate name (if it sorts to the end of the file list) and using + `--delete`. (Wayne Davison) + + - Fixed the file-name duplicate-removal code when dealing with multiple dups + in a row. (Wayne Davison) + + - Fixed a bug that caused rsync to lose the exit status of its child processes + and sometimes return an exit code of 0 instead of showing an error. (David + R. Staples, Dave Dykstra) + + - Fixed bug in `--copy-unsafe-links` that caused it to be completely broken. + (Dave Dykstra) + + - Prevent infinite recursion in cleanup code under certain circumstances. + (Sviatoslav Sviridov and Marc Espie) + + - Fixed a bug that prevented rsync from creating intervening directories when + `--relative-paths`/`-R` is set. (Craig Barratt) + + - Prevent `Connection reset by peer` messages from Cygwin. (Randy O'Meara) + +### INTERNAL: + + - Many code cleanups and improved internal documentation. (Martin Pool, Nelson + Beebe) + + - Portability fixes. (Dave Dykstra and Wayne Davison) + + - More test cases. (Martin Pool) + + - Some test-case fixes. (Brian Poole, Wayne Davison) + + - Updated included popt to the latest vendor drop, version 1.6.4. (Jos + Backus) + + - Updated config.guess and config.sub to latest versions; this means rsync + should build on more platforms. (Paul Green) + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.5, aka Snowy River (2 Apr 2002) + +Protocol: 26 (unchanged) + +## Changes since 2.5.4: + +### ENHANCEMENTS: + + - With `--progress`, when a transfer is complete show the time taken; + otherwise show expected time to complete. (Cameron Simpson) + + - Make `make install-strip` works properly, and `make install` accepts a + DESTDIR variable for help in building binary packages. (Peter + Breitenlohner, Greg Louis) + + - If configured with `--enable-maintainer-mode`, then on receipt of a fatal + signal rsync will try to open an xterm running gdb, similarly to Samba's + `panic action` or GNOME's bug-buddy. (Martin Pool) + +### BUG FIXES: + + - Fix situation where failure to fork (e.g. because out of process slots) + would cause rsync to kill all processes owned by the current user. Yes, + really! (Paul Haas, Martin Pool) + + - Fix test suite on Solaris. (Jos Backus, Martin Pool) + + - Fix minor memory leak in socket code. (Dave Dykstra, Martin Pool.) + + - Fix `--whole-file` problem that caused it to be the default even for remote + connections. (Martin Pool, Frank Schulz) + + - Work around bug in Mac OS X mkdir(2), which cannot handle trailing slashes. + <http://www.opensource.apple.com/bugs/X/BSD%20Kernel/2734739.html> (Martin + Pool) + + - Improved network error handling. (Greg A. Woods) + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.4, aka "Imitation lizard skin" (13 Mar 2002) + +Protocol: 26 (unchanged) + +## Changes since 2.5.3: + +### BUG FIXES: + + - Additional fix for zlib double-free bug. (Martin Pool, Andrew Tridgell) (CVE + CAN-2002-0059) + +### ENHANCEMENTS: + + - Merge in changes from zlib 1.1.3 to zlib 1.1.4. (Jos Backus) (Note that + rsync still uses a custom version of zlib; you can not just link against a + system library. See zlib/README.rsync) + + - Additional test cases for `--compress`. (Martin Pool) + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.3, aka "Happy 26" (11 Mar 2002) + +Protocol: 26 (unchanged) + +## Changes since 2.5.2: + +### SECURITY FIXES: + + - Make sure that supplementary groups are removed from a server + process after changing uid and gid. (Ethan Benson) (Debian bug + #132272, CVE CAN-2002-0080) + +### BUG FIXES: + + - Fix zlib double-free bug. (Owen Taylor, Mark J Cox) (CVE CAN-2002-0059) + + - Fixed problem that in many cases caused the error message unexpected read + size of 0 in `map_ptr` and resulted in the wrong data being copied. + + - Fixed compilation errors on some systems caused by the use of `unsigned + int64` in rsync.h. + + - Fixed problem on systems such as Sunos4 that do not support realloc on a + NULL pointer; error was 'out of memory in "flist_expand"'. + + - Fix for rsync server processes hanging around after the client unexpectedly + disconnects. (Colin Walters) (Debian bug #128632) + + - Cope with BSD systems on which mkdir() will not accept a trailing slash. + +### ENHANCEMENTS: + + - Merge in changes from zlib 1.1.2 to zlib 1.1.3. (Note that rsync still uses + a custom version of zlib; you can not just link against a system library. + See zlib/README.rsync) + + - Command to initiate connections is only shown with `-vv`, rather than `-v` + as in 2.5.2. Output from plain `-v` is more similar to what was historically + used so as not to break scripts that try to parse the output. + + - Added `--no-whole-file` and `--no-blocking-io` options (Dave Dykstra) + + - Made the `--write-batch` and `--read-batch` options actually work and added + documentation in the man page (Jos Backus) + + - If the daemon is unable to fork a child to accept a connection, print an + error message. (Colin Walters) + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.2 (26 Jan 2002) + +Protocol: 26 (changed) + +## Changes since 2.5.1: + +### SECURITY FIXES: + + - Signedness security patch from Sebastian Krahmer <krahmer@suse.de> -- in + some cases we were not sufficiently careful about reading integers from the + network. + +### BUG FIXES: + + - Fix possible string mangling in log files. + + - Fix for setting local address of outgoing sockets. + + - Better handling of hardlinks and devices on platforms with 64-bit `dev_t` or + `ino_t`. + + - Name resolution on machines supporting IPv6 is improved. + + - Fix for device nodes. (dann frazier) (Debian #129135) + +### ENHANCEMENTS: + + - With `-v`, rsync now shows the command used to initiate an ssh/rsh + connection. + + - `--statistics` now shows memory heap usage on platforms that support + mallinfo(). + + - "The Ted T'so school of program optimization": make progress visible and + people will think it's faster. (With `--progress`, rsync will show you how + many files it has seen as it builds the `file_list`, giving some indication + that it has not hung.) + + - Improvements to batch mode support. This is still experimental but testing + would be welcome. (Jos Backus) + + - New `--ignore-existing` option, patch previously distributed with Vipul's + Razor. (Debian #124286) + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.1 (3 Jan 2002) + +Protocol: 25 (unchanged) + +## Changes since 2.5.0: + +### BUG FIXES: + + - Fix for segfault in `--daemon` mode configuration parser. (Paul Mackerras) + + - Correct `string<->address` parsing for both IPv4 and 6. (YOSHIFUJI Hideaki, + SUMIKAWA Munechika and Jun-ichiro `itojun` Hagino) + + - Various fixes for IPv6 support. (Dave Dykstra) + + - rsync.1 typo fix. (Matt Kraai) + + - Test suite typo fixes. (Tom Schmidt) + + - rsync.1 grammar and clarity improvements. (Edward Welbourne) + + - Correction to ./configure tests for `inet_ntop`. (Jeff Garzik) + +### ENHANCEMENTS: + + - `--progress` and `-P` now show estimated data transfer rate (in a multiple + of bytes/s) and estimated time to completion. (Rik Faith) + + - `--no-detach` option, required to run as a W32 service and also useful when + running on Unix under daemontools, AIX's SRC, or a debugger. (Max Bowsher, + Jos Backus) + + - Clearer error messages for some conditions. + +------------------------------------------------------------------------------ + +# NEWS for rsync 2.5.0 (30 Nov 2001) + +Protocol: 25 (changed) + +## Changes since 2.4.6: + +### ANNOUNCEMENTS: + + - Martin Pool <mbp@samba.org> is now a co-maintainer. + +### NEW FEATURES: + + - Support for LSB-compliant packaging <http://www.linuxbase.org/> + + - Shell wildcards are allowed in `auth users` lines. + + - Merged UNC rsync+ patch to support creation of standalone patch sets. By + Bert J. Dempsey and Debra Weiss, updated by Jos Backus. + <http://www.ils.unc.edu/i2dsi/unc_rsync+.html> + + - IPv6 support based on a patch from KAME.net, on systems including modern + versions of Linux, Solaris, and HP-UX. Also includes IPv6 compatibility + functions for old OSs by the Internet Software Consortium, Paul Vixie, the + OpenSSH portability project, and OpenBSD. + +### ENHANCEMENTS: + + - Include/exclude cluestick: with `-vv`, print out whether files are included + or excluded and why. + + - Many error messages have more friendly explanations and more details. + + - Manual page improvements plus scanty protocol documentation. + + - When running as `--daemon` in the background and using a `log file` + rsyncd.conf directive, close the log file every time it is open when going + to sleep on the socket. This allows the log file to get cleaned out by + another process. + + - Change to using libpopt rather than getopt for processing options. This + makes the code cleaner and the behaviour more consistent across platforms. + popt is included and built if not installed on the platform. + + - More details in `--version`, including note about whether 64-bit files, + symlinks and hardlinks are supported. + + - MD4 code may use less CPU cycles. + + - Use mkstemp on systems where it is secure. If we use mktemp, explain that we + do it in a secure way. + + - `--whole-file` is the default when source and target are on the local + machine. + +### BUG FIXES: + + - Fix for various bugs causing rsync to hang. + + - Attempt to fix Large File Summit support on AIX. + + - Attempt to fix error handling lockup bug. + + - Give a non-0 exit code if **any** of the files we have been asked to + transfer fail to transfer. + + - For log messages containing ridiculously long strings that might overflow a + buffer rsync no longer aborts, but rather prints an ellipsis at the end of + the string. (Patch from Ed Santiago.) + +### PLATFORMS: + + - Improved support for UNICOS (tested on Cray T3E and Cray SV1) + + - autoconf2.52 (or later) is now required to rebuild the autoconf scripts. It + is not required to simply build rsync. + + - Platforms thought to work in this release: + + - Cray SV1 UNICOS 10.0.0.8 cc + - Debian Linux 2.2 UltraSparc gcc + - Debian Linux testing/unstable ARM gcc + - FreeBSD 3.3-RELEASE i386 cc + - FreeBSD 4.1.1-RELEASE i386 cc + - FreeBSD 4.3-STABLE i386 cc + - HP PA-RISC HP-UX 10.20 gcc + - HP PA-RISC HP-UX 11.11 cc + - IRIX 6.5 MIPS cc + - IRIX 6.5 MIPS gcc + - Mac OS X PPC (--disable-ipv6) cc + - NetBSD 1.5 i386 gcc + - NetBSD Current i386 cc + - OpenBSD 2.5 Sparc gcc + - OpenBSD 2.9 i386 cc + - OpenBSD Current i386 cc + - RedHat 6.2 i386 gcc + - RedHat 6.2 i386 insure++ + - RedHat 7.0 i386 gcc + - RedHat 7.1 i386 (Kernel 2.4.10) gcc + - Slackware 8.0 i686 (Kernel 2.4.10) + - Solaris 8 UltraSparc cc + - Solaris 8 UltraSparc gcc + - Solaris 8 i386 gcc + - SuSE 7.1 i386 gcc2.95.2 + - SuSE 7.1 ppc gcc2.95.2 + - i386-pc-sco3.2v5.0.5 cc + - i386-pc-sco3.2v5.0.5 gcc + - powerpc-ibm-aix4.3.3.0 cc + - i686-unknown-sysv5UnixWare7.1.0 gcc + - i686-unknown-sysv5UnixWare7.1.0 cc + +### TESTING: + + - The existing test.sh script by Phil Hands has been merged into a + test framework that works from both `make check` and the Samba + build farm. + +------------------------------------------------------------------------------ + +## Partial Protocol History + +| RELEASE DATE | VER. | DATE OF COMMIT\* | PROTOCOL | +|--------------|--------|------------------|-------------| +| 19 Jun 2020 | 3.2.0 | | 31 | +| 28 Jan 2018 | 3.1.3 | | 31 | +| 21 Dec 2015 | 3.1.2 | | 31 | +| 22 Jun 2014 | 3.1.1 | | 31 | +| 28 Sep 2013 | 3.1.0 | 31 Aug 2008 | 31 | +| 23 Sep 2011 | 3.0.9 | | 30 | +| 26 Mar 2011 | 3.0.8 | | 30 | +| 31 Dec 2009 | 3.0.7 | | 30 | +| 08 May 2009 | 3.0.6 | | 30 | +| 28 Dec 2008 | 3.0.5 | | 30 | +| 06 Sep 2008 | 3.0.4 | | 30 | +| 29 Jun 2008 | 3.0.3 | | 30 | +| 08 Apr 2008 | 3.0.2 | | 30 | +| 03 Apr 2008 | 3.0.1 | | 30 | +| 01 Mar 2008 | 3.0.0 | 11 Nov 2006 | 30 | +| 06 Nov 2006 | 2.6.9 | | 29 | +| 22 Apr 2006 | 2.6.8 | | 29 | +| 11 Mar 2006 | 2.6.7 | | 29 | +| 28 Jul 2005 | 2.6.6 | | 29 | +| 01 Jun 2005 | 2.6.5 | | 29 | +| 30 Mar 2005 | 2.6.4 | 17 Jan 2005 | 29 | +| 30 Sep 2004 | 2.6.3 | | 28 | +| 30 Apr 2004 | 2.6.2 | | 28 | +| 26 Apr 2004 | 2.6.1 | 08 Jan 2004 | 28 | +| 01 Jan 2004 | 2.6.0 | 10 Apr 2003 | 27 (MAX=40) | +| 04 Dec 2003 | 2.5.7 | | 26 | +| 26 Jan 2003 | 2.5.6 | | 26 | +| 02 Apr 2002 | 2.5.5 | | 26 | +| 13 Mar 2002 | 2.5.4 | | 26 | +| 11 Mar 2002 | 2.5.3 | | 26 | +| 26 Jan 2002 | 2.5.2 | 11 Jan 2002 | 26 | +| 03 Jan 2002 | 2.5.1 | | 25 | +| 30 Nov 2001 | 2.5.0 | 23 Aug 2001 | 25 | +| 06 Sep 2000 | 2.4.6 | | 24 | +| 19 Aug 2000 | 2.4.5 | | 24 | +| 29 Jul 2000 | 2.4.4 | | 24 | +| 09 Apr 2000 | 2.4.3 | | 24 | +| 30 Mar 2000 | 2.4.2 | | 24 | +| 30 Jan 2000 | 2.4.1 | 29 Jan 2000 | 24 | +| 29 Jan 2000 | 2.4.0 | 28 Jan 2000 | 23 | +| 25 Jan 2000 | 2.3.3 | 23 Jan 2000 | 22 | +| 08 Nov 1999 | 2.3.2 | 26 Jun 1999 | 21 | +| 06 Apr 1999 | 2.3.1 | | 20 | +| 15 Mar 1999 | 2.3.0 | 15 Mar 1999 | 20 | +| 25 Nov 1998 | 2.2.1 | | 19 | +| 03 Nov 1998 | 2.2.0 | | 19 | +| 09 Sep 1998 | 2.1.1 | | 19 | +| 20 Jul 1998 | 2.1.0 | | 19 | +| 17 Jul 1998 | 2.0.19 | | 19 | +| 18 Jun 1998 | 2.0.17 | | 19 | +| 01 Jun 1998 | 2.0.16 | | 19 | +| 27 May 1998 | 2.0.13 | 27 May 1998 | 19 | +| 26 May 1998 | 2.0.12 | | 18 | +| 22 May 1998 | 2.0.11 | | 18 | +| 18 May 1998 | 2.0.9 | 18 May 1998 | 18 | +| 17 May 1998 | 2.0.8 | | 17 | +| 15 May 1998 | 2.0.1 | | 17 | +| 14 May 1998 | 2.0.0 | | 17 | +| 17 Apr 1998 | 1.7.4 | | 17 | +| 13 Apr 1998 | 1.7.3 | | 17 | +| 05 Apr 1998 | 1.7.2 | | 17 | +| 26 Mar 1998 | 1.7.1 | | 17 | +| 26 Mar 1998 | 1.7.0 | 26 Mar 1998 | 17 (MAX=30) | +| 13 Jan 1998 | 1.6.9 | 13 Jan 1998 | 15 (MAX=20) | + +\* DATE OF COMMIT is the date the protocol change was committed to version +control. @@ -65,7 +65,7 @@ WEB SITE The main rsync web site is here: - http://rsync.samba.org/ +> http://rsync.samba.org/ You'll find a FAQ list, downloads, resources, HTML versions of the manpages, etc. @@ -80,7 +80,7 @@ list, and there is also an announcement-only mailing list for those that want official announcements. See the mailing-list page for full details: - http://rsync.samba.org/lists.html +> http://rsync.samba.org/lists.html BUG REPORTS @@ -88,34 +88,38 @@ BUG REPORTS To visit this web page for full the details on bug reporting: - http://rsync.samba.org/bugzilla.html +> http://rsync.samba.org/bugzilla.html That page contains links to the current bug list, and information on how to report a bug well. You might also like to try searching the Internet for the error message you've received, or looking in the mailing list archives at: - http://mail-archive.com/rsync@lists.samba.org/ +> http://mail-archive.com/rsync@lists.samba.org/ To send a bug report, follow the instructions on the bug-tracking page of the web site. -Alternately, email your bug report to rsync@lists.samba.org . +Alternately, email your bug report to <rsync@lists.samba.org>. GIT REPOSITORY -------------- If you want to get the very latest version of rsync direct from the -source code repository then you can use git: +source code repository, then you will need to use git. The git repo +is hosted on github and on samba's site. Feel free to access it here: - git clone git://git.samba.org/rsync.git +> https://github.com/WayneD/rsync + +or clone it from its samba repo: + +> git clone git://git.samba.org/rsync.git See the download page for full details on all the ways to grab the -source, including nightly tar files, web-browsing of the git repository, -etc.: +source: - http://rsync.samba.org/download.html +> http://rsync.samba.org/download.html COPYRIGHT @@ -129,12 +133,17 @@ Rsync may be used, modified and redistributed only under the terms of the GNU General Public License, found in the file COPYING in this distribution, or at: - http://www.fsf.org/licenses/gpl.html +> http://www.fsf.org/licenses/gpl.html AVAILABILITY ------------ The main web site for rsync is http://rsync.samba.org/ + The main ftp site is ftp://rsync.samba.org/pub/rsync/ -This is also available as rsync://rsync.samba.org/rsyncftp/ + +This is also available as rsync://download.samba.org/rsyncftp/ if you +connect via ssl. Use the `rsync-ssl` script if you have it, otherwise +connect to the rsync server via a normal rsync command and it will +output some instructions for how to connect. @@ -94,7 +94,7 @@ Handling IPv6 on old machines platforms that have a half-working implementation, so redefining these functions clashes with system headers, and leaving them out breaks. This affects at least OSF/1, RedHat 5, and Cobalt, which - are moderately improtant. + are moderately important. Perhaps the simplest solution would be to have two different files implementing the same interface, and choose either the new or the @@ -236,7 +236,7 @@ Memory accounting At exit, show how much memory was used for the file list, etc. - Also we do a wierd exponential-growth allocation in flist.c. I'm + We also do a weird exponential-growth allocation in flist.c. I'm not sure this makes sense with modern mallocs. At any rate it will make us allocate a huge amount of memory for large file lists. @@ -2,7 +2,7 @@ * Routines to authenticate access to a daemon (hosts allow/deny). * * Copyright (C) 1998 Andrew Tridgell - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -158,8 +158,7 @@ static int match_address(const char *addr, const char *tok) break; #ifdef INET6 - case PF_INET6: - { + case PF_INET6: { struct sockaddr_in6 *sin6a, *sin6t; sin6a = (struct sockaddr_in6 *)resa->ai_addr; @@ -171,20 +170,19 @@ static int match_address(const char *addr, const char *tok) addrlen = 16; #ifdef HAVE_SOCKADDR_IN6_SCOPE_ID - if (sin6t->sin6_scope_id && - sin6a->sin6_scope_id != sin6t->sin6_scope_id) { + if (sin6t->sin6_scope_id && sin6a->sin6_scope_id != sin6t->sin6_scope_id) { ret = 0; goto out; } #endif break; - } + } #endif default: - rprintf(FLOG, "unknown family %u\n", rest->ai_family); - ret = 0; - goto out; + rprintf(FLOG, "unknown family %u\n", rest->ai_family); + ret = 0; + goto out; } bits = -1; @@ -1,6 +1,6 @@ -# generated automatically by aclocal 1.14.1 -*- Autoconf -*- +# generated automatically by aclocal 1.16.1 -*- Autoconf -*- -# Copyright (C) 1996-2013 Free Software Foundation, Inc. +# Copyright (C) 1996-2018 Free Software Foundation, Inc. # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -13,4 +13,5 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) m4_include([m4/have_type.m4]) +m4_include([m4/header_major_fixed.m4]) m4_include([m4/socklen_t.m4]) @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2006-2018 Wayne Davison + * Copyright (C) 2006-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -48,7 +48,7 @@ extern int preserve_specials; /* When we send the access bits over the wire, we shift them 2 bits to the * left and use the lower 2 bits as flags (relevant only to a name entry). * This makes the protocol more efficient than sending a value that would - * be likely to have its hightest bits set. */ + * be likely to have its highest bits set. */ #define XFLAG_NAME_FOLLOWS 0x0001u #define XFLAG_NAME_IS_USER 0x0002u @@ -332,14 +332,12 @@ static BOOL unpack_smb_acl(SMB_ACL_T sacl, rsync_acl *racl) if (temp_ida_list.count) { #ifdef SMB_ACL_NEED_SORT if (temp_ida_list.count > 1) { - qsort(temp_ida_list.items, temp_ida_list.count, - sizeof (id_access), id_access_sorter); + qsort(temp_ida_list.items, temp_ida_list.count, sizeof (id_access), id_access_sorter); } #endif if (!(racl->names.idas = new_array(id_access, temp_ida_list.count))) out_of_memory("unpack_smb_acl"); - memcpy(racl->names.idas, temp_ida_list.items, - temp_ida_list.count * sizeof (id_access)); + memcpy(racl->names.idas, temp_ida_list.items, temp_ida_list.count * sizeof (id_access)); } else racl->names.idas = NULL; @@ -825,14 +823,12 @@ void cache_tmp_acl(struct file_struct *file, stat_x *sxp) if (prior_access_count == (size_t)-1) prior_access_count = access_acl_list.count; - F_ACL(file) = cache_rsync_acl(sxp->acc_acl, - SMB_ACL_TYPE_ACCESS, &access_acl_list); + F_ACL(file) = cache_rsync_acl(sxp->acc_acl, SMB_ACL_TYPE_ACCESS, &access_acl_list); if (S_ISDIR(sxp->st.st_mode)) { if (prior_default_count == (size_t)-1) prior_default_count = default_acl_list.count; - F_DIR_DEFACL(file) = cache_rsync_acl(sxp->def_acl, - SMB_ACL_TYPE_DEFAULT, &default_acl_list); + F_DIR_DEFACL(file) = cache_rsync_acl(sxp->def_acl, SMB_ACL_TYPE_DEFAULT, &default_acl_list); } } @@ -996,8 +992,7 @@ static int set_rsync_acl(const char *fname, acl_duo *duo_item, mode = 0; /* eliminate compiler warning */ #else if (type == SMB_ACL_TYPE_ACCESS) { - cur_mode = change_sacl_perms(duo_item->sacl, &duo_item->racl, - cur_mode, mode); + cur_mode = change_sacl_perms(duo_item->sacl, &duo_item->racl, cur_mode, mode); if (cur_mode == (mode_t)-1) return 0; } @@ -1117,14 +1112,12 @@ int default_perms_for_dir(const char *dir) case ENOSYS: /* No ACLs are available. */ break; - case ENOENT: - if (dry_run) { + default: + if (dry_run && errno == ENOENT) { /* We're doing a dry run, so the containing directory * wasn't actually created. Don't worry about it. */ break; } - /* Otherwise fall through. */ - default: rprintf(FWARNING, "default_perms_for_dir: sys_acl_get_file(%s, %s): %s, falling back on umask\n", dir, str_acl_type(SMB_ACL_TYPE_DEFAULT), strerror(errno)); diff --git a/authenticate.c b/authenticate.c index eca5baf1..169331e5 100644 --- a/authenticate.c +++ b/authenticate.c @@ -2,7 +2,7 @@ * Support rsync daemon authentication. * * Copyright (C) 1998-2000 Andrew Tridgell - * Copyright (C) 2002-2018 Wayne Davison + * Copyright (C) 2002-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -162,8 +162,8 @@ static const char *check_secret(int module, const char *user, const char *group, fclose(fh); - memset(line, 0, sizeof line); - memset(pass2, 0, sizeof pass2); + force_memzero(line, sizeof line); + force_memzero(pass2, sizeof pass2); return err; } @@ -318,8 +318,8 @@ char *auth_server(int f_in, int f_out, int module, const char *host, err = check_secret(module, line, group, challenge, pass); } - memset(challenge, 0, sizeof challenge); - memset(pass, 0, strlen(pass)); + force_memzero(challenge, sizeof challenge); + force_memzero(pass, strlen(pass)); if (auth_uid_groups) { int j; @@ -357,12 +357,12 @@ void auth_client(int fd, const char *user, const char *challenge) /* XXX: cyeoh says that getpass is deprecated, because * it may return a truncated password on some systems, * and it is not in the LSB. - * - * Andrew Klein says that getpassphrase() is present - * on Solaris and reads up to 256 characters. - * - * OpenBSD has a readpassphrase() that might be more suitable. - */ + * + * Andrew Klein says that getpassphrase() is present + * on Solaris and reads up to 256 characters. + * + * OpenBSD has a readpassphrase() that might be more suitable. + */ pass = getpass("Password: "); } @@ -2,7 +2,7 @@ * Backup handling code. * * Copyright (C) 1999 Andrew Tridgell - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -336,7 +336,7 @@ int make_backup(const char *fname, BOOL prefer_rename) save_preserve_xattrs = preserve_xattrs; preserve_xattrs = 0; - set_file_attrs(buf, file, NULL, fname, ATTRS_SET_NANO); + set_file_attrs(buf, file, NULL, fname, ATTRS_ACCURATE_TIME); preserve_xattrs = save_preserve_xattrs; unmake_file(file); @@ -3,7 +3,7 @@ * * Copyright (C) 1999 Weiss * Copyright (C) 2004 Chris Shoemaker - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -37,14 +37,22 @@ extern int always_checksum; extern int do_compression; extern int inplace; extern int append_mode; +extern int write_batch; +extern int xfersum_type; extern int protocol_version; +extern int raw_argc, cooked_argc; +extern char **raw_argv, **cooked_argv; extern char *batch_name; +extern const char *checksum_choice; +extern const char *compress_choice; #ifdef ICONV_OPTION extern char *iconv_opt; #endif extern filter_rule_list filter_list; +int batch_fd = -1; +int batch_sh_fd = -1; int batch_stream_flags; static int tweaked_append; @@ -156,37 +164,45 @@ void check_batch_flags(void) append_mode = 2; } -static int write_arg(int fd, char *arg) +static int write_arg(const char *arg) { - char *x, *s; - int len, ret = 0; + const char *x, *s; + int len, err = 0; if (*arg == '-' && (x = strchr(arg, '=')) != NULL) { - if (write(fd, arg, x - arg + 1) != x - arg + 1) - ret = -1; + err |= write(batch_sh_fd, arg, x - arg + 1) != x - arg + 1; arg += x - arg + 1; } if (strpbrk(arg, " \"'&;|[]()$#!*?^\\") != NULL) { - if (write(fd, "'", 1) != 1) - ret = -1; + err |= write(batch_sh_fd, "'", 1) != 1; for (s = arg; (x = strchr(s, '\'')) != NULL; s = x + 1) { - if (write(fd, s, x - s + 1) != x - s + 1 - || write(fd, "'", 1) != 1) - ret = -1; + err |= write(batch_sh_fd, s, x - s + 1) != x - s + 1; + err |= write(batch_sh_fd, "'", 1) != 1; } len = strlen(s); - if (write(fd, s, len) != len - || write(fd, "'", 1) != 1) - ret = -1; - return ret; + err |= write(batch_sh_fd, s, len) != len; + err |= write(batch_sh_fd, "'", 1) != 1; + return err; } len = strlen(arg); - if (write(fd, arg, len) != len) - ret = -1; + err |= write(batch_sh_fd, arg, len) != len; - return ret; + return err; +} + +/* Writes out a space and then an option (or other string) with an optional "=" + arg suffix. */ +static int write_opt(const char *opt, const char *arg) +{ + int len = strlen(opt); + int err = write(batch_sh_fd, " ", 1) != 1; + err = write(batch_sh_fd, opt, len) != len ? 1 : 0; + if (arg) { + err |= write(batch_sh_fd, "=", 1) != 1; + err |= write_arg(arg); + } + return err; } static void write_filter_rules(int fd) @@ -208,41 +224,74 @@ static void write_filter_rules(int fd) write_sbuf(fd, "#E#"); } +/* This sets batch_fd and (for --write-batch) batch_sh_fd. */ +void open_batch_files(void) +{ + if (write_batch) { + char filename[MAXPATHLEN]; + + stringjoin(filename, sizeof filename, batch_name, ".sh", NULL); + + batch_sh_fd = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IXUSR); + if (batch_sh_fd < 0) { + rsyserr(FERROR, errno, "Batch file %s open error", full_fname(filename)); + exit_cleanup(RERR_FILESELECT); + } + + batch_fd = do_open(batch_name, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + } else if (strcmp(batch_name, "-") == 0) + batch_fd = STDIN_FILENO; + else + batch_fd = do_open(batch_name, O_RDONLY, S_IRUSR | S_IWUSR); + + if (batch_fd < 0) { + rsyserr(FERROR, errno, "Batch file %s open error", full_fname(batch_name)); + exit_cleanup(RERR_FILEIO); + } +} + /* This routine tries to write out an equivalent --read-batch command * given the user's --write-batch args. However, it doesn't really * understand most of the options, so it uses some overly simple * heuristics to munge the command line into something that will * (hopefully) work. */ -void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt) +void write_batch_shell_file(void) { - int fd, i, len, err = 0; - char *p, filename[MAXPATHLEN]; - - stringjoin(filename, sizeof filename, - batch_name, ".sh", NULL); - fd = do_open(filename, O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR | S_IXUSR); - if (fd < 0) { - rsyserr(FERROR, errno, "Batch file %s open error", - filename); - exit_cleanup(RERR_FILESELECT); - } + int i, j, len, err = 0; + char *p, *p2; /* Write argvs info to BATCH.sh file */ - if (write_arg(fd, argv[0]) < 0) - err = 1; + err |= write_arg(raw_argv[0]); if (filter_list.head) { if (protocol_version >= 29) - write_sbuf(fd, " --filter=._-"); + err |= write_opt("--filter", "._-"); else - write_sbuf(fd, " --exclude-from=-"); + err |= write_opt("--exclude-from", "-"); } - for (i = 1; i < argc - file_arg_cnt; i++) { - p = argv[i]; + + /* We need to make sure that any protocol-based or negotiated choices get accurately + * reflected in the options we save AND that we avoid any need for --read-batch to + * do a string-based negotiation (since we don't write them into the file). */ + if (do_compression) + err |= write_opt("--compress-choice", compress_choice); + if (strchr(checksum_choice, ',') || xfersum_type != parse_csum_name(NULL, -1)) + err |= write_opt("--checksum-choice", checksum_choice); + + /* Elide the filename args from the option list, but scan for them in reverse. */ + for (i = raw_argc-1, j = cooked_argc-1; i > 0 && j >= 0; i--) { + if (strcmp(raw_argv[i], cooked_argv[j]) == 0) { + raw_argv[i] = NULL; + j--; + } + } + + for (i = 1; i < raw_argc; i++) { + if (!(p = raw_argv[i])) + continue; if (strncmp(p, "--files-from", 12) == 0 - || strncmp(p, "--filter", 8) == 0 - || strncmp(p, "--include", 9) == 0 - || strncmp(p, "--exclude", 9) == 0) { + || strncmp(p, "--filter", 8) == 0 + || strncmp(p, "--include", 9) == 0 + || strncmp(p, "--exclude", 9) == 0) { if (strchr(p, '=') == NULL) i++; continue; @@ -251,33 +300,24 @@ void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt) i++; continue; } - if (write(fd, " ", 1) != 1) - err = 1; if (strncmp(p, "--write-batch", len = 13) == 0 - || strncmp(p, "--only-write-batch", len = 18) == 0) { - if (write(fd, "--read-batch", 12) != 12) - err = 1; - if (p[len] == '=') { - if (write(fd, "=", 1) != 1 - || write_arg(fd, p + len + 1) < 0) - err = 1; - } - } else { - if (write_arg(fd, p) < 0) - err = 1; + || strncmp(p, "--only-write-batch", len = 18) == 0) + err |= write_opt("--read-batch", p[len] == '=' ? p + len + 1 : NULL); + else { + err |= write(batch_sh_fd, " ", 1) != 1; + err |= write_arg(p); } } - if (!(p = check_for_hostspec(argv[argc - 1], &p, &i))) - p = argv[argc - 1]; - if (write(fd, " ${1:-", 6) != 6 - || write_arg(fd, p) < 0) - err = 1; - write_byte(fd, '}'); + if (!(p = check_for_hostspec(cooked_argv[cooked_argc - 1], &p2, &i))) + p = cooked_argv[cooked_argc - 1]; + err |= write_opt("${1:-", NULL); + err |= write_arg(p); + err |= write(batch_sh_fd, "}", 1) != 1; if (filter_list.head) - write_filter_rules(fd); - if (write(fd, "\n", 1) != 1 || close(fd) < 0 || err) { - rsyserr(FERROR, errno, "Batch file %s write error", - filename); + write_filter_rules(batch_sh_fd); + if (write(batch_sh_fd, "\n", 1) != 1 || close(batch_sh_fd) < 0 || err) { + rsyserr(FERROR, errno, "Batch file %s.sh write error", batch_name); exit_cleanup(RERR_FILEIO); } + batch_sh_fd = -1; } diff --git a/byteorder.h b/byteorder.h index ac9ee419..525eaba0 100644 --- a/byteorder.h +++ b/byteorder.h @@ -2,7 +2,7 @@ * Simple byteorder handling. * * Copyright (C) 1992-1995 Andrew Tridgell - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,7 +19,6 @@ */ #undef CAREFUL_ALIGNMENT -#undef AVOID_BYTEORDER_INLINE /* We know that the x86 can handle misalignment and has the same * byte order (LSB-first) as the 32-bit numbers we transmit. */ @@ -36,16 +35,36 @@ #if CAREFUL_ALIGNMENT -#define PVAL(buf,pos) (UVAL(buf,pos)|UVAL(buf,(pos)+1)<<8) -#define IVAL(buf,pos) (PVAL(buf,pos)|PVAL(buf,(pos)+2)<<16) -#define IVAL64(buf,pos) (IVAL(buf,pos)|(int64)IVAL(buf,(pos)+4)<<32) -#define SSVALX(buf,pos,val) (CVAL(buf,pos)=(val)&0xFF,CVAL(buf,pos+1)=(val)>>8) -#define SIVALX(buf,pos,val) (SSVALX(buf,pos,val&0xFFFF),SSVALX(buf,pos+2,val>>16)) -#define SIVAL(buf,pos,val) SIVALX(buf,pos,(uint32)(val)) -#define SIVAL64(buf,pos,val) (SIVAL(buf,pos,val),SIVAL(buf,(pos)+4,(val)>>32)) +static inline uint32 +IVALu(const uchar *buf, int pos) +{ + return UVAL(buf, pos) + | UVAL(buf, pos + 1) << 8 + | UVAL(buf, pos + 2) << 16 + | UVAL(buf, pos + 3) << 24; +} -#define IVALu(buf,pos) IVAL(buf,pos) -#define SIVALu(buf,pos,val) SIVAL(buf,pos,val) +static inline void +SIVALu(uchar *buf, int pos, uint32 val) +{ + CVAL(buf, pos) = val; + CVAL(buf, pos + 1) = val >> 8; + CVAL(buf, pos + 2) = val >> 16; + CVAL(buf, pos + 3) = val >> 24; +} + +static inline int64 +IVAL64(const char *buf, int pos) +{ + return IVALu((uchar*)buf, pos) | (int64)IVALu((uchar*)buf, pos + 4) << 32; +} + +static inline void +SIVAL64(char *buf, int pos, int64 val) +{ + SIVALu((uchar*)buf, pos, val); + SIVALu((uchar*)buf, pos + 4, val >> 32); +} #else /* !CAREFUL_ALIGNMENT */ @@ -53,16 +72,6 @@ * WARNING: This section is dependent on the length of an int32 (and thus a uint32) * being correct (4 bytes)! Set CAREFUL_ALIGNMENT if it is not. */ -# ifdef AVOID_BYTEORDER_INLINE - -#define IVAL(buf,pos) (*(uint32 *)((char *)(buf) + (pos))) -#define SIVAL(buf,pos,val) IVAL(buf,pos)=((uint32)(val)) - -#define IVALu(buf,pos) IVAL(buf,pos) -#define SIVALu(buf,pos,val) SIVAL(buf,pos,val) - -# else /* !AVOID_BYTEORDER_INLINE */ - static inline uint32 IVALu(const uchar *buf, int pos) { @@ -85,18 +94,6 @@ SIVALu(uchar *buf, int pos, uint32 val) *u.num = val; } -static inline uint32 -IVAL(const char *buf, int pos) -{ - return IVALu((uchar*)buf, pos); -} - -static inline void -SIVAL(char *buf, int pos, uint32 val) -{ - SIVALu((uchar*)buf, pos, val); -} - static inline int64 IVAL64(const char *buf, int pos) { @@ -119,6 +116,17 @@ SIVAL64(char *buf, int pos, int64 val) *u.num = val; } -# endif /* !AVOID_BYTEORDER_INLINE */ - #endif /* !CAREFUL_ALIGNMENT */ + +static inline uint32 +IVAL(const char *buf, int pos) +{ + return IVALu((uchar*)buf, pos); +} + +static inline void +SIVAL(char *buf, int pos, uint32 val) +{ + SIVALu((uchar*)buf, pos, val); +} + @@ -1,7 +1,7 @@ /* * Allow an arbitrary sequence of case labels. * - * Copyright (C) 2006-2018 Wayne Davison + * Copyright (C) 2006-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,7 +17,7 @@ * with this program; if not, visit the http://fsf.org website. */ -/* This is included multiple times, once for every segement in a switch statement. +/* This is included multiple times, once for every segment in a switch statement. * This produces the next "case N:" statement in sequence. */ #if !defined CASE_N_STATE_0 @@ -25,51 +25,67 @@ case 0: #elif !defined CASE_N_STATE_1 #define CASE_N_STATE_1 + /* FALLTHROUGH */ case 1: #elif !defined CASE_N_STATE_2 #define CASE_N_STATE_2 + /* FALLTHROUGH */ case 2: #elif !defined CASE_N_STATE_3 #define CASE_N_STATE_3 + /* FALLTHROUGH */ case 3: #elif !defined CASE_N_STATE_4 #define CASE_N_STATE_4 + /* FALLTHROUGH */ case 4: #elif !defined CASE_N_STATE_5 #define CASE_N_STATE_5 + /* FALLTHROUGH */ case 5: #elif !defined CASE_N_STATE_6 #define CASE_N_STATE_6 + /* FALLTHROUGH */ case 6: #elif !defined CASE_N_STATE_7 #define CASE_N_STATE_7 + /* FALLTHROUGH */ case 7: #elif !defined CASE_N_STATE_8 #define CASE_N_STATE_8 + /* FALLTHROUGH */ case 8: #elif !defined CASE_N_STATE_9 #define CASE_N_STATE_9 + /* FALLTHROUGH */ case 9: #elif !defined CASE_N_STATE_10 #define CASE_N_STATE_10 + /* FALLTHROUGH */ case 10: #elif !defined CASE_N_STATE_11 #define CASE_N_STATE_11 + /* FALLTHROUGH */ case 11: #elif !defined CASE_N_STATE_12 #define CASE_N_STATE_12 + /* FALLTHROUGH */ case 12: #elif !defined CASE_N_STATE_13 #define CASE_N_STATE_13 + /* FALLTHROUGH */ case 13: #elif !defined CASE_N_STATE_14 #define CASE_N_STATE_14 + /* FALLTHROUGH */ case 14: #elif !defined CASE_N_STATE_15 #define CASE_N_STATE_15 + /* FALLTHROUGH */ case 15: #elif !defined CASE_N_STATE_16 #define CASE_N_STATE_16 + /* FALLTHROUGH */ case 16: #else #error Need to add more case statements! @@ -3,13 +3,20 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * + * In addition, as a special exception, the copyright holders give + * permission to dynamically link rsync with the OpenSSL and xxhash + * libraries when those libraries are being distributed in compliance + * with their license terms, and to distribute a dynamically linked + * combination of rsync and these libraries. This is also considered + * to be covered under the GPL's System Libraries exception. + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the @@ -20,36 +27,37 @@ */ #include "rsync.h" +#ifdef SUPPORT_XXHASH +#include "xxhash.h" +#endif +extern int am_server; +extern int whole_file; extern int checksum_seed; extern int protocol_version; extern int proper_seed_order; -extern char *checksum_choice; - -#define CSUM_NONE 0 -#define CSUM_MD4_ARCHAIC 1 -#define CSUM_MD4_BUSTED 2 -#define CSUM_MD4_OLD 3 -#define CSUM_MD4 4 -#define CSUM_MD5 5 +extern const char *checksum_choice; + +struct name_num_obj valid_checksums = { + "checksum", NULL, NULL, 0, 0, { +#ifdef SUPPORT_XXHASH + { CSUM_XXH64, "xxh64", NULL }, + { CSUM_XXH64, "xxhash", NULL }, +#endif + { CSUM_MD5, "md5", NULL }, + { CSUM_MD4, "md4", NULL }, + { CSUM_NONE, "none", NULL }, + { 0, NULL, NULL } + } +}; int xfersum_type = 0; /* used for the file transfer checksums */ int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */ -/* Returns 1 if --whole-file must be enabled. */ -int parse_checksum_choice(void) -{ - char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL; - if (cp) { - xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice); - checksum_type = parse_csum_name(cp+1, -1); - } else - xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1); - return xfersum_type == CSUM_NONE; -} - int parse_csum_name(const char *name, int len) { + struct name_num_item *nni; + if (len < 0 && name) len = strlen(name); @@ -62,15 +70,52 @@ int parse_csum_name(const char *name, int len) return CSUM_MD4_BUSTED; return CSUM_MD4_ARCHAIC; } - if (len == 3 && strncasecmp(name, "md4", 3) == 0) - return CSUM_MD4; - if (len == 3 && strncasecmp(name, "md5", 3) == 0) - return CSUM_MD5; - if (len == 4 && strncasecmp(name, "none", 4) == 0) - return CSUM_NONE; - - rprintf(FERROR, "unknown checksum name: %s\n", name); - exit_cleanup(RERR_UNSUPPORTED); + + nni = get_nni_by_name(&valid_checksums, name, len); + + if (!nni) { + rprintf(FERROR, "unknown checksum name: %s\n", name); + exit_cleanup(RERR_UNSUPPORTED); + } + + return nni->num; +} + +static const char *checksum_name(int num) +{ + struct name_num_item *nni = get_nni_by_num(&valid_checksums, num); + + return nni ? nni->name : num < CSUM_MD4 ? "MD4" : "UNKNOWN"; +} + +void parse_checksum_choice(int final_call) +{ + if (valid_checksums.negotiated_name) + xfersum_type = checksum_type = valid_checksums.negotiated_num; + else { + char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL; + if (cp) { + xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice); + checksum_type = parse_csum_name(cp+1, -1); + } else + xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1); + } + + if (xfersum_type == CSUM_NONE) + whole_file = 1; + + /* Snag the checksum name for both write_batch's option output & the following debug output. */ + if (valid_checksums.negotiated_name) + checksum_choice = valid_checksums.negotiated_name; + else if (checksum_choice == NULL) + checksum_choice = checksum_name(xfersum_type); + + if (final_call && DEBUG_GTE(NSTR, am_server ? 3 : 1)) { + rprintf(FINFO, "%s%s checksum: %s\n", + am_server ? "Server" : "Client", + valid_checksums.negotiated_name ? " negotiated" : "", + checksum_choice); + } } int csum_len_for_type(int cst, BOOL flist_csum) @@ -88,67 +133,111 @@ int csum_len_for_type(int cst, BOOL flist_csum) return MD4_DIGEST_LEN; case CSUM_MD5: return MD5_DIGEST_LEN; +#ifdef SUPPORT_XXHASH + case CSUM_XXH64: + return 64/8; +#endif default: /* paranoia to prevent missing case values */ exit_cleanup(RERR_UNSUPPORTED); } return 0; } +/* Returns 0 if the checksum is not canonical (i.e. it includes a seed value). + * Returns 1 if the public sum order matches our internal sum order. + * Returns -1 if the public sum order is the reverse of our internal sum order. + */ int canonical_checksum(int csum_type) { - return csum_type >= CSUM_MD4 ? 1 : 0; + switch (csum_type) { + case CSUM_NONE: + case CSUM_MD4_ARCHAIC: + case CSUM_MD4_OLD: + case CSUM_MD4_BUSTED: + break; + case CSUM_MD4: + case CSUM_MD5: + return -1; +#ifdef SUPPORT_XXHASH + case CSUM_XXH64: + return 1; +#endif + default: /* paranoia to prevent missing case values */ + exit_cleanup(RERR_UNSUPPORTED); + } + return 0; } +#ifndef HAVE_SIMD /* See simd-checksum-*.cpp. */ /* - a simple 32 bit checksum that can be upadted from either end + a simple 32 bit checksum that can be updated from either end (inspired by Mark Adler's Adler-32 checksum) */ uint32 get_checksum1(char *buf1, int32 len) { - int32 i; - uint32 s1, s2; - schar *buf = (schar *)buf1; - - s1 = s2 = 0; - for (i = 0; i < (len-4); i+=4) { - s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + - 10*CHAR_OFFSET; - s1 += (buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET); - } - for (; i < len; i++) { - s1 += (buf[i]+CHAR_OFFSET); s2 += s1; - } - return (s1 & 0xffff) + (s2 << 16); + int32 i; + uint32 s1, s2; + schar *buf = (schar *)buf1; + + s1 = s2 = 0; + for (i = 0; i < (len-4); i+=4) { + s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + 10*CHAR_OFFSET; + s1 += (buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET); + } + for (; i < len; i++) { + s1 += (buf[i]+CHAR_OFFSET); s2 += s1; + } + return (s1 & 0xffff) + (s2 << 16); } +#endif void get_checksum2(char *buf, int32 len, char *sum) { - md_context m; - switch (xfersum_type) { +#ifdef SUPPORT_XXHASH + case CSUM_XXH64: + SIVAL64(sum, 0, XXH64(buf, len, checksum_seed)); + break; +#endif case CSUM_MD5: { + MD5_CTX m5; uchar seedbuf[4]; - md5_begin(&m); + MD5_Init(&m5); if (proper_seed_order) { if (checksum_seed) { SIVALu(seedbuf, 0, checksum_seed); - md5_update(&m, seedbuf, 4); + MD5_Update(&m5, seedbuf, 4); } - md5_update(&m, (uchar *)buf, len); + MD5_Update(&m5, (uchar *)buf, len); } else { - md5_update(&m, (uchar *)buf, len); + MD5_Update(&m5, (uchar *)buf, len); if (checksum_seed) { SIVALu(seedbuf, 0, checksum_seed); - md5_update(&m, seedbuf, 4); + MD5_Update(&m5, seedbuf, 4); } } - md5_result(&m, (uchar *)sum); + MD5_Final((uchar *)sum, &m5); break; } case CSUM_MD4: +#ifdef USE_OPENSSL + { + MD4_CTX m4; + MD4_Init(&m4); + MD4_Update(&m4, (uchar *)buf, len); + if (checksum_seed) { + uchar seedbuf[4]; + SIVALu(seedbuf, 0, checksum_seed); + MD4_Update(&m4, seedbuf, 4); + } + MD4_Final((uchar *)sum, &m4); + break; + } +#endif case CSUM_MD4_OLD: case CSUM_MD4_BUSTED: case CSUM_MD4_ARCHAIC: { + md_context m; int32 i; static char *buf1; static int32 len1; @@ -194,7 +283,6 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) { struct map_struct *buf; OFF_T i, len = st_p->st_size; - md_context m; int32 remainder; int fd; @@ -204,33 +292,70 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) if (fd == -1) return; - buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK); + buf = map_file(fd, len, MAX_MAP_SIZE, CHUNK_SIZE); switch (checksum_type) { - case CSUM_MD5: - md5_begin(&m); +#ifdef SUPPORT_XXHASH + case CSUM_XXH64: { + static XXH64_state_t* state = NULL; + if (!state && !(state = XXH64_createState())) + out_of_memory("file_checksum"); - for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) { - md5_update(&m, (uchar *)map_ptr(buf, i, CSUM_CHUNK), - CSUM_CHUNK); - } + XXH64_reset(state, 0); + + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + XXH64_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); + + remainder = (int32)(len - i); + if (remainder > 0) + XXH64_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); + + SIVAL64(sum, 0, XXH64_digest(state)); + break; + } +#endif + case CSUM_MD5: { + MD5_CTX m5; + + MD5_Init(&m5); + + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + MD5_Update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); remainder = (int32)(len - i); if (remainder > 0) - md5_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder); + MD5_Update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder); - md5_result(&m, (uchar *)sum); + MD5_Final((uchar *)sum, &m5); break; + } case CSUM_MD4: +#ifdef USE_OPENSSL + { + MD4_CTX m4; + + MD4_Init(&m4); + + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + MD4_Update(&m4, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); + + remainder = (int32)(len - i); + if (remainder > 0) + MD4_Update(&m4, (uchar *)map_ptr(buf, i, remainder), remainder); + + MD4_Final((uchar *)sum, &m4); + break; + } +#endif case CSUM_MD4_OLD: case CSUM_MD4_BUSTED: - case CSUM_MD4_ARCHAIC: + case CSUM_MD4_ARCHAIC: { + md_context m; + mdfour_begin(&m); - for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) { - mdfour_update(&m, (uchar *)map_ptr(buf, i, CSUM_CHUNK), - CSUM_CHUNK); - } + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + mdfour_update(&m, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); /* Prior to version 27 an incorrect MD4 checksum was computed * by failing to call mdfour_tail() for block sizes that @@ -242,8 +367,10 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) mdfour_result(&m, (uchar *)sum); break; + } default: - rprintf(FERROR, "invalid checksum-choice for the --checksum option (%d)\n", checksum_type); + rprintf(FERROR, "Invalid checksum-choice for --checksum: %s (%d)\n", + checksum_name(checksum_type), checksum_type); exit_cleanup(RERR_UNSUPPORTED); } @@ -252,7 +379,16 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) } static int32 sumresidue; -static md_context md; +static union { + md_context md; +#ifdef USE_OPENSSL + MD4_CTX m4; +#endif + MD5_CTX m5; +} ctx; +#ifdef SUPPORT_XXHASH +static XXH64_state_t* xxh64_state; +#endif static int cursum_type; void sum_init(int csum_type, int seed) @@ -264,17 +400,28 @@ void sum_init(int csum_type, int seed) cursum_type = csum_type; switch (csum_type) { +#ifdef SUPPORT_XXHASH + case CSUM_XXH64: + if (!xxh64_state && !(xxh64_state = XXH64_createState())) + out_of_memory("sum_init"); + XXH64_reset(xxh64_state, 0); + break; +#endif case CSUM_MD5: - md5_begin(&md); + MD5_Init(&ctx.m5); break; case CSUM_MD4: - mdfour_begin(&md); +#ifdef USE_OPENSSL + MD4_Init(&ctx.m4); +#else + mdfour_begin(&ctx.md); sumresidue = 0; +#endif break; case CSUM_MD4_OLD: case CSUM_MD4_BUSTED: case CSUM_MD4_ARCHAIC: - mdfour_begin(&md); + mdfour_begin(&ctx.md); sumresidue = 0; SIVAL(s, 0, seed); sum_update(s, 4); @@ -297,36 +444,45 @@ void sum_init(int csum_type, int seed) void sum_update(const char *p, int32 len) { switch (cursum_type) { +#ifdef SUPPORT_XXHASH + case CSUM_XXH64: + XXH64_update(xxh64_state, p, len); + break; +#endif case CSUM_MD5: - md5_update(&md, (uchar *)p, len); + MD5_Update(&ctx.m5, (uchar *)p, len); break; case CSUM_MD4: +#ifdef USE_OPENSSL + MD4_Update(&ctx.m4, (uchar *)p, len); + break; +#endif case CSUM_MD4_OLD: case CSUM_MD4_BUSTED: case CSUM_MD4_ARCHAIC: if (len + sumresidue < CSUM_CHUNK) { - memcpy(md.buffer + sumresidue, p, len); + memcpy(ctx.md.buffer + sumresidue, p, len); sumresidue += len; break; } if (sumresidue) { int32 i = CSUM_CHUNK - sumresidue; - memcpy(md.buffer + sumresidue, p, i); - mdfour_update(&md, (uchar *)md.buffer, CSUM_CHUNK); + memcpy(ctx.md.buffer + sumresidue, p, i); + mdfour_update(&ctx.md, (uchar *)ctx.md.buffer, CSUM_CHUNK); len -= i; p += i; } while (len >= CSUM_CHUNK) { - mdfour_update(&md, (uchar *)p, CSUM_CHUNK); + mdfour_update(&ctx.md, (uchar *)p, CSUM_CHUNK); len -= CSUM_CHUNK; p += CSUM_CHUNK; } sumresidue = len; if (sumresidue) - memcpy(md.buffer, p, sumresidue); + memcpy(ctx.md.buffer, p, sumresidue); break; case CSUM_NONE: break; @@ -342,19 +498,28 @@ void sum_update(const char *p, int32 len) int sum_end(char *sum) { switch (cursum_type) { +#ifdef SUPPORT_XXHASH + case CSUM_XXH64: + SIVAL64(sum, 0, XXH64_digest(xxh64_state)); + break; +#endif case CSUM_MD5: - md5_result(&md, (uchar *)sum); + MD5_Final((uchar *)sum, &ctx.m5); break; case CSUM_MD4: +#ifdef USE_OPENSSL + MD4_Final((uchar *)sum, &ctx.m4); + break; +#endif case CSUM_MD4_OLD: - mdfour_update(&md, (uchar *)md.buffer, sumresidue); - mdfour_result(&md, (uchar *)sum); + mdfour_update(&ctx.md, (uchar *)ctx.md.buffer, sumresidue); + mdfour_result(&ctx.md, (uchar *)sum); break; case CSUM_MD4_BUSTED: case CSUM_MD4_ARCHAIC: if (sumresidue) - mdfour_update(&md, (uchar *)md.buffer, sumresidue); - mdfour_result(&md, (uchar *)sum); + mdfour_update(&ctx.md, (uchar *)ctx.md.buffer, sumresidue); + mdfour_result(&ctx.md, (uchar *)sum); break; case CSUM_NONE: *sum = '\0'; @@ -2,7 +2,7 @@ * Implement the core of the --chmod option. * * Copyright (C) 2002 Scott Howard - * Copyright (C) 2005-2018 Wayne Davison + * Copyright (C) 2005-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -44,7 +44,7 @@ struct chmod_mode_struct { #define STATE_OCTAL_NUM 3 /* Parse a chmod-style argument, and break it down into one or more AND/OR - * pairs in a linked list. We return a pointer to new items on succcess + * pairs in a linked list. We return a pointer to new items on success * (appending the items to the specified list), or NULL on error. */ struct chmod_mode_struct *parse_chmod(const char *modestr, struct chmod_mode_struct **root_mode_ptr) @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,6 +26,7 @@ extern int dry_run; extern int am_server; extern int am_daemon; extern int am_receiver; +extern int am_sender; extern int io_error; extern int keep_partial; extern int got_xfer_error; @@ -34,6 +35,7 @@ extern int output_needs_newline; extern char *partial_dir; extern char *logfile_name; +int called_from_signal_handler = 0; BOOL shutting_down = False; BOOL flush_ok_after_signal = False; @@ -138,7 +140,6 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) who_am_i(), code, file, line); } - /* FALLTHROUGH */ #include "case_N.h" switch_step++; @@ -152,7 +153,6 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) } } - /* FALLTHROUGH */ #include "case_N.h" switch_step++; @@ -172,18 +172,17 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) const char *fname = cleanup_fname; cleanup_fname = NULL; if (!partial_dir) { - /* We don't want to leave a partial file with a modern time or it - * could be skipped via --update. Setting the time to something - * really old also helps it to stand out as unfinished in an ls. */ - tweak_modtime = 1; - cleanup_file->modtime = 0; + /* We don't want to leave a partial file with a modern time or it + * could be skipped via --update. Setting the time to something + * really old also helps it to stand out as unfinished in an ls. */ + tweak_modtime = 1; + cleanup_file->modtime = 0; } finish_transfer(cleanup_new_fname, fname, NULL, NULL, cleanup_file, tweak_modtime, !partial_dir); } } - /* FALLTHROUGH */ #include "case_N.h" switch_step++; @@ -195,7 +194,6 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) if (!exit_code && !code) io_flush(FULL_FLUSH); - /* FALLTHROUGH */ #include "case_N.h" switch_step++; @@ -226,7 +224,6 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1)))) log_exit(exit_code, exit_file, exit_line); - /* FALLTHROUGH */ #include "case_N.h" switch_step++; @@ -238,23 +235,27 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) dry_run ? " (DRY RUN)" : ""); } - /* FALLTHROUGH */ #include "case_N.h" switch_step++; if (exit_code && exit_code != RERR_SOCKETIO && exit_code != RERR_STREAMIO && exit_code != RERR_SIGNAL1 - && exit_code != RERR_TIMEOUT && !shutting_down && (protocol_version >= 31 || am_receiver)) { - if (line > 0) { - if (DEBUG_GTE(EXIT, 3)) { - rprintf(FINFO, "[%s] sending MSG_ERROR_EXIT with exit_code %d\n", - who_am_i(), exit_code); + && exit_code != RERR_TIMEOUT && !shutting_down) { + if (protocol_version >= 31 || am_receiver) { + if (line > 0) { + if (DEBUG_GTE(EXIT, 3)) { + rprintf(FINFO, "[%s] sending MSG_ERROR_EXIT with exit_code %d\n", + who_am_i(), exit_code); + } + send_msg_int(MSG_ERROR_EXIT, exit_code); } - send_msg_int(MSG_ERROR_EXIT, exit_code); + if (!am_sender) + io_flush(MSG_FLUSH); /* Be sure to send all messages */ + noop_io_until_death(); } - noop_io_until_death(); + else if (!am_sender) + io_flush(MSG_FLUSH); /* Be sure to send all messages */ } - /* FALLTHROUGH */ #include "case_N.h" switch_step++; @@ -267,6 +268,8 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) break; } + if (called_from_signal_handler) + _exit(exit_code); exit(exit_code); } diff --git a/clientname.c b/clientname.c index 4ac8b485..1e85c84e 100644 --- a/clientname.c +++ b/clientname.c @@ -3,7 +3,7 @@ * * Copyright (C) 1992-2001 Andrew Tridgell <tridge@samba.org> * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2002-2018 Wayne Davison + * Copyright (C) 2002-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -27,52 +27,60 @@ */ #include "rsync.h" +#include "itypes.h" + +extern int am_daemon; static const char default_name[] = "UNKNOWN"; -extern int am_server; +static const char proxyv2sig[] = "\r\n\r\n\0\r\nQUIT\n"; +static char ipaddr_buf[100]; -/** - * Return the IP addr of the client as a string - **/ +#define PROXY_V2_SIG_SIZE ((int)sizeof proxyv2sig - 1) +#define PROXY_V2_HEADER_SIZE (PROXY_V2_SIG_SIZE + 1 + 1 + 2) + +#define CMD_LOCAL 0 +#define CMD_PROXY 1 + +#define PROXY_FAM_TCPv4 0x11 +#define PROXY_FAM_TCPv6 0x21 + +#define GET_SOCKADDR_FAMILY(ss) ((struct sockaddr*)ss)->sa_family + +static void client_sockaddr(int fd, struct sockaddr_storage *ss, socklen_t *ss_len); +static int check_name(const char *ipaddr, const struct sockaddr_storage *ss, char *name_buf, size_t name_buf_size); +static int valid_ipaddr(const char *s); + +/* Return the IP addr of the client as a string. */ char *client_addr(int fd) { - static char addr_buf[100]; - static int initialised; struct sockaddr_storage ss; socklen_t length = sizeof ss; - if (initialised) - return addr_buf; + if (*ipaddr_buf) + return ipaddr_buf; - initialised = 1; - - if (am_server) { /* daemon over --rsh mode */ + if (am_daemon < 0) { /* daemon over --rsh mode */ char *env_str; - strlcpy(addr_buf, "0.0.0.0", sizeof addr_buf); + strlcpy(ipaddr_buf, "0.0.0.0", sizeof ipaddr_buf); if ((env_str = getenv("REMOTE_HOST")) != NULL || (env_str = getenv("SSH_CONNECTION")) != NULL || (env_str = getenv("SSH_CLIENT")) != NULL || (env_str = getenv("SSH2_CLIENT")) != NULL) { char *p; - strlcpy(addr_buf, env_str, sizeof addr_buf); + strlcpy(ipaddr_buf, env_str, sizeof ipaddr_buf); /* Truncate the value to just the IP address. */ - if ((p = strchr(addr_buf, ' ')) != NULL) + if ((p = strchr(ipaddr_buf, ' ')) != NULL) *p = '\0'; } - } else { - client_sockaddr(fd, &ss, &length); - getnameinfo((struct sockaddr *)&ss, length, - addr_buf, sizeof addr_buf, NULL, 0, NI_NUMERICHOST); + if (valid_ipaddr(ipaddr_buf)) + return ipaddr_buf; } - return addr_buf; -} + client_sockaddr(fd, &ss, &length); + getnameinfo((struct sockaddr *)&ss, length, ipaddr_buf, sizeof ipaddr_buf, NULL, 0, NI_NUMERICHOST); - -static int get_sockaddr_family(const struct sockaddr_storage *ss) -{ - return ((struct sockaddr *) ss)->sa_family; + return ipaddr_buf; } @@ -89,71 +97,216 @@ static int get_sockaddr_family(const struct sockaddr_storage *ss) * After translation from sockaddr to name we do a forward lookup to * make sure nobody is spoofing PTR records. **/ -char *client_name(int fd) +char *client_name(const char *ipaddr) { static char name_buf[100]; - static char port_buf[100]; - static int initialised; + char port_buf[100]; struct sockaddr_storage ss; socklen_t ss_len; + struct addrinfo hint, *answer; + int err; - if (initialised) + if (*name_buf) return name_buf; strlcpy(name_buf, default_name, sizeof name_buf); - initialised = 1; - - memset(&ss, 0, sizeof ss); - - if (am_server) { /* daemon over --rsh mode */ - char *addr = client_addr(fd); - struct addrinfo hint, *answer; - int err; - if (strcmp(addr, "0.0.0.0") == 0) - return name_buf; + if (strcmp(ipaddr, "0.0.0.0") == 0) + return name_buf; - memset(&hint, 0, sizeof hint); + memset(&ss, 0, sizeof ss); + memset(&hint, 0, sizeof hint); #ifdef AI_NUMERICHOST - hint.ai_flags = AI_NUMERICHOST; + hint.ai_flags = AI_NUMERICHOST; #endif - hint.ai_socktype = SOCK_STREAM; + hint.ai_socktype = SOCK_STREAM; - if ((err = getaddrinfo(addr, NULL, &hint, &answer)) != 0) { - rprintf(FLOG, "malformed address %s: %s\n", - addr, gai_strerror(err)); - return name_buf; - } + if ((err = getaddrinfo(ipaddr, NULL, &hint, &answer)) != 0) { + rprintf(FLOG, "malformed address %s: %s\n", ipaddr, gai_strerror(err)); + return name_buf; + } - switch (answer->ai_family) { - case AF_INET: - ss_len = sizeof (struct sockaddr_in); - memcpy(&ss, answer->ai_addr, ss_len); - break; + switch (answer->ai_family) { + case AF_INET: + ss_len = sizeof (struct sockaddr_in); + memcpy(&ss, answer->ai_addr, ss_len); + break; #ifdef INET6 - case AF_INET6: - ss_len = sizeof (struct sockaddr_in6); - memcpy(&ss, answer->ai_addr, ss_len); - break; + case AF_INET6: + ss_len = sizeof (struct sockaddr_in6); + memcpy(&ss, answer->ai_addr, ss_len); + break; #endif - default: - exit_cleanup(RERR_SOCKETIO); - } - freeaddrinfo(answer); - } else { - ss_len = sizeof ss; - client_sockaddr(fd, &ss, &ss_len); + default: + assert(0); } + freeaddrinfo(answer); - if (lookup_name(fd, &ss, ss_len, name_buf, sizeof name_buf, - port_buf, sizeof port_buf) == 0) - check_name(fd, &ss, name_buf, sizeof name_buf); + /* reverse lookup */ + err = getnameinfo((struct sockaddr*)&ss, ss_len, name_buf, sizeof name_buf, + port_buf, sizeof port_buf, NI_NAMEREQD | NI_NUMERICSERV); + if (err) { + strlcpy(name_buf, default_name, sizeof name_buf); + rprintf(FLOG, "name lookup failed for %s: %s\n", ipaddr, gai_strerror(err)); + } else + check_name(ipaddr, &ss, name_buf, sizeof name_buf); return name_buf; } +/* Try to read an proxy protocol header (V1 or V2). Returns 1 on success or 0 on failure. */ +int read_proxy_protocol_header(int fd) +{ + union { + struct { + char line[108]; + } v1; + struct { + char sig[PROXY_V2_SIG_SIZE]; + char ver_cmd; + char fam; + char len[2]; + union { + struct { + char src_addr[4]; + char dst_addr[4]; + char src_port[2]; + char dst_port[2]; + } ip4; + struct { + char src_addr[16]; + char dst_addr[16]; + char src_port[2]; + char dst_port[2]; + } ip6; + struct { + char src_addr[108]; + char dst_addr[108]; + } unx; + } addr; + } v2; + } hdr; + + read_buf(fd, (char*)&hdr, PROXY_V2_SIG_SIZE); + + if (memcmp(hdr.v2.sig, proxyv2sig, PROXY_V2_SIG_SIZE) == 0) { /* Proxy V2 */ + int ver, cmd, size; + + read_buf(fd, (char*)&hdr + PROXY_V2_SIG_SIZE, PROXY_V2_HEADER_SIZE - PROXY_V2_SIG_SIZE); + + ver = (hdr.v2.ver_cmd & 0xf0) >> 4; + cmd = (hdr.v2.ver_cmd & 0x0f); + size = (hdr.v2.len[0] << 8) + hdr.v2.len[1]; + + if (ver != 2 || size + PROXY_V2_HEADER_SIZE > (int)sizeof hdr) + return 0; + + /* Grab all the remaining data in the binary request. */ + read_buf(fd, (char*)&hdr + PROXY_V2_HEADER_SIZE, size); + + switch (cmd) { + case CMD_PROXY: + switch (hdr.v2.fam) { + case PROXY_FAM_TCPv4: + if (size != sizeof hdr.v2.addr.ip4) + return 0; + inet_ntop(AF_INET, hdr.v2.addr.ip4.src_addr, ipaddr_buf, sizeof ipaddr_buf); + return valid_ipaddr(ipaddr_buf); + case PROXY_FAM_TCPv6: + if (size != sizeof hdr.v2.addr.ip6) + return 0; + inet_ntop(AF_INET6, hdr.v2.addr.ip6.src_addr, ipaddr_buf, sizeof ipaddr_buf); + return valid_ipaddr(ipaddr_buf); + default: + break; + } + /* For an unsupported protocol we'll ignore the proxy data (leaving ipaddr_buf unset) + * and accept the connection, which will get handled as a normal socket addr. */ + return 1; + case CMD_LOCAL: + return 1; + default: + break; + } + + return 0; + } + + if (memcmp(hdr.v1.line, "PROXY", 5) == 0) { /* Proxy V1 */ + char *endc, *sp, *p = hdr.v1.line + PROXY_V2_SIG_SIZE; + int port_chk; + + *p = '\0'; + if (!strchr(hdr.v1.line, '\n')) { + while (1) { + read_buf(fd, p, 1); + if (*p++ == '\n') + break; + if (p - hdr.v1.line >= (int)sizeof hdr.v1.line - 1) + return 0; + } + *p = '\0'; + } + + endc = strchr(hdr.v1.line, '\r'); + if (!endc || endc[1] != '\n' || endc[2]) + return 0; + *endc = '\0'; + + p = hdr.v1.line + 5; + + if (!isSpace(p++)) + return 0; + if (strncmp(p, "TCP4", 4) == 0) + p += 4; + else if (strncmp(p, "TCP6", 4) == 0) + p += 4; + else if (strncmp(p, "UNKNOWN", 7) == 0) + return 1; + else + return 0; + + if (!isSpace(p++)) + return 0; + + if ((sp = strchr(p, ' ')) == NULL) + return 0; + *sp = '\0'; + if (!valid_ipaddr(p)) + return 0; + strlcpy(ipaddr_buf, p, sizeof ipaddr_buf); /* It will always fit when valid. */ + + p = sp + 1; + if ((sp = strchr(p, ' ')) == NULL) + return 0; + *sp = '\0'; + if (!valid_ipaddr(p)) + return 0; + /* Ignore destination address. */ + + p = sp + 1; + if ((sp = strchr(p, ' ')) == NULL) + return 0; + *sp = '\0'; + port_chk = strtol(p, &endc, 10); + if (*endc || port_chk == 0) + return 0; + /* Ignore source port. */ + + p = sp + 1; + port_chk = strtol(p, &endc, 10); + if (*endc || port_chk == 0) + return 0; + /* Ignore destination port. */ + + return 1; + } + + return 0; +} + /** * Get the sockaddr for the client. @@ -161,9 +314,7 @@ char *client_name(int fd) * If it comes in as an ipv4 address mapped into IPv6 format then we * convert it back to a regular IPv4. **/ -void client_sockaddr(int fd, - struct sockaddr_storage *ss, - socklen_t *ss_len) +static void client_sockaddr(int fd, struct sockaddr_storage *ss, socklen_t *ss_len) { memset(ss, 0, sizeof *ss); @@ -174,8 +325,8 @@ void client_sockaddr(int fd, } #ifdef INET6 - if (get_sockaddr_family(ss) == AF_INET6 && - IN6_IS_ADDR_V4MAPPED(&((struct sockaddr_in6 *)ss)->sin6_addr)) { + if (GET_SOCKADDR_FAMILY(ss) == AF_INET6 + && IN6_IS_ADDR_V4MAPPED(&((struct sockaddr_in6 *)ss)->sin6_addr)) { /* OK, so ss is in the IPv6 family, but it is really * an IPv4 address: something like * "::ffff:10.130.1.2". If we use it as-is, then the @@ -198,51 +349,20 @@ void client_sockaddr(int fd, /* There is a macro to extract the mapped part * (IN6_V4MAPPED_TO_SINADDR ?), but it does not seem * to be present in the Linux headers. */ - memcpy(&sin->sin_addr, &sin6.sin6_addr.s6_addr[12], - sizeof sin->sin_addr); + memcpy(&sin->sin_addr, &sin6.sin6_addr.s6_addr[12], sizeof sin->sin_addr); } #endif } /** - * Look up a name from @p ss into @p name_buf. - * - * @param fd file descriptor for client socket. - **/ -int lookup_name(int fd, const struct sockaddr_storage *ss, - socklen_t ss_len, - char *name_buf, size_t name_buf_size, - char *port_buf, size_t port_buf_size) -{ - int name_err; - - /* reverse lookup */ - name_err = getnameinfo((struct sockaddr *) ss, ss_len, - name_buf, name_buf_size, - port_buf, port_buf_size, - NI_NAMEREQD | NI_NUMERICSERV); - if (name_err != 0) { - strlcpy(name_buf, default_name, name_buf_size); - rprintf(FLOG, "name lookup failed for %s: %s\n", - client_addr(fd), gai_strerror(name_err)); - return name_err; - } - - return 0; -} - - - -/** * Compare an addrinfo from the resolver to a sockinfo. * * Like strcmp, returns 0 for identical. **/ -int compare_addrinfo_sockaddr(const struct addrinfo *ai, - const struct sockaddr_storage *ss) +static int compare_addrinfo_sockaddr(const struct addrinfo *ai, const struct sockaddr_storage *ss) { - int ss_family = get_sockaddr_family(ss); + int ss_family = GET_SOCKADDR_FAMILY(ss); const char fn[] = "compare_addrinfo_sockaddr"; if (ai->ai_family != ss_family) { @@ -258,8 +378,7 @@ int compare_addrinfo_sockaddr(const struct addrinfo *ai, sin1 = (const struct sockaddr_in *) ss; sin2 = (const struct sockaddr_in *) ai->ai_addr; - return memcmp(&sin1->sin_addr, &sin2->sin_addr, - sizeof sin1->sin_addr); + return memcmp(&sin1->sin_addr, &sin2->sin_addr, sizeof sin1->sin_addr); } #ifdef INET6 @@ -269,14 +388,13 @@ int compare_addrinfo_sockaddr(const struct addrinfo *ai, sin1 = (const struct sockaddr_in6 *) ss; sin2 = (const struct sockaddr_in6 *) ai->ai_addr; - if (ai->ai_addrlen < sizeof (struct sockaddr_in6)) { + if (ai->ai_addrlen < (int)sizeof (struct sockaddr_in6)) { rprintf(FLOG, "%s: too short sockaddr_in6; length=%d\n", fn, (int)ai->ai_addrlen); return 1; } - if (memcmp(&sin1->sin6_addr, &sin2->sin6_addr, - sizeof sin1->sin6_addr)) + if (memcmp(&sin1->sin6_addr, &sin2->sin6_addr, sizeof sin1->sin6_addr)) return 1; #ifdef HAVE_SOCKADDR_IN6_SCOPE_ID @@ -302,13 +420,11 @@ int compare_addrinfo_sockaddr(const struct addrinfo *ai, * because it doesn't seem that it could be spoofed in any way, and * getaddrinfo on random service names seems to cause problems on AIX. **/ -int check_name(int fd, - const struct sockaddr_storage *ss, - char *name_buf, size_t name_buf_size) +static int check_name(const char *ipaddr, const struct sockaddr_storage *ss, char *name_buf, size_t name_buf_size) { struct addrinfo hints, *res, *res0; int error; - int ss_family = get_sockaddr_family(ss); + int ss_family = GET_SOCKADDR_FAMILY(ss); memset(&hints, 0, sizeof hints); hints.ai_family = ss_family; @@ -339,10 +455,74 @@ int check_name(int fd, /* We hit the end of the list without finding an * address that was the same as ss. */ rprintf(FLOG, "%s is not a known address for \"%s\": " - "spoofed address?\n", client_addr(fd), name_buf); + "spoofed address?\n", ipaddr, name_buf); strlcpy(name_buf, default_name, name_buf_size); } freeaddrinfo(res0); return 0; } + +/* Returns 1 for a valid IPv4 or IPv6 addr, or 0 for a bad one. */ +static int valid_ipaddr(const char *s) +{ + int i; + + if (strchr(s, ':') != NULL) { /* Only IPv6 has a colon. */ + int count, saw_double_colon = 0; + int ipv4_at_end = 0; + + if (*s == ':') { /* A colon at the start must be a :: */ + if (*++s != ':') + return 0; + saw_double_colon = 1; + s++; + } + + for (count = 0; count < 8; count++) { + if (!*s) + return saw_double_colon && count < 7; + + if (strchr(s, ':') == NULL && strchr(s, '.') != NULL) { + if ((!saw_double_colon && count != 6) || (saw_double_colon && count > 6)) + return 0; + ipv4_at_end = 1; + break; + } + + if (!isHexDigit(s++)) /* Need 1-4 hex digits */ + return 0; + if (isHexDigit(s) && isHexDigit(++s) && isHexDigit(++s) && isHexDigit(++s)) + return 0; + + if (*s == ':') { + if (!*++s) + return 0; + if (*s == ':') { + if (saw_double_colon) + return 0; + saw_double_colon = 1; + s++; + } + } + } + + if (!ipv4_at_end) + return !*s; + } + + /* IPv4 */ + for (i = 0; i < 4; i++) { + long n; + char *end; + + if (i && *s++ != '.') + return 0; + n = strtol(s, &end, 10); + if (n > 255 || n < 0 || end <= s || end > s+3) + return 0; + s = end; + } + + return !*s; +} diff --git a/clientserver.c b/clientserver.c index e2e2dc02..b9325186 100644 --- a/clientserver.c +++ b/clientserver.c @@ -3,7 +3,7 @@ * * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org> * Copyright (C) 2001-2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2002-2018 Wayne Davison + * Copyright (C) 2002-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -30,6 +30,7 @@ extern int am_sender; extern int am_server; extern int am_daemon; extern int am_root; +extern int msgs2stderr; extern int rsync_port; extern int protect_args; extern int ignore_errors; @@ -65,6 +66,7 @@ extern gid_t our_gid; char *auth_user; int read_only = 0; int module_id = -1; +int pid_file_fd = -1; struct chmod_mode_struct *daemon_chmod_modes; /* module_dirlen is the length of the module_dir string when in daemon @@ -121,8 +123,7 @@ int start_socket_client(char *host, int remote_argc, char *remote_argv[], *p = '\0'; } - fd = open_socket_out_wrapped(host, rsync_port, bind_address, - default_af_hint); + fd = open_socket_out_wrapped(host, rsync_port, bind_address, default_af_hint); if (fd == -1) exit_cleanup(RERR_SOCKETIO); @@ -347,11 +348,122 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char return 0; } -static char *finish_pre_exec(pid_t pid, int write_fd, int read_fd, char *request, - char **early_argv, char **argv) +#ifdef HAVE_PUTENV +static int read_arg_from_pipe(int fd, char *buf, int limit) +{ + char *bp = buf, *eob = buf + limit - 1; + + while (1) { + int got = read(fd, bp, 1); + if (got != 1) { + if (got < 0 && errno == EINTR) + continue; + return -1; + } + if (*bp == '\0') + break; + if (bp < eob) + bp++; + } + *bp = '\0'; + + return bp - buf; +} +#endif + +static void set_env_str(const char *var, const char *str) +{ +#ifdef HAVE_PUTENV + char *mem; + if (asprintf(&mem, "%s=%s", var, str) < 0) + out_of_memory("set_env_str"); + putenv(mem); +#endif +} + +#ifdef HAVE_PUTENV +void set_env_num(const char *var, long num) +{ + char *mem; + if (asprintf(&mem, "%s=%ld", var, num) < 0) + out_of_memory("set_env_num"); + putenv(mem); +} +#endif + +/* Used for both early exec & pre-xfer exec */ +static pid_t start_pre_exec(const char *cmd, int *arg_fd_ptr, int *error_fd_ptr) +{ + int arg_fds[2], error_fds[2], arg_fd; + pid_t pid; + + if ((error_fd_ptr && pipe(error_fds) < 0) || (arg_fd_ptr && pipe(arg_fds) < 0) || (pid = fork()) < 0) + return (pid_t)-1; + + if (pid == 0) { + char buf[BIGPATHBUFLEN]; + int j, len, status; + + if (error_fd_ptr) { + close(error_fds[0]); + set_blocking(error_fds[1]); + } + + if (arg_fd_ptr) { + close(arg_fds[1]); + arg_fd = arg_fds[0]; + set_blocking(arg_fd); + + len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN); + if (len <= 0) + _exit(1); + set_env_str("RSYNC_REQUEST", buf); + + for (j = 0; ; j++) { + char *p; + len = read_arg_from_pipe(arg_fd, buf, BIGPATHBUFLEN); + if (len <= 0) { + if (!len) + break; + _exit(1); + } + if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0) + putenv(p); + } + close(arg_fd); + } + + if (error_fd_ptr) { + close(STDIN_FILENO); + dup2(error_fds[1], STDOUT_FILENO); + close(error_fds[1]); + } + + status = shell_exec(cmd); + + if (!WIFEXITED(status)) + _exit(1); + _exit(WEXITSTATUS(status)); + } + + if (error_fd_ptr) { + close(error_fds[1]); + *error_fd_ptr = error_fds[0]; + set_blocking(error_fds[0]); + } + + if (arg_fd_ptr) { + close(arg_fds[0]); + arg_fd = *arg_fd_ptr = arg_fds[1]; + set_blocking(arg_fd); + } + + return pid; +} + +static void write_pre_exec_args(int write_fd, char *request, char **early_argv, char **argv) { - char buf[BIGPATHBUFLEN], *bp; - int j = 0, status = -1, msglen = sizeof buf - 1; + int j = 0; if (!request) request = "(NONE)"; @@ -367,33 +479,51 @@ static char *finish_pre_exec(pid_t pid, int write_fd, int read_fd, char *request write_byte(write_fd, 0); close(write_fd); +} - /* Read the stdout from the pre-xfer exec program. This it is only - * displayed to the user if the script also returns an error status. */ - for (bp = buf; msglen > 0; msglen -= j) { - if ((j = read(read_fd, bp, msglen)) <= 0) { - if (j == 0) - break; - if (errno == EINTR) - continue; - break; /* Just ignore the read error for now... */ - } - bp += j; - if (j > 1 && bp[-1] == '\n' && bp[-2] == '\r') { - bp--; - j--; - bp[-1] = '\n'; +static char *finish_pre_exec(const char *desc, pid_t pid, int read_fd) +{ + char buf[BIGPATHBUFLEN], *bp, *cr; + int j, status = -1, msglen = sizeof buf - 1; + + if (read_fd >= 0) { + /* Read the stdout from the program. This it is only displayed + * to the user if the script also returns an error status. */ + for (bp = buf, cr = buf; msglen > 0; msglen -= j) { + if ((j = read(read_fd, bp, msglen)) <= 0) { + if (j == 0) + break; + if (errno == EINTR) + continue; + break; /* Just ignore the read error for now... */ + } + bp[j] = '\0'; + while (1) { + if ((cr = strchr(cr, '\r')) == NULL) { + cr = bp + j; + break; + } + if (!cr[1]) + break; /* wait for more data before we decide what to do */ + if (cr[1] == '\n') { + memmove(cr, cr+1, j - (cr - bp)); + j--; + } else + cr++; + } + bp += j; } - } - *bp = '\0'; + *bp = '\0'; - close(read_fd); + close(read_fd); + } else + *buf = '\0'; if (wait_process(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { char *e; - if (asprintf(&e, "pre-xfer exec returned failure (%d)%s%s%s\n%s", - status, status < 0 ? ": " : "", + if (asprintf(&e, "%s returned failure (%d)%s%s%s\n%s", + desc, status, status < 0 ? ": " : "", status < 0 ? strerror(errno) : "", *buf ? ":" : "", buf) < 0) return "out_of_memory in finish_pre_exec\n"; @@ -402,29 +532,6 @@ static char *finish_pre_exec(pid_t pid, int write_fd, int read_fd, char *request return NULL; } -#ifdef HAVE_PUTENV -static int read_arg_from_pipe(int fd, char *buf, int limit) -{ - char *bp = buf, *eob = buf + limit - 1; - - while (1) { - int got = read(fd, bp, 1); - if (got != 1) { - if (got < 0 && errno == EINTR) - continue; - return -1; - } - if (*bp == '\0') - break; - if (bp < eob) - bp++; - } - *bp = '\0'; - - return bp - buf; -} -#endif - static int path_failure(int f_out, const char *dir, BOOL was_chdir) { if (was_chdir) @@ -476,26 +583,6 @@ static struct passwd *want_all_groups(int f_out, uid_t uid) } #endif -static void set_env_str(const char *var, const char *str) -{ -#ifdef HAVE_PUTENV - char *mem; - if (asprintf(&mem, "%s=%s", var, str) < 0) - out_of_memory("set_env_str"); - putenv(mem); -#endif -} - -#ifdef HAVE_PUTENV -static void set_env_num(const char *var, long num) -{ - char *mem; - if (asprintf(&mem, "%s=%ld", var, num) < 0) - out_of_memory("set_env_num"); - putenv(mem); -} -#endif - static int rsync_module(int f_in, int f_out, int i, const char *addr, const char *host) { int argc; @@ -526,7 +613,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char /* If reverse lookup is disabled globally but enabled for this module, * we need to do it now before the access check. */ if (host == undetermined_hostname && lp_reverse_lookup(i)) - host = client_name(f_in); + host = client_name(client_addr(f_in)); set_env_str("RSYNC_HOST_NAME", host); set_env_str("RSYNC_HOST_ADDR", addr); @@ -543,7 +630,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char return -1; } - if (am_daemon && am_server) { + if (am_daemon > 0) { rprintf(FLOG, "rsync allowed access on module %s from %s (%s)\n", name, host, addr); } @@ -666,30 +753,31 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char p = lp_filter(i); parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT), - XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3); + XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3); p = lp_include_from(i); parse_filter_file(&daemon_filter_list, p, rule_template(FILTRULE_INCLUDE), - XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS); + XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS); p = lp_include(i); parse_filter_str(&daemon_filter_list, p, - rule_template(FILTRULE_INCLUDE | FILTRULE_WORD_SPLIT), - XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES); + rule_template(FILTRULE_INCLUDE | FILTRULE_WORD_SPLIT), + XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES); p = lp_exclude_from(i); parse_filter_file(&daemon_filter_list, p, rule_template(0), - XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS); + XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS); p = lp_exclude(i); parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT), - XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES); + XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES); log_init(1); #ifdef HAVE_PUTENV - if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) { - int status; + if ((*lp_early_exec(i) || *lp_prexfer_exec(i) || *lp_postxfer_exec(i)) + && !getenv("RSYNC_NO_XFER_EXEC")) { + set_env_num("RSYNC_PID", (long)getpid()); /* For post-xfer exec, fork a new process to run the rsync * daemon while this process waits for the exit status and @@ -702,10 +790,10 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char return -1; } if (pid) { + int status; close(f_in); if (f_out != f_in) close(f_out); - set_env_num("RSYNC_PID", (long)pid); if (wait_process(pid, &status, 0) < 0) status = -1; set_env_num("RSYNC_RAW_STATUS", status); @@ -714,61 +802,38 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char else status = -1; set_env_num("RSYNC_EXIT_STATUS", status); - if (system(lp_postxfer_exec(i)) < 0) + if (shell_exec(lp_postxfer_exec(i)) < 0) status = -1; _exit(status); } } + + /* For early exec, fork a child process to run the indicated + * command and wait for it to exit. */ + if (*lp_early_exec(i)) { + pid_t pid = start_pre_exec(lp_early_exec(i), NULL, NULL); + if (pid == (pid_t)-1) { + rsyserr(FLOG, errno, "early exec preparation failed"); + io_printf(f_out, "@ERROR: early exec preparation failed\n"); + return -1; + } + if (finish_pre_exec("early exec", pid, -1) != NULL) { + rsyserr(FLOG, errno, "early exec failed"); + io_printf(f_out, "@ERROR: early exec failed\n"); + return -1; + } + } + /* For pre-xfer exec, fork a child process to run the indicated * command, though it first waits for the parent process to * send us the user's request via a pipe. */ if (*lp_prexfer_exec(i)) { - int arg_fds[2], error_fds[2]; - set_env_num("RSYNC_PID", (long)getpid()); - if (pipe(arg_fds) < 0 || pipe(error_fds) < 0 || (pre_exec_pid = fork()) < 0) { + pre_exec_pid = start_pre_exec(lp_prexfer_exec(i), &pre_exec_arg_fd, &pre_exec_error_fd); + if (pre_exec_pid == (pid_t)-1) { rsyserr(FLOG, errno, "pre-xfer exec preparation failed"); io_printf(f_out, "@ERROR: pre-xfer exec preparation failed\n"); return -1; } - if (pre_exec_pid == 0) { - char buf[BIGPATHBUFLEN]; - int j, len; - close(arg_fds[1]); - close(error_fds[0]); - pre_exec_arg_fd = arg_fds[0]; - pre_exec_error_fd = error_fds[1]; - set_blocking(pre_exec_arg_fd); - set_blocking(pre_exec_error_fd); - len = read_arg_from_pipe(pre_exec_arg_fd, buf, BIGPATHBUFLEN); - if (len <= 0) - _exit(1); - set_env_str("RSYNC_REQUEST", buf); - for (j = 0; ; j++) { - len = read_arg_from_pipe(pre_exec_arg_fd, buf, - BIGPATHBUFLEN); - if (len <= 0) { - if (!len) - break; - _exit(1); - } - if (asprintf(&p, "RSYNC_ARG%d=%s", j, buf) >= 0) - putenv(p); - } - close(pre_exec_arg_fd); - close(STDIN_FILENO); - dup2(pre_exec_error_fd, STDOUT_FILENO); - close(pre_exec_error_fd); - status = system(lp_prexfer_exec(i)); - if (!WIFEXITED(status)) - _exit(1); - _exit(WEXITSTATUS(status)); - } - close(arg_fds[0]); - close(error_fds[1]); - pre_exec_arg_fd = arg_fds[1]; - pre_exec_error_fd = error_fds[0]; - set_blocking(pre_exec_arg_fd); - set_blocking(pre_exec_error_fd); } } #endif @@ -883,10 +948,12 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char orig_early_argv = NULL; munge_symlinks = save_munge_symlinks; /* The client mustn't control this. */ + if (am_daemon > 0) + msgs2stderr = 0; /* A non-rsh-run daemon doesn't have stderr for msgs. */ if (pre_exec_pid) { - err_msg = finish_pre_exec(pre_exec_pid, pre_exec_arg_fd, pre_exec_error_fd, - request, orig_early_argv, orig_argv); + write_pre_exec_args(pre_exec_arg_fd, request, orig_early_argv, orig_argv); + err_msg = finish_pre_exec("pre-xfer exec", pre_exec_pid, pre_exec_error_fd); } if (orig_early_argv) @@ -927,8 +994,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char limit_output_verbosity(lp_max_verbosity(i)); #endif - if (protocol_version < 23 - && (protocol_version == 22 || am_sender)) + if (protocol_version < 23 && (protocol_version == 22 || am_sender)) io_start_multiplex_out(f_out); else if (!ret || err_msg) { /* We have to get I/O multiplexing started so that we can @@ -965,6 +1031,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char } if (*err_msg) rprintf(FERROR, "%s\n", err_msg); + io_flush(MSG_FLUSH); } else option_error(); msleep(400); @@ -1028,7 +1095,7 @@ static void send_listing(int fd) static int load_config(int globals_only) { if (!config_file) { - if (am_server && am_root <= 0) + if (am_daemon < 0 && am_root <= 0) config_file = RSYNCD_USERCONF; else config_file = RSYNCD_SYSCONF; @@ -1046,6 +1113,13 @@ int start_daemon(int f_in, int f_out) char *p; int i; + /* At this point, am_server is only set for a daemon started via rsh. + * Because am_server gets forced on soon, we'll set am_daemon to -1 as + * a flag that can be checked later on to distinguish a normal daemon + * from an rsh-run daemon. */ + if (am_server) + am_daemon = -1; + io_set_sock_fds(f_in, f_out); /* We must load the config file before calling any function that @@ -1055,6 +1129,9 @@ int start_daemon(int f_in, int f_out) if (!load_config(0)) exit_cleanup(RERR_SYNTAX); + if (lp_proxy_protocol() && !read_proxy_protocol_header(f_in)) + return -1; + p = lp_daemon_chroot(); if (*p) { log_init(0); /* Make use we've initialized syslog before chrooting. */ @@ -1092,10 +1169,10 @@ int start_daemon(int f_in, int f_out) } addr = client_addr(f_in); - host = lp_reverse_lookup(-1) ? client_name(f_in) : undetermined_hostname; + host = lp_reverse_lookup(-1) ? client_name(addr) : undetermined_hostname; rprintf(FLOG, "connect from %s (%s)\n", host, addr); - if (!am_server) { + if (am_daemon > 0) { set_socket_options(f_in, "SO_KEEPALIVE"); set_nonblocking(f_in); } @@ -1138,26 +1215,65 @@ int start_daemon(int f_in, int f_out) static void create_pid_file(void) { char *pid_file = lp_pid_file(); - char pidbuf[16]; - pid_t pid = getpid(); - int fd, len; + char pidbuf[32]; + STRUCT_STAT st1, st2; + char *fail = NULL; if (!pid_file || !*pid_file) return; - cleanup_set_pid(pid); - if ((fd = do_open(pid_file, O_WRONLY|O_CREAT|O_EXCL, 0666)) == -1) { - failure: - cleanup_set_pid(0); - fprintf(stderr, "failed to create pid file %s: %s\n", pid_file, strerror(errno)); - rsyserr(FLOG, errno, "failed to create pid file %s", pid_file); +#ifdef O_NOFOLLOW +#define SAFE_OPEN_FLAGS (O_CREAT|O_NOFOLLOW) +#else +#define SAFE_OPEN_FLAGS (O_CREAT) +#endif + + /* These tests make sure that a temp-style lock dir is handled safely. */ + st1.st_mode = 0; + if (do_lstat(pid_file, &st1) == 0 && !S_ISREG(st1.st_mode) && unlink(pid_file) < 0) + fail = "unlink"; + else if ((pid_file_fd = do_open(pid_file, O_RDWR|SAFE_OPEN_FLAGS, 0664)) < 0) + fail = S_ISREG(st1.st_mode) ? "open" : "create"; + else if (!lock_range(pid_file_fd, 0, 4)) + fail = "lock"; + else if (do_fstat(pid_file_fd, &st1) < 0) + fail = "fstat opened"; + else if (st1.st_size > (int)sizeof pidbuf) + fail = "find small"; + else if (do_lstat(pid_file, &st2) < 0) + fail = "lstat"; + else if (!S_ISREG(st1.st_mode)) + fail = "avoid file overwrite race for"; + else if (st1.st_dev != st2.st_dev || st1.st_ino != st2.st_ino) + fail = "verify stat info for"; +#ifdef HAVE_FTRUNCATE + else if (do_ftruncate(pid_file_fd, 0) < 0) + fail = "truncate"; +#endif + else { + pid_t pid = getpid(); + int len = snprintf(pidbuf, sizeof pidbuf, "%d\n", (int)pid); +#ifndef HAVE_FTRUNCATE + /* What can we do with a too-long file and no truncate? I guess we'll add extra newlines. */ + while (len < st1.st_size) /* We already verified that st_size chars fits in the buffer. */ + pidbuf[len++] = '\n'; + /* We don't need the buffer to end in a '\0' (and we may not have room to add it). */ +#endif + if (write(pid_file_fd, pidbuf, len) != len) + fail = "write"; + cleanup_set_pid(pid); /* Mark the file for removal on exit, even if the write failed. */ + } + + if (fail) { + char msg[1024]; + snprintf(msg, sizeof msg, "failed to %s pid file %s: %s\n", + fail, pid_file, strerror(errno)); + fputs(msg, stderr); + rprintf(FLOG, "%s", msg); exit_cleanup(RERR_FILEIO); } - snprintf(pidbuf, sizeof pidbuf, "%d\n", (int)pid); - len = strlen(pidbuf); - if (write(fd, pidbuf, len) != len) - goto failure; - close(fd); + + /* The file is left open so that the lock remains valid. It is closed in our forked child procs. */ } /* Become a daemon, discarding the controlling terminal. */ @@ -3,7 +3,7 @@ * * Copyright (C) Andrew Tridgell 1996 * Copyright (C) Paul Mackerras 1996 - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,14 +21,6 @@ #include "rsync.h" -int remote_protocol = 0; -int file_extra_cnt = 0; /* count of file-list extras that everyone gets */ -int inc_recurse = 0; -int compat_flags = 0; -int use_safe_inc_flist = 0; -int want_xattr_optim = 0; -int proper_seed_order = 0; - extern int am_server; extern int am_sender; extern int local_server; @@ -40,6 +32,7 @@ extern int preallocate_files; extern int append_mode; extern int fuzzy_basis; extern int read_batch; +extern int write_batch; extern int delay_updates; extern int checksum_seed; extern int basis_dir_cnt; @@ -48,24 +41,40 @@ extern int protocol_version; extern int protect_args; extern int preserve_uid; extern int preserve_gid; +extern int preserve_atimes; extern int preserve_acls; extern int preserve_xattrs; +extern int xfer_flags_as_varint; extern int need_messages_from_generator; extern int delete_mode, delete_before, delete_during, delete_after; +extern int do_compression; +extern int do_compression_level; extern char *shell_cmd; extern char *partial_dir; -extern char *dest_option; extern char *files_from; extern char *filesfrom_host; +extern const char *checksum_choice; +extern const char *compress_choice; extern filter_rule_list filter_list; extern int need_unsorted_flist; #ifdef ICONV_OPTION extern iconv_t ic_send, ic_recv; extern char *iconv_opt; #endif +extern struct name_num_obj valid_checksums; + +int remote_protocol = 0; +int file_extra_cnt = 0; /* count of file-list extras that everyone gets */ +int inc_recurse = 0; +int compat_flags = 0; +int use_safe_inc_flist = 0; +int want_xattr_optim = 0; +int proper_seed_order = 0; +int inplace_partial = 0; +int do_negotiated_strings = 0; /* These index values are for the file-list's extra-attribute array. */ -int uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx; +int pathname_ndx, depth_ndx, atimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx; int receiver_symlink_times = 0; /* receiver can set the time on a symlink */ int sender_symlink_iconv = 0; /* sender should convert symlink content */ @@ -74,12 +83,31 @@ int sender_symlink_iconv = 0; /* sender should convert symlink content */ int filesfrom_convert = 0; #endif +#define MAX_NSTR_STRLEN 256 + +struct name_num_obj valid_compressions = { + "compress", NULL, NULL, 0, 0, { +#ifdef SUPPORT_ZSTD + { CPRES_ZSTD, "zstd", NULL }, +#endif +#ifdef SUPPORT_LZ4 + { CPRES_LZ4, "lz4", NULL }, +#endif + { CPRES_ZLIBX, "zlibx", NULL }, + { CPRES_ZLIB, "zlib", NULL }, + { CPRES_NONE, "none", NULL }, + { 0, NULL, NULL } + } +}; + #define CF_INC_RECURSE (1<<0) #define CF_SYMLINK_TIMES (1<<1) #define CF_SYMLINK_ICONV (1<<2) #define CF_SAFE_FLIST (1<<3) #define CF_AVOID_XATTR_OPTIM (1<<4) #define CF_CHKSUM_SEED_FIX (1<<5) +#define CF_INPLACE_PARTIAL_DIR (1<<6) +#define CF_VARINT_FLIST_FLAGS (1<<7) static const char *client_info; @@ -134,12 +162,308 @@ void set_allow_inc_recurse(void) allow_inc_recurse = 0; } +void parse_compress_choice(int final_call) +{ + if (valid_compressions.negotiated_name) + do_compression = valid_compressions.negotiated_num; + else if (compress_choice) { + struct name_num_item *nni = get_nni_by_name(&valid_compressions, compress_choice, -1); + if (!nni) { + rprintf(FERROR, "unknown compress name: %s\n", compress_choice); + exit_cleanup(RERR_UNSUPPORTED); + } + do_compression = nni->num; + } else if (do_compression) + do_compression = CPRES_ZLIB; + else + do_compression = CPRES_NONE; + + if (do_compression != CPRES_NONE && final_call) + init_compression_level(); /* There's a chance this might turn compression off! */ + + if (do_compression == CPRES_NONE) + compress_choice = NULL; + + /* Snag the compression name for both write_batch's option output & the following debug output. */ + if (valid_compressions.negotiated_name) + compress_choice = valid_compressions.negotiated_name; + else if (compress_choice == NULL) { + struct name_num_item *nni = get_nni_by_num(&valid_compressions, do_compression); + compress_choice = nni ? nni->name : "UNKNOWN"; + } + + if (final_call && DEBUG_GTE(NSTR, am_server ? 3 : 1) + && (do_compression != CPRES_NONE || do_compression_level != CLVL_NOT_SPECIFIED)) { + rprintf(FINFO, "%s%s compress: %s (level %d)\n", + am_server ? "Server" : "Client", + valid_compressions.negotiated_name ? " negotiated" : "", + compress_choice, do_compression_level); + } +} + +struct name_num_item *get_nni_by_name(struct name_num_obj *nno, const char *name, int len) +{ + struct name_num_item *nni; + + if (len < 0) + len = strlen(name); + + for (nni = nno->list; nni->name; nni++) { + if (strncasecmp(name, nni->name, len) == 0 && nni->name[len] == '\0') + return nni; + } + + return NULL; +} + +struct name_num_item *get_nni_by_num(struct name_num_obj *nno, int num) +{ + struct name_num_item *nni; + + for (nni = nno->list; nni->name; nni++) { + if (num == nni->num) + return nni; + } + + return NULL; +} + +static void init_nno_saw(struct name_num_obj *nno, int val) +{ + struct name_num_item *nni; + int cnt; + + if (!nno->saw_len) { + for (nni = nno->list; nni->name; nni++) { + if (nni->num >= nno->saw_len) + nno->saw_len = nni->num + 1; + } + } + + if (!nno->saw) { + if (!(nno->saw = new_array0(uchar, nno->saw_len))) + out_of_memory("init_nno_saw"); + + /* We'll take this opportunity to make sure that the main_name values are set right. */ + for (cnt = 1, nni = nno->list; nni->name; nni++, cnt++) { + if (nno->saw[nni->num]) + nni->main_name = nno->list[nno->saw[nni->num]-1].name; + else + nno->saw[nni->num] = cnt; + } + } + + memset(nno->saw, val, nno->saw_len); +} + +/* Simplify the user-provided string so that it contains valid names without any duplicates. + * It also sets the "saw" flags to a 1-relative count of which name was seen first. */ +static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf, int tobuf_len) +{ + char *to = tobuf, *tok = NULL; + int cnt = 0; + + while (1) { + if (*from == ' ' || !*from) { + if (tok) { + struct name_num_item *nni = get_nni_by_name(nno, tok, to - tok); + if (nni && !nno->saw[nni->num]) { + nno->saw[nni->num] = ++cnt; + if (nni->main_name) { + to = tok + strlcpy(tok, nni->main_name, tobuf_len - (tok - tobuf)); + if (to - tobuf >= tobuf_len) { + to = tok - 1; + break; + } + } + } else + to = tok - (tok != tobuf); + tok = NULL; + } + if (!*from++) + break; + continue; + } + if (!tok) { + if (to != tobuf) + *to++ = ' '; + tok = to; + } + if (to - tobuf >= tobuf_len - 1) { + to = tok - (tok != tobuf); + break; + } + *to++ = *from++; + } + *to = '\0'; + + return to - tobuf; +} + +static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf, int len) +{ + struct name_num_item *ret = NULL; + + if (len < 0) + len = read_vstring(f_in, tmpbuf, MAX_NSTR_STRLEN); + + if (DEBUG_GTE(NSTR, am_server ? 3 : 2)) { + if (am_server) + rprintf(FINFO, "Client %s list (on server): %s\n", nno->type, tmpbuf); + else + rprintf(FINFO, "Server %s list (on client): %s\n", nno->type, tmpbuf); + } + + if (len > 0) { + int best = nno->saw_len; /* We want best == 1 from the client list, so start with a big number. */ + char *tok; + if (am_server) + init_nno_saw(nno, 1); /* Since we're parsing client names, anything we parse first is #1. */ + for (tok = strtok(tmpbuf, " \t"); tok; tok = strtok(NULL, " \t")) { + struct name_num_item *nni = get_nni_by_name(nno, tok, -1); + if (!nni || !nno->saw[nni->num] || best <= nno->saw[nni->num]) + continue; + ret = nni; + best = nno->saw[nni->num]; + if (best == 1) + break; + } + if (ret) { + free(nno->saw); + nno->saw = NULL; + nno->negotiated_name = ret->main_name ? ret->main_name : ret->name; + nno->negotiated_num = ret->num; + return; + } + } + + if (!am_server) + rprintf(FERROR, "Failed to negotiate a common %s\n", nno->type); + exit_cleanup(RERR_UNSUPPORTED); +} + +/* The saw buffer is initialized and used to store ordinal values from 1 to N + * for the order of the args in the array. If dup_markup == '\0', duplicates + * are removed otherwise the char is prefixed to the duplicate term and, if it + * is an opening paren/bracket/brace, the matching closing char is suffixed. */ +int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup) +{ + struct name_num_item *nni; + int len = 0, cnt = 0; + char delim = '\0', post_delim; + + switch (dup_markup) { + case '(': post_delim = ')'; break; + case '[': post_delim = ']'; break; + case '{': post_delim = '}'; break; + default: post_delim = '\0'; break; + } + + init_nno_saw(nno, 0); + + for (nni = nno->list, len = 0; nni->name; nni++) { + if (nni->main_name) { + if (!dup_markup) + continue; + delim = dup_markup; + } + if (len) + to_buf[len++]= ' '; + if (delim) { + to_buf[len++]= delim; + delim = post_delim; + } + len += strlcpy(to_buf+len, nni->name, to_buf_len - len); + if (len >= to_buf_len - 3) + exit_cleanup(RERR_UNSUPPORTED); /* IMPOSSIBLE... */ + if (delim) { + to_buf[len++]= delim; + delim = '\0'; + } + nno->saw[nni->num] = ++cnt; + } + + return len; +} + +static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char *env_name) +{ + char tmpbuf[MAX_NSTR_STRLEN]; + const char *list_str = getenv(env_name); + int len, fail_if_empty = list_str && strstr(list_str, "FAIL"); + + if (!do_negotiated_strings) { + if (!am_server && fail_if_empty) { + rprintf(FERROR, "Remote rsync is too old for %s negotiation\n", nno->type); + exit_cleanup(RERR_UNSUPPORTED); + } + return; + } + + if (list_str && *list_str && (!am_server || local_server)) { + init_nno_saw(nno, 0); + len = parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN); + if (fail_if_empty && !len) + len = strlcpy(tmpbuf, "FAIL", MAX_NSTR_STRLEN); + list_str = tmpbuf; + } else + list_str = NULL; + + if (!list_str || !*list_str) + len = get_default_nno_list(nno, tmpbuf, MAX_NSTR_STRLEN, '\0'); + + if (DEBUG_GTE(NSTR, am_server ? 3 : 2)) { + if (am_server) + rprintf(FINFO, "Server %s list (on server): %s\n", nno->type, tmpbuf); + else + rprintf(FINFO, "Client %s list (on client): %s\n", nno->type, tmpbuf); + } + + if (local_server) { + /* A local server doesn't bother to send/recv the strings, it just constructs + * and parses the same string on both sides. */ + recv_negotiate_str(-1, nno, tmpbuf, len); + } else { + /* Each side sends their list of valid names to the other side and then both sides + * pick the first name in the client's list that is also in the server's list. */ + write_vstring(f_out, tmpbuf, len); + } +} + +static void negotiate_the_strings(int f_in, int f_out) +{ + /* We send all the negotiation strings before we start to read them to help avoid a slow startup. */ + + if (!checksum_choice) + send_negotiate_str(f_out, &valid_checksums, "RSYNC_CHECKSUM_LIST"); + + if (do_compression && !compress_choice) + send_negotiate_str(f_out, &valid_compressions, "RSYNC_COMPRESS_LIST"); + + if (valid_checksums.saw) { + char tmpbuf[MAX_NSTR_STRLEN]; + recv_negotiate_str(f_in, &valid_checksums, tmpbuf, -1); + } + + if (valid_compressions.saw) { + char tmpbuf[MAX_NSTR_STRLEN]; + recv_negotiate_str(f_in, &valid_compressions, tmpbuf, -1); + } +} + void setup_protocol(int f_out,int f_in) { - if (am_sender) - file_extra_cnt += PTR_EXTRA_CNT; + assert(file_extra_cnt == 0); + assert(EXTRA64_CNT == 2 || EXTRA64_CNT == 1); + + /* All int64 values must be set first so that they are guaranteed to be + * aligned for direct int64-pointer memory access. */ + if (preserve_atimes) + atimes_ndx = (file_extra_cnt += EXTRA64_CNT); + if (am_sender) /* This is most likely in the in64 union as well. */ + pathname_ndx = (file_extra_cnt += PTR_EXTRA_CNT); else - file_extra_cnt++; + depth_ndx = ++file_extra_cnt; if (preserve_uid) uid_ndx = ++file_extra_cnt; if (preserve_gid) @@ -207,16 +531,16 @@ void setup_protocol(int f_out,int f_in) append_mode = 2; if (preserve_acls && !local_server) { rprintf(FERROR, - "--acls requires protocol 30 or higher" - " (negotiated %d).\n", - protocol_version); + "--acls requires protocol 30 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } if (preserve_xattrs && !local_server) { rprintf(FERROR, - "--xattrs requires protocol 30 or higher" - " (negotiated %d).\n", - protocol_version); + "--xattrs requires protocol 30 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } } @@ -231,33 +555,33 @@ void setup_protocol(int f_out,int f_in) if (protocol_version < 29) { if (fuzzy_basis) { rprintf(FERROR, - "--fuzzy requires protocol 29 or higher" - " (negotiated %d).\n", - protocol_version); + "--fuzzy requires protocol 29 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } if (basis_dir_cnt && inplace) { rprintf(FERROR, - "%s with --inplace requires protocol 29 or higher" - " (negotiated %d).\n", - dest_option, protocol_version); + "%s with --inplace requires protocol 29 or higher" + " (negotiated %d).\n", + alt_dest_opt(0), protocol_version); exit_cleanup(RERR_PROTOCOL); } if (basis_dir_cnt > 1) { rprintf(FERROR, - "Using more than one %s option requires protocol" - " 29 or higher (negotiated %d).\n", - dest_option, protocol_version); + "Using more than one %s option requires protocol" + " 29 or higher (negotiated %d).\n", + alt_dest_opt(0), protocol_version); exit_cleanup(RERR_PROTOCOL); } if (prune_empty_dirs) { rprintf(FERROR, - "--prune-empty-dirs requires protocol 29 or higher" - " (negotiated %d).\n", - protocol_version); + "--prune-empty-dirs requires protocol 29 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } } else if (protocol_version >= 30) { @@ -275,13 +599,30 @@ void setup_protocol(int f_out,int f_in) compat_flags |= CF_AVOID_XATTR_OPTIM; if (local_server || strchr(client_info, 'C') != NULL) compat_flags |= CF_CHKSUM_SEED_FIX; - write_byte(f_out, compat_flags); - } else - compat_flags = read_byte(f_in); + if (local_server || strchr(client_info, 'I') != NULL) + compat_flags |= CF_INPLACE_PARTIAL_DIR; + if (local_server || strchr(client_info, 'v') != NULL) { + if (!write_batch || protocol_version >= 30) { + do_negotiated_strings = 1; + compat_flags |= CF_VARINT_FLIST_FLAGS; + } + } + if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */ + if (!write_batch) + compat_flags |= CF_VARINT_FLIST_FLAGS; + write_byte(f_out, compat_flags); + } else + write_varint(f_out, compat_flags); + } else { /* read_varint() is compatible with the older write_byte() when the 0x80 bit isn't on. */ + compat_flags = read_varint(f_in); + if (compat_flags & CF_VARINT_FLIST_FLAGS) + do_negotiated_strings = 1; + } /* The inc_recurse var MUST be set to 0 or 1. */ inc_recurse = compat_flags & CF_INC_RECURSE ? 1 : 0; want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM); proper_seed_order = compat_flags & CF_CHKSUM_SEED_FIX ? 1 : 0; + xfer_flags_as_varint = compat_flags & CF_VARINT_FLIST_FLAGS ? 1 : 0; if (am_sender) { receiver_symlink_times = am_server ? strchr(client_info, 'L') != NULL @@ -299,12 +640,14 @@ void setup_protocol(int f_out,int f_in) if (inc_recurse && !allow_inc_recurse) { /* This should only be able to happen in a batch. */ fprintf(stderr, - "Incompatible options specified for inc-recursive %s.\n", - read_batch ? "batch file" : "connection"); + "Incompatible options specified for inc-recursive %s.\n", + read_batch ? "batch file" : "connection"); exit_cleanup(RERR_SYNTAX); } use_safe_inc_flist = (compat_flags & CF_SAFE_FLIST) || protocol_version >= 31; need_messages_from_generator = 1; + if (compat_flags & CF_INPLACE_PARTIAL_DIR) + inplace_partial = 1; #ifdef CAN_SET_SYMLINK_TIMES } else if (!am_sender) { receiver_symlink_times = 1; @@ -331,6 +674,8 @@ void setup_protocol(int f_out,int f_in) } #endif + negotiate_the_strings(f_in, f_out); + if (am_server) { if (!checksum_seed) checksum_seed = time(NULL) ^ (getpid() << 6); @@ -339,5 +684,11 @@ void setup_protocol(int f_out,int f_in) checksum_seed = read_int(f_in); } + parse_checksum_choice(1); /* Sets checksum_type & xfersum_type */ + parse_compress_choice(1); /* Sets do_compression */ + + if (write_batch && !am_server) + write_batch_shell_file(); + init_flist(); } diff --git a/config.guess b/config.guess index 120cc0d2..92bfc33e 100644 --- a/config.guess +++ b/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2013 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2013-05-16' +timestamp='2020-04-26' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -15,7 +15,7 @@ timestamp='2013-05-16' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. +# along with this program; if not, see <https://www.gnu.org/licenses/>. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a @@ -24,12 +24,12 @@ timestamp='2013-05-16' # program. This Exception is an additional permission under section 7 # of the GNU General Public License, version 3 ("GPLv3"). # -# Originally written by Per Bothner. +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess # -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to <config-patches@gnu.org>. me=`echo "$0" | sed -e 's,.*/,,'` @@ -39,7 +39,7 @@ Usage: $0 [OPTION] Output the configuration name of the system \`$me' is run on. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2013 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -84,8 +84,6 @@ if test $# != 0; then exit 1 fi -trap 'exit 1' 1 2 15 - # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a @@ -96,34 +94,40 @@ trap 'exit 1' 1 2 15 # Portable tmp directory creation inspired by the Autoconf team. -set_cc_for_build=' -trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; -trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; -: ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; -dummy=$tmp/dummy ; -tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; -case $CC_FOR_BUILD,$HOST_CC,$CC in - ,,) echo "int x;" > $dummy.c ; - for c in cc gcc c89 c99 ; do - if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then - CC_FOR_BUILD="$c"; break ; - fi ; - done ; - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found ; - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ; set_cc_for_build= ;' +tmp= +# shellcheck disable=SC2172 +trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 + +set_cc_for_build() { + # prevent multiple calls if $tmp is already set + test "$tmp" && return 0 + : "${TMPDIR=/tmp}" + # shellcheck disable=SC2039 + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } + dummy=$tmp/dummy + case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in + ,,) echo "int x;" > "$dummy.c" + for driver in cc gcc c89 c99 ; do + if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then + CC_FOR_BUILD="$driver" + break + fi + done + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; + esac +} # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) -if (test -f /.attbin/uname) >/dev/null 2>&1 ; then +if test -f /.attbin/uname ; then PATH=$PATH:/.attbin ; export PATH fi @@ -132,14 +136,14 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown -case "${UNAME_SYSTEM}" in +case "$UNAME_SYSTEM" in Linux|GNU|GNU/*) # If the system lacks a compiler, then just pick glibc. # We could probably try harder. LIBC=gnu - eval $set_cc_for_build - cat <<-EOF > $dummy.c + set_cc_for_build + cat <<-EOF > "$dummy.c" #include <features.h> #if defined(__UCLIBC__) LIBC=uclibc @@ -149,13 +153,20 @@ Linux|GNU|GNU/*) LIBC=gnu #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + + # If ldd exists, use it to detect musl libc. + if command -v ldd >/dev/null && \ + ldd --version 2>&1 | grep -q ^musl + then + LIBC=musl + fi ;; esac # Note: order is significant - the case branches are not exclusive. -case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in +case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, @@ -168,21 +179,31 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` - case "${UNAME_MACHINE_ARCH}" in + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + "/sbin/$sysctl" 2>/dev/null || \ + "/usr/sbin/$sysctl" 2>/dev/null || \ + echo unknown)` + case "$UNAME_MACHINE_ARCH" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; - *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + earmv*) + arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` + machine="${arch}${endian}"-unknown + ;; + *) machine="$UNAME_MACHINE_ARCH"-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. - case "${UNAME_MACHINE_ARCH}" in + # to ELF recently (or will in the future) and ABI. + case "$UNAME_MACHINE_ARCH" in + earm*) + os=netbsdelf + ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) - eval $set_cc_for_build + set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then @@ -197,43 +218,72 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "$UNAME_MACHINE_ARCH" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. - case "${UNAME_VERSION}" in + case "$UNAME_VERSION" in Debian*) release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "$machine-${os}${release}${abi-}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" + exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" + exit ;; + *:MidnightBSD:*:*) + echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" exit ;; *:ekkoBSD:*:*) - echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" exit ;; *:SolidBSD:*:*) - echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" + exit ;; + *:OS108:*:*) + echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE" exit ;; macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd${UNAME_RELEASE} + echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:MirBSD:*:*) - echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" + exit ;; + *:Sortix:*:*) + echo "$UNAME_MACHINE"-unknown-sortix + exit ;; + *:Twizzler:*:*) + echo "$UNAME_MACHINE"-unknown-twizzler + exit ;; + *:Redox:*:*) + echo "$UNAME_MACHINE"-unknown-redox + exit ;; + mips:OSF1:*.*) + echo mips-dec-osf1 exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in @@ -251,63 +301,54 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 exit $exitcode ;; - Alpha\ *:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # Should we change UNAME_MACHINE based on the output of uname instead - # of the specific Alpha model? - echo alpha-pc-interix - exit ;; - 21064:Windows_NT:50:3) - echo alpha-dec-winnt3.5 - exit ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 exit ;; *:[Aa]miga[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-amigaos + echo "$UNAME_MACHINE"-unknown-amigaos exit ;; *:[Mm]orph[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-morphos + echo "$UNAME_MACHINE"-unknown-morphos exit ;; *:OS/390:*:*) echo i370-ibm-openedition @@ -319,7 +360,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix${UNAME_RELEASE} + echo arm-acorn-riscix"$UNAME_RELEASE" exit ;; arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos @@ -346,38 +387,38 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) - echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux${UNAME_RELEASE} + echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - eval $set_cc_for_build - SUN_ARCH="i386" + set_cc_for_build + SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then - SUN_ARCH="x86_64" + SUN_ARCH=x86_64 fi fi - echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$SUN_ARCH"-pc-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in @@ -386,25 +427,25 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" exit ;; sun3*:SunOS:*:*) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" ;; sun4) - echo sparc-sun-sunos${UNAME_RELEASE} + echo sparc-sun-sunos"$UNAME_RELEASE" ;; esac exit ;; aushp:SunOS:*:*) - echo sparc-auspex-sunos${UNAME_RELEASE} + echo sparc-auspex-sunos"$UNAME_RELEASE" exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not @@ -415,44 +456,44 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} + echo m68k-milan-mint"$UNAME_RELEASE" exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} + echo m68k-hades-mint"$UNAME_RELEASE" exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} + echo m68k-unknown-mint"$UNAME_RELEASE" exit ;; m68k:machten:*:*) - echo m68k-apple-machten${UNAME_RELEASE} + echo m68k-apple-machten"$UNAME_RELEASE" exit ;; powerpc:machten:*:*) - echo powerpc-apple-machten${UNAME_RELEASE} + echo powerpc-apple-machten"$UNAME_RELEASE" exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 exit ;; RISC*:ULTRIX:*:*) - echo mips-dec-ultrix${UNAME_RELEASE} + echo mips-dec-ultrix"$UNAME_RELEASE" exit ;; VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix${UNAME_RELEASE} + echo vax-dec-ultrix"$UNAME_RELEASE" exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix${UNAME_RELEASE} + echo clipper-intergraph-clix"$UNAME_RELEASE" exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #ifdef __cplusplus #include <stdio.h> /* for printf() prototype */ int main (int argc, char *argv[]) { @@ -461,23 +502,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in #endif #if defined (host_mips) && defined (MIPSEB) #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); #endif #endif exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && - dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`$dummy $dummyarg` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && + dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`"$dummy" "$dummyarg"` && { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos${UNAME_RELEASE} + echo mips-mips-riscos"$UNAME_RELEASE" exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax @@ -503,17 +544,17 @@ EOF AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] then - if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ - [ ${TARGET_BINARY_INTERFACE}x = x ] + if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ + [ "$TARGET_BINARY_INTERFACE"x = x ] then - echo m88k-dg-dgux${UNAME_RELEASE} + echo m88k-dg-dgux"$UNAME_RELEASE" else - echo m88k-dg-dguxbcs${UNAME_RELEASE} + echo m88k-dg-dguxbcs"$UNAME_RELEASE" fi else - echo i586-dg-dgux${UNAME_RELEASE} + echo i586-dg-dgux"$UNAME_RELEASE" fi exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) @@ -530,7 +571,7 @@ EOF echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) - echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id @@ -542,14 +583,14 @@ EOF if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include <sys/systemcfg.h> main() @@ -560,7 +601,7 @@ EOF exit(0); } EOF - if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` then echo "$SYSTEM_NAME" else @@ -574,26 +615,27 @@ EOF exit ;; *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` - if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${IBM_ARCH}-ibm-aix${IBM_REV} + echo "$IBM_ARCH"-ibm-aix"$IBM_REV" exit ;; *:AIX:*:*) echo rs6000-ibm-aix exit ;; - ibmrt:4.4BSD:*|romp-ibm:BSD:*) + ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) echo romp-ibm-bsd4.4 exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx @@ -608,28 +650,28 @@ EOF echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - case "${UNAME_MACHINE}" in - 9000/31? ) HP_ARCH=m68000 ;; - 9000/[34]?? ) HP_ARCH=m68k ;; + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + case "$UNAME_MACHINE" in + 9000/31?) HP_ARCH=m68000 ;; + 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + case "$sc_cpu_version" in + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + case "$sc_kernel_bits" in + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi - if [ "${HP_ARCH}" = "" ]; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + if [ "$HP_ARCH" = "" ]; then + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #define _HPUX_SOURCE #include <stdlib.h> @@ -662,13 +704,13 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if [ "$HP_ARCH" = hppa2.0w ] then - eval $set_cc_for_build + set_cc_for_build # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler @@ -679,23 +721,23 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi - echo ${HP_ARCH}-hp-hpux${HPUX_REV} + echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - echo ia64-hp-hpux${HPUX_REV} + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" #include <unistd.h> int main () @@ -720,11 +762,11 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) echo hppa1.1-hp-bsd exit ;; 9000/8??:4.3bsd:*:*) @@ -733,7 +775,7 @@ EOF *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix exit ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) echo hppa1.1-hp-osf exit ;; hp8??:OSF1:*:*) @@ -741,9 +783,9 @@ EOF exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then - echo ${UNAME_MACHINE}-unknown-osf1mk + echo "$UNAME_MACHINE"-unknown-osf1mk else - echo ${UNAME_MACHINE}-unknown-osf1 + echo "$UNAME_MACHINE"-unknown-osf1 fi exit ;; parisc*:Lites*:*:*) @@ -768,130 +810,123 @@ EOF echo c4-convex-bsd exit ;; CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*[A-Z]90:*:*:*) - echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) - echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*SV1:*:*:*) - echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" exit ;; sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi${UNAME_RELEASE} + echo sparc-unknown-bsdi"$UNAME_RELEASE" exit ;; *:BSD/OS:*:*) - echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" + exit ;; + arm:FreeBSD:*:*) + UNAME_PROCESSOR=`uname -p` + set_cc_for_build + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + else + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + fi exit ;; *:FreeBSD:*:*) UNAME_PROCESSOR=`/usr/bin/uname -p` - case ${UNAME_PROCESSOR} in + case "$UNAME_PROCESSOR" in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; i*:CYGWIN*:*) - echo ${UNAME_MACHINE}-pc-cygwin + echo "$UNAME_MACHINE"-pc-cygwin exit ;; *:MINGW64*:*) - echo ${UNAME_MACHINE}-pc-mingw64 + echo "$UNAME_MACHINE"-pc-mingw64 exit ;; *:MINGW*:*) - echo ${UNAME_MACHINE}-pc-mingw32 + echo "$UNAME_MACHINE"-pc-mingw32 exit ;; - i*:MSYS*:*) - echo ${UNAME_MACHINE}-pc-msys - exit ;; - i*:windows32*:*) - # uname -m includes "-pc" on this system. - echo ${UNAME_MACHINE}-mingw32 + *:MSYS*:*) + echo "$UNAME_MACHINE"-pc-msys exit ;; i*:PW*:*) - echo ${UNAME_MACHINE}-pc-pw32 + echo "$UNAME_MACHINE"-pc-pw32 exit ;; *:Interix*:*) - case ${UNAME_MACHINE} in + case "$UNAME_MACHINE" in x86) - echo i586-pc-interix${UNAME_RELEASE} + echo i586-pc-interix"$UNAME_RELEASE" exit ;; authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix${UNAME_RELEASE} + echo x86_64-unknown-interix"$UNAME_RELEASE" exit ;; IA64) - echo ia64-unknown-interix${UNAME_RELEASE} + echo ia64-unknown-interix"$UNAME_RELEASE" exit ;; esac ;; - [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) - echo i${UNAME_MACHINE}-pc-mks - exit ;; - 8664:Windows_NT:*) - echo x86_64-pc-mks - exit ;; - i*:Windows_NT*:* | Pentium*:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we - # UNAME_MACHINE based on the output of uname instead of i386? - echo i586-pc-interix - exit ;; i*:UWIN*:*) - echo ${UNAME_MACHINE}-pc-uwin + echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-unknown-cygwin - exit ;; - p*:CYGWIN*:*) - echo powerpcle-unknown-cygwin + echo x86_64-pc-cygwin exit ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" exit ;; - i*86:Minix:*:*) - echo ${UNAME_MACHINE}-pc-minix + *:Minix:*:*) + echo "$UNAME_MACHINE"-unknown-minix exit ;; aarch64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null` in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; @@ -901,123 +936,179 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="gnulibc1" ; fi - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arc:Linux:*:* | arceb:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arm*:Linux:*:*) - eval $set_cc_for_build + set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi else - echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf fi fi exit ;; avr32*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; cris:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-${LIBC} + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; crisv32:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-${LIBC} + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" + exit ;; + e2k:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; frv:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; hexagon:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:Linux:*:*) - echo ${UNAME_MACHINE}-pc-linux-${LIBC} + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + k1om:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m32r*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; mips:Linux:*:* | mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + set_cc_for_build + IS_GLIBC=0 + test x"${LIBC}" = xgnu && IS_GLIBC=1 + sed 's/^ //' << EOF > "$dummy.c" #undef CPU - #undef ${UNAME_MACHINE} - #undef ${UNAME_MACHINE}el + #undef mips + #undef mipsel + #undef mips64 + #undef mips64el + #if ${IS_GLIBC} && defined(_ABI64) + LIBCABI=gnuabi64 + #else + #if ${IS_GLIBC} && defined(_ABIN32) + LIBCABI=gnuabin32 + #else + LIBCABI=${LIBC} + #endif + #endif + + #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa64r6 + #else + #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa32r6 + #else + #if defined(__mips64) + CPU=mips64 + #else + CPU=mips + #endif + #endif + #endif + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=${UNAME_MACHINE}el + MIPS_ENDIAN=el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=${UNAME_MACHINE} + MIPS_ENDIAN= #else - CPU= + MIPS_ENDIAN= #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` - test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI'`" + test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } ;; - or1k:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + mips64el:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; - or32:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + openrisc*:Linux:*:*) + echo or1k-unknown-linux-"$LIBC" + exit ;; + or32:Linux:*:* | or1k*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; padre:Linux:*:*) - echo sparc-unknown-linux-${LIBC} + echo sparc-unknown-linux-"$LIBC" exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-${LIBC} + echo hppa64-unknown-linux-"$LIBC" exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; - PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; - *) echo hppa-unknown-linux-${LIBC} ;; + PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; + PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; + *) echo hppa-unknown-linux-"$LIBC" ;; esac exit ;; ppc64:Linux:*:*) - echo powerpc64-unknown-linux-${LIBC} + echo powerpc64-unknown-linux-"$LIBC" exit ;; ppc:Linux:*:*) - echo powerpc-unknown-linux-${LIBC} + echo powerpc-unknown-linux-"$LIBC" + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-"$LIBC" + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-"$LIBC" + exit ;; + riscv32:Linux:*:* | riscv64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux-${LIBC} + echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; tile*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; vax:Linux:*:*) - echo ${UNAME_MACHINE}-dec-linux-${LIBC} + echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + set_cc_for_build + LIBCABI=$LIBC + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_X32 >/dev/null + then + LIBCABI="$LIBC"x32 + fi + fi + echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI" exit ;; xtensa*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. @@ -1031,34 +1122,34 @@ EOF # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. - echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. - echo ${UNAME_MACHINE}-pc-os2-emx + echo "$UNAME_MACHINE"-pc-os2-emx exit ;; i*86:XTS-300:*:STOP) - echo ${UNAME_MACHINE}-unknown-stop + echo "$UNAME_MACHINE"-unknown-stop exit ;; i*86:atheos:*:*) - echo ${UNAME_MACHINE}-unknown-atheos + echo "$UNAME_MACHINE"-unknown-atheos exit ;; i*86:syllable:*:*) - echo ${UNAME_MACHINE}-pc-syllable + echo "$UNAME_MACHINE"-pc-syllable exit ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos${UNAME_RELEASE} + echo i386-unknown-lynxos"$UNAME_RELEASE" exit ;; i*86:*DOS:*:*) - echo ${UNAME_MACHINE}-pc-msdosdjgpp + echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; - i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) - UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + i*86:*:4.*:*) + UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" fi exit ;; i*86:*:5:[678]*) @@ -1068,12 +1159,12 @@ EOF *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name` - echo ${UNAME_MACHINE}-pc-isc$UNAME_REL + echo "$UNAME_MACHINE"-pc-isc"$UNAME_REL" elif /bin/uname -X 2>/dev/null >/dev/null ; then UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 @@ -1083,9 +1174,9 @@ EOF && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 - echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv32 + echo "$UNAME_MACHINE"-pc-sysv32 fi exit ;; pc:*:*:*) @@ -1093,7 +1184,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1105,9 +1196,9 @@ EOF exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 fi exit ;; mini*:CTIX:SYS*5:*) @@ -1127,9 +1218,9 @@ EOF test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4; exit; } ;; @@ -1138,28 +1229,28 @@ EOF test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos${UNAME_RELEASE} + echo m68k-unknown-lynxos"$UNAME_RELEASE" exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 exit ;; TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos${UNAME_RELEASE} + echo sparc-unknown-lynxos"$UNAME_RELEASE" exit ;; rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos${UNAME_RELEASE} + echo rs6000-unknown-lynxos"$UNAME_RELEASE" exit ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos${UNAME_RELEASE} + echo powerpc-unknown-lynxos"$UNAME_RELEASE" exit ;; SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv${UNAME_RELEASE} + echo mips-dde-sysv"$UNAME_RELEASE" exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 @@ -1170,7 +1261,7 @@ EOF *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` - echo ${UNAME_MACHINE}-sni-sysv4 + echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv fi @@ -1190,23 +1281,23 @@ EOF exit ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. - echo ${UNAME_MACHINE}-stratus-vos + echo "$UNAME_MACHINE"-stratus-vos exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos exit ;; mc68*:A/UX:*:*) - echo m68k-apple-aux${UNAME_RELEASE} + echo m68k-apple-aux"$UNAME_RELEASE" exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv"$UNAME_RELEASE" else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv"$UNAME_RELEASE" fi exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. @@ -1225,67 +1316,94 @@ EOF echo x86_64-unknown-haiku exit ;; SX-4:SUPER-UX:*:*) - echo sx4-nec-superux${UNAME_RELEASE} + echo sx4-nec-superux"$UNAME_RELEASE" exit ;; SX-5:SUPER-UX:*:*) - echo sx5-nec-superux${UNAME_RELEASE} + echo sx5-nec-superux"$UNAME_RELEASE" exit ;; SX-6:SUPER-UX:*:*) - echo sx6-nec-superux${UNAME_RELEASE} + echo sx6-nec-superux"$UNAME_RELEASE" exit ;; SX-7:SUPER-UX:*:*) - echo sx7-nec-superux${UNAME_RELEASE} + echo sx7-nec-superux"$UNAME_RELEASE" exit ;; SX-8:SUPER-UX:*:*) - echo sx8-nec-superux${UNAME_RELEASE} + echo sx8-nec-superux"$UNAME_RELEASE" exit ;; SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux${UNAME_RELEASE} + echo sx8r-nec-superux"$UNAME_RELEASE" + exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux"$UNAME_RELEASE" exit ;; Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody${UNAME_RELEASE} + echo powerpc-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Rhapsody:*:*) - echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Darwin:*:*) - UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - eval $set_cc_for_build - if test "$UNAME_PROCESSOR" = unknown ; then - UNAME_PROCESSOR=powerpc + UNAME_PROCESSOR=`uname -p` + case $UNAME_PROCESSOR in + unknown) UNAME_PROCESSOR=powerpc ;; + esac + if command -v xcode-select > /dev/null 2> /dev/null && \ + ! xcode-select --print-path > /dev/null 2> /dev/null ; then + # Avoid executing cc if there is no toolchain installed as + # cc will be a stub that puts up a graphical alert + # prompting the user to install developer tools. + CC_FOR_BUILD=no_compiler_found + else + set_cc_for_build fi - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null then case $UNAME_PROCESSOR in i386) UNAME_PROCESSOR=x86_64 ;; powerpc) UNAME_PROCESSOR=powerpc64 ;; esac fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc + fi + elif test "$UNAME_PROCESSOR" = i386 ; then + # uname -m returns i386 or x86_64 + UNAME_PROCESSOR=$UNAME_MACHINE fi - echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi - echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" exit ;; *:QNX:*:4*) echo i386-pc-qnx exit ;; - NEO-?:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk${UNAME_RELEASE} + NEO-*:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk"$UNAME_RELEASE" exit ;; NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk${UNAME_RELEASE} + echo nse-tandem-nsk"$UNAME_RELEASE" exit ;; - NSR-?:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk${UNAME_RELEASE} + NSR-*:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSV-*:NONSTOP_KERNEL:*:*) + echo nsv-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk"$UNAME_RELEASE" exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux @@ -1294,18 +1412,19 @@ EOF echo bs2000-siemens-sysv exit ;; DS/*:UNIX_System_V:*:*) - echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + # shellcheck disable=SC2154 + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" fi - echo ${UNAME_MACHINE}-unknown-plan9 + echo "$UNAME_MACHINE"-unknown-plan9 exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 @@ -1326,14 +1445,14 @@ EOF echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} + echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) - echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; *:*VMS:*:*) UNAME_MACHINE=`(uname -p) 2>/dev/null` - case "${UNAME_MACHINE}" in + case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; V*) echo vax-dec-vms ; exit ;; @@ -1342,24 +1461,39 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" exit ;; i*86:rdos:*:*) - echo ${UNAME_MACHINE}-pc-rdos + echo "$UNAME_MACHINE"-pc-rdos exit ;; i*86:AROS:*:*) - echo ${UNAME_MACHINE}-pc-aros + echo "$UNAME_MACHINE"-pc-aros exit ;; x86_64:VMkernel:*:*) - echo ${UNAME_MACHINE}-unknown-esx + echo "$UNAME_MACHINE"-unknown-esx + exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs + exit ;; + *:Unleashed:*:*) + echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" exit ;; esac -eval $set_cc_for_build -cat >$dummy.c <<EOF +# No uname command or uname output not recognized. +set_cc_for_build +cat > "$dummy.c" <<EOF #ifdef _SEQUENT_ -# include <sys/types.h> -# include <sys/utsname.h> +#include <sys/types.h> +#include <sys/utsname.h> +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#include <signal.h> +#if defined(_SIZE_T_) || defined(SIGLOST) +#include <sys/utsname.h> +#endif +#endif #endif main () { @@ -1372,22 +1506,14 @@ main () #include <sys/param.h> printf ("m68k-sony-newsos%s\n", #ifdef NEWSOS4 - "4" + "4" #else - "" + "" #endif - ); exit (0); + ); exit (0); #endif #endif -#if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix\n"); exit (0); -#endif - -#if defined (hp300) && !defined (hpux) - printf ("m68k-hp-bsd\n"); exit (0); -#endif - #if defined (NeXT) #if !defined (__ARCHITECTURE__) #define __ARCHITECTURE__ "m68k" @@ -1427,39 +1553,54 @@ main () #endif #if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); + struct utsname un; + uname(&un); + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); #endif #if defined (vax) -# if !defined (ultrix) -# include <sys/param.h> -# if defined (BSD) -# if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -# else -# if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# endif -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# else - printf ("vax-dec-ultrix\n"); exit (0); -# endif +#if !defined (ultrix) +#include <sys/param.h> +#if defined (BSD) +#if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +#else +#if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#endif +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#else +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname un; + uname (&un); + printf ("vax-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("vax-dec-ultrix\n"); exit (0); +#endif +#endif +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname *un; + uname (&un); + printf ("mips-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("mips-dec-ultrix\n"); exit (0); +#endif +#endif #endif #if defined (alliant) && defined (i860) @@ -1470,54 +1611,44 @@ main () } EOF -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && +$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=`$dummy` && { echo "$SYSTEM_NAME"; exit; } # Apollos put the system type in the environment. +test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } +echo "$0: unable to guess system type" >&2 -# Convex versions that predate uname can use getsysinfo(1) +case "$UNAME_MACHINE:$UNAME_SYSTEM" in + mips:Linux | mips64:Linux) + # If we got here on MIPS GNU/Linux, output extra information. + cat >&2 <<EOF -if [ -x /usr/convex/getsysinfo ] -then - case `getsysinfo -f cpu_type` in - c1*) - echo c1-convex-bsd - exit ;; - c2*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - c34*) - echo c34-convex-bsd - exit ;; - c38*) - echo c38-convex-bsd - exit ;; - c4*) - echo c4-convex-bsd - exit ;; - esac -fi +NOTE: MIPS GNU/Linux systems require a C compiler to fully recognize +the system type. Please install a C compiler and try again. +EOF + ;; +esac cat >&2 <<EOF -$0: unable to guess system type -This script, last modified $timestamp, has failed to recognize -the operating system you are using. It is advised that you -download the most up to date version of the config scripts from +This script (version $timestamp), has failed to recognize the +operating system you are using. If your script is old, overwrite *all* +copies of config.guess and config.sub with the latest versions from: - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD + https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess and - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub +EOF + +year=`echo $timestamp | sed 's,-.*,,'` +# shellcheck disable=SC2003 +if test "`expr "\`date +%Y\`" - "$year"`" -lt 3 ; then + cat >&2 <<EOF -If the version you run ($0) is already up to date, please -send the following data and any information you think might be -pertinent to <config-patches@gnu.org> in order to provide the needed -information to handle your system. +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp @@ -1536,16 +1667,17 @@ hostinfo = `(hostinfo) 2>/dev/null` /usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` -UNAME_MACHINE = ${UNAME_MACHINE} -UNAME_RELEASE = ${UNAME_RELEASE} -UNAME_SYSTEM = ${UNAME_SYSTEM} -UNAME_VERSION = ${UNAME_VERSION} +UNAME_MACHINE = "$UNAME_MACHINE" +UNAME_RELEASE = "$UNAME_RELEASE" +UNAME_SYSTEM = "$UNAME_SYSTEM" +UNAME_VERSION = "$UNAME_VERSION" EOF +fi exit 1 # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/config.h.in b/config.h.in index 03d66484..6a37faf2 100644 --- a/config.h.in +++ b/config.h.in @@ -76,6 +76,9 @@ /* Define to 1 if vsprintf has a C99-compatible return value */ #undef HAVE_C99_VSNPRINTF +/* Define to 1 if you have the `chflags' function. */ +#undef HAVE_CHFLAGS + /* Define to 1 if you have the `chmod' function. */ #undef HAVE_CHMOD @@ -135,6 +138,9 @@ /* Define to 1 if you have the "getaddrinfo" function and required types. */ #undef HAVE_GETADDRINFO +/* Define to 1 if you have the `getattrlist' function. */ +#undef HAVE_GETATTRLIST + /* Define to 1 if you have the `getcwd' function. */ #undef HAVE_GETCWD @@ -266,6 +272,9 @@ /* Define to 1 if you have the `lutimes' function. */ #undef HAVE_LUTIMES +/* Define to 1 if you have the <lz4.h> header file. */ +#undef HAVE_LZ4_H + /* Define to 1 if you have the `mallinfo' function. */ #undef HAVE_MALLINFO @@ -296,6 +305,9 @@ /* Define to 1 if you have the `mtrace' function. */ #undef HAVE_MTRACE +/* Define to 1 if you have the `nanosleep' function. */ +#undef HAVE_NANOSLEEP + /* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */ #undef HAVE_NDIR_H @@ -317,6 +329,12 @@ /* Define to 1 if you have the `open64' function. */ #undef HAVE_OPEN64 +/* Define to 1 if you have the <openssl/md4.h> header file. */ +#undef HAVE_OPENSSL_MD4_H + +/* Define to 1 if you have the <openssl/md5.h> header file. */ +#undef HAVE_OPENSSL_MD5_H + /* true if you have Mac OS X ACLs */ #undef HAVE_OSX_ACLS @@ -377,6 +395,9 @@ /* Define to 1 if you have the `sigprocmask' function. */ #undef HAVE_SIGPROCMASK +/* Define to 1 to enable SIMD optimizations */ +#undef HAVE_SIMD + /* Define to 1 if the system has the type `size_t'. */ #undef HAVE_SIZE_T @@ -574,9 +595,15 @@ /* Define to 1 if you have the `waitpid' function. */ #undef HAVE_WAITPID +/* Define to 1 if you have the <xxhash.h> header file. */ +#undef HAVE_XXHASH_H + /* Define to 1 if you have the <zlib.h> header file. */ #undef HAVE_ZLIB_H +/* Define to 1 if you have the <zstd.h> header file. */ +#undef HAVE_ZSTD_H + /* Define to 1 if you have the `_acl' function. */ #undef HAVE__ACL @@ -595,7 +622,7 @@ /* Define as const if the declaration of iconv() needs const. */ #undef ICONV_CONST -/* Define if you want the --iconv option. Specifing a value will set the +/* Define if you want the --iconv option. Specifying a value will set the default iconv setting (a NULL means no --iconv processing by default). */ #undef ICONV_OPTION @@ -676,6 +703,9 @@ /* Define to 1 if "signed char" is a valid type */ #undef SIGNED_CHAR_OK +/* The size of `char*', as computed by sizeof. */ +#undef SIZEOF_CHARP + /* The size of `int', as computed by sizeof. */ #undef SIZEOF_INT @@ -726,15 +756,30 @@ /* Define to 1 to add support for ACLs */ #undef SUPPORT_ACLS +/* Undefine if you do not want LZ4 compression. By default this is defined. */ +#undef SUPPORT_LZ4 + /* Define to 1 to add support for extended attributes */ #undef SUPPORT_XATTRS +/* Undefine if you do not want xxhash checksums. By default this is defined. + */ +#undef SUPPORT_XXHASH + +/* Undefine if you do not want zstd compression. By default this is defined. + */ +#undef SUPPORT_ZSTD + /* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */ #undef TIME_WITH_SYS_TIME /* Define to 1 if you want rsync to make use of iconv_open() */ #undef USE_ICONV_OPEN +/* Undefine if you do not want to use openssl crypto library. By default this + is defined. */ +#undef USE_OPENSSL + /* String to pass to iconv() for the UTF-8 charset. */ #undef UTF8_CHARSET @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2013 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2013-04-24' +timestamp='2020-05-04' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -15,7 +15,7 @@ timestamp='2013-04-24' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, see <http://www.gnu.org/licenses/>. +# along with this program; if not, see <https://www.gnu.org/licenses/>. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a @@ -25,7 +25,7 @@ timestamp='2013-04-24' # of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to <config-patches@gnu.org>. # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. @@ -33,7 +33,7 @@ timestamp='2013-04-24' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -53,12 +53,11 @@ timestamp='2013-04-24' me=`echo "$0" | sed -e 's,.*/,,'` usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. -Operation modes: +Options: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit @@ -68,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright 1992-2013 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -90,12 +89,12 @@ while test $# -gt 0 ; do - ) # Use stdin as input. break ;; -* ) - echo "$me: invalid option $1$help" + echo "$me: invalid option $1$help" >&2 exit 1 ;; *local*) # First pass through any local machine types. - echo $1 + echo "$1" exit ;; * ) @@ -111,1209 +110,1164 @@ case $# in exit 1;; esac -# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). -# Here we must recognize all the valid KERNEL-OS combinations. -maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` -case $maybe_os in - nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ - linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ - kopensolaris*-gnu* | \ - storm-chaos* | os2-emx* | rtmk-nova*) - os=-$maybe_os - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` - ;; - android-linux) - os=-linux-android - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown - ;; - *) - basic_machine=`echo $1 | sed 's/-[^-]*$//'` - if [ $basic_machine != $1 ] - then os=`echo $1 | sed 's/.*-/-/'` - else os=; fi - ;; -esac +# Split fields of configuration type +# shellcheck disable=SC2162 +IFS="-" read field1 field2 field3 field4 <<EOF +$1 +EOF -### Let's recognize common machines as not being operating systems so -### that things like config.sub decstation-3100 work. We also -### recognize some manufacturers as not being operating systems, so we -### can provide default operating systems below. -case $os in - -sun*os*) - # Prevent following clause from handling this invalid input. - ;; - -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ - -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ - -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ - -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ - -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ - -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis | -knuth | -cray | -microblaze*) - os= - basic_machine=$1 - ;; - -bluegene*) - os=-cnk - ;; - -sim | -cisco | -oki | -wec | -winbond) - os= - basic_machine=$1 - ;; - -scout) - ;; - -wrs) - os=-vxworks - basic_machine=$1 - ;; - -chorusos*) - os=-chorusos - basic_machine=$1 - ;; - -chorusrdb) - os=-chorusrdb - basic_machine=$1 - ;; - -hiux*) - os=-hiuxwe2 - ;; - -sco6) - os=-sco5v6 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco5) - os=-sco3.2v5 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco4) - os=-sco3.2v4 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2.[4-9]*) - os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2v[4-9]*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco5v6*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco*) - os=-sco3.2v2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -udk*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -isc) - os=-isc2.2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -clix*) - basic_machine=clipper-intergraph - ;; - -isc*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -lynx*178) - os=-lynxos178 - ;; - -lynx*5) - os=-lynxos5 - ;; - -lynx*) - os=-lynxos +# Separate into logical components for further validation +case $1 in + *-*-*-*-*) + echo Invalid configuration \`"$1"\': more than four components >&2 + exit 1 ;; - -ptx*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + *-*-*-*) + basic_machine=$field1-$field2 + os=$field3-$field4 ;; - -windowsnt*) - os=`echo $os | sed -e 's/windowsnt/winnt/'` + *-*-*) + # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two + # parts + maybe_os=$field2-$field3 + case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \ + | linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \ + | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ + | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ + | storm-chaos* | os2-emx* | rtmk-nova*) + basic_machine=$field1 + os=$maybe_os + ;; + android-linux) + basic_machine=$field1-unknown + os=linux-android + ;; + *) + basic_machine=$field1-$field2 + os=$field3 + ;; + esac ;; - -psos*) - os=-psos + *-*) + # A lone config we happen to match not fitting any pattern + case $field1-$field2 in + decstation-3100) + basic_machine=mips-dec + os= + ;; + *-*) + # Second component is usually, but not always the OS + case $field2 in + # Prevent following clause from handling this valid os + sun*os*) + basic_machine=$field1 + os=$field2 + ;; + # Manufacturers + dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ + | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ + | unicom* | ibm* | next | hp | isi* | apollo | altos* \ + | convergent* | ncr* | news | 32* | 3600* | 3100* \ + | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ + | ultra | tti* | harris | dolphin | highlevel | gould \ + | cbm | ns | masscomp | apple | axis | knuth | cray \ + | microblaze* | sim | cisco \ + | oki | wec | wrs | winbond) + basic_machine=$field1-$field2 + os= + ;; + *) + basic_machine=$field1 + os=$field2 + ;; + esac + ;; + esac ;; - -mint | -mint[0-9]*) - basic_machine=m68k-atari - os=-mint + *) + # Convert single-component short-hands not valid as part of + # multi-component configurations. + case $field1 in + 386bsd) + basic_machine=i386-pc + os=bsd + ;; + a29khif) + basic_machine=a29k-amd + os=udi + ;; + adobe68k) + basic_machine=m68010-adobe + os=scout + ;; + alliant) + basic_machine=fx80-alliant + os= + ;; + altos | altos3068) + basic_machine=m68k-altos + os= + ;; + am29k) + basic_machine=a29k-none + os=bsd + ;; + amdahl) + basic_machine=580-amdahl + os=sysv + ;; + amiga) + basic_machine=m68k-unknown + os= + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=bsd + ;; + aros) + basic_machine=i386-pc + os=aros + ;; + aux) + basic_machine=m68k-apple + os=aux + ;; + balance) + basic_machine=ns32k-sequent + os=dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=linux + ;; + cegcc) + basic_machine=arm-unknown + os=cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=bsd + ;; + convex-c2) + basic_machine=c2-convex + os=bsd + ;; + convex-c32) + basic_machine=c32-convex + os=bsd + ;; + convex-c34) + basic_machine=c34-convex + os=bsd + ;; + convex-c38) + basic_machine=c38-convex + os=bsd + ;; + cray) + basic_machine=j90-cray + os=unicos + ;; + crds | unos) + basic_machine=m68k-crds + os= + ;; + da30) + basic_machine=m68k-da30 + os= + ;; + decstation | pmax | pmin | dec3100 | decstatn) + basic_machine=mips-dec + os= + ;; + delta88) + basic_machine=m88k-motorola + os=sysv3 + ;; + dicos) + basic_machine=i686-pc + os=dicos + ;; + djgpp) + basic_machine=i586-pc + os=msdosdjgpp + ;; + ebmon29k) + basic_machine=a29k-amd + os=ebmon + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=ose + ;; + gmicro) + basic_machine=tron-gmicro + os=sysv + ;; + go32) + basic_machine=i386-pc + os=go32 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=hms + ;; + harris) + basic_machine=m88k-harris + os=sysv3 + ;; + hp300 | hp300hpux) + basic_machine=m68k-hp + os=hpux + ;; + hp300bsd) + basic_machine=m68k-hp + os=bsd + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=proelf + ;; + i386mach) + basic_machine=i386-mach + os=mach + ;; + isi68 | isi) + basic_machine=m68k-isi + os=sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=linux + ;; + magnum | m3230) + basic_machine=mips-mips + os=sysv + ;; + merlin) + basic_machine=ns32k-utek + os=sysv + ;; + mingw64) + basic_machine=x86_64-pc + os=mingw64 + ;; + mingw32) + basic_machine=i686-pc + os=mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=mingw32ce + ;; + monitor) + basic_machine=m68k-rom68k + os=coff + ;; + morphos) + basic_machine=powerpc-unknown + os=morphos + ;; + moxiebox) + basic_machine=moxie-unknown + os=moxiebox + ;; + msdos) + basic_machine=i386-pc + os=msdos + ;; + msys) + basic_machine=i686-pc + os=msys + ;; + mvs) + basic_machine=i370-ibm + os=mvs + ;; + nacl) + basic_machine=le32-unknown + os=nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=sysv4 + ;; + netbsd386) + basic_machine=i386-pc + os=netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=newsos + ;; + news1000) + basic_machine=m68030-sony + os=newsos + ;; + necv70) + basic_machine=v70-nec + os=sysv + ;; + nh3000) + basic_machine=m68k-harris + os=cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=cxux + ;; + nindy960) + basic_machine=i960-intel + os=nindy + ;; + mon960) + basic_machine=i960-intel + os=mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=nonstopux + ;; + os400) + basic_machine=powerpc-ibm + os=os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=ose + ;; + os68k) + basic_machine=m68k-none + os=os68k + ;; + paragon) + basic_machine=i860-intel + os=osf + ;; + parisc) + basic_machine=hppa-unknown + os=linux + ;; + pw32) + basic_machine=i586-unknown + os=pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=rdos + ;; + rdos32) + basic_machine=i386-pc + os=rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=coff + ;; + sa29200) + basic_machine=a29k-amd + os=udi + ;; + sei) + basic_machine=mips-sei + os=seiux + ;; + sequent) + basic_machine=i386-sequent + os= + ;; + sps7) + basic_machine=m68k-bull + os=sysv2 + ;; + st2000) + basic_machine=m68k-tandem + os= + ;; + stratus) + basic_machine=i860-stratus + os=sysv4 + ;; + sun2) + basic_machine=m68000-sun + os= + ;; + sun2os3) + basic_machine=m68000-sun + os=sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=sunos4 + ;; + sun3) + basic_machine=m68k-sun + os= + ;; + sun3os3) + basic_machine=m68k-sun + os=sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=sunos4 + ;; + sun4) + basic_machine=sparc-sun + os= + ;; + sun4os3) + basic_machine=sparc-sun + os=sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=solaris2 + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + os= + ;; + sv1) + basic_machine=sv1-cray + os=unicos + ;; + symmetry) + basic_machine=i386-sequent + os=dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=unicos + ;; + t90) + basic_machine=t90-cray + os=unicos + ;; + toad1) + basic_machine=pdp10-xkl + os=tops20 + ;; + tpf) + basic_machine=s390x-ibm + os=tpf + ;; + udi29k) + basic_machine=a29k-amd + os=udi + ;; + ultra3) + basic_machine=a29k-nyu + os=sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=none + ;; + vaxv) + basic_machine=vax-dec + os=sysv + ;; + vms) + basic_machine=vax-dec + os=vms + ;; + vsta) + basic_machine=i386-pc + os=vsta + ;; + vxworks960) + basic_machine=i960-wrs + os=vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=vxworks + ;; + xbox) + basic_machine=i686-pc + os=mingw32 + ;; + ymp) + basic_machine=ymp-cray + os=unicos + ;; + *) + basic_machine=$1 + os= + ;; + esac ;; esac -# Decode aliases for certain CPU-COMPANY combinations. +# Decode 1-component or ad-hoc basic machines case $basic_machine in - # Recognize the basic CPU types without company name. - # Some are omitted here because they have special meanings below. - 1750a | 580 \ - | a29k \ - | aarch64 | aarch64_be \ - | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ - | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ - | am33_2.0 \ - | arc | arceb \ - | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ - | avr | avr32 \ - | be32 | be64 \ - | bfin \ - | c4x | clipper \ - | d10v | d30v | dlx | dsp16xx \ - | epiphany \ - | fido | fr30 | frv \ - | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ - | hexagon \ - | i370 | i860 | i960 | ia64 \ - | ip2k | iq2000 \ - | le32 | le64 \ - | lm32 \ - | m32c | m32r | m32rle | m68000 | m68k | m88k \ - | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ - | mn10200 | mn10300 \ - | moxie \ - | mt \ - | msp430 \ - | nds32 | nds32le | nds32be \ - | nios | nios2 | nios2eb | nios2el \ - | ns16k | ns32k \ - | open8 \ - | or1k | or32 \ - | pdp10 | pdp11 | pj | pjl \ - | powerpc | powerpc64 | powerpc64le | powerpcle \ - | pyramid \ - | rl78 | rx \ - | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ - | sh64 | sh64le \ - | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ - | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ - | spu \ - | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ - | ubicom32 \ - | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ - | we32k \ - | x86 | xc16x | xstormy16 | xtensa \ - | z8k | z80) - basic_machine=$basic_machine-unknown - ;; - c54x) - basic_machine=tic54x-unknown - ;; - c55x) - basic_machine=tic55x-unknown - ;; - c6x) - basic_machine=tic6x-unknown - ;; - m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip) - basic_machine=$basic_machine-unknown - os=-none - ;; - m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) - ;; - ms1) - basic_machine=mt-unknown + # Here we handle the default manufacturer of certain CPU types. It is in + # some cases the only manufacturer, in others, it is the most popular. + w89k) + cpu=hppa1.1 + vendor=winbond ;; - - strongarm | thumb | xscale) - basic_machine=arm-unknown + op50n) + cpu=hppa1.1 + vendor=oki ;; - xgate) - basic_machine=$basic_machine-unknown - os=-none + op60c) + cpu=hppa1.1 + vendor=oki ;; - xscaleeb) - basic_machine=armeb-unknown + ibm*) + cpu=i370 + vendor=ibm ;; - - xscaleel) - basic_machine=armel-unknown + orion105) + cpu=clipper + vendor=highlevel ;; - - # We use `pc' rather than `unknown' - # because (1) that's what they normally are, and - # (2) the word "unknown" tends to confuse beginning users. - i*86 | x86_64) - basic_machine=$basic_machine-pc - ;; - # Object if more than one company name word. - *-*-*) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 + mac | mpw | mac-mpw) + cpu=m68k + vendor=apple ;; - # Recognize the basic CPU types with company name. - 580-* \ - | a29k-* \ - | aarch64-* | aarch64_be-* \ - | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ - | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ - | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ - | avr-* | avr32-* \ - | be32-* | be64-* \ - | bfin-* | bs2000-* \ - | c[123]* | c30-* | [cjt]90-* | c4x-* \ - | clipper-* | craynv-* | cydra-* \ - | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ - | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ - | h8300-* | h8500-* \ - | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ - | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ - | ip2k-* | iq2000-* \ - | le32-* | le64-* \ - | lm32-* \ - | m32c-* | m32r-* | m32rle-* \ - | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ - | microblaze-* | microblazeel-* \ - | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ - | mips16-* \ - | mips64-* | mips64el-* \ - | mips64octeon-* | mips64octeonel-* \ - | mips64orion-* | mips64orionel-* \ - | mips64r5900-* | mips64r5900el-* \ - | mips64vr-* | mips64vrel-* \ - | mips64vr4100-* | mips64vr4100el-* \ - | mips64vr4300-* | mips64vr4300el-* \ - | mips64vr5000-* | mips64vr5000el-* \ - | mips64vr5900-* | mips64vr5900el-* \ - | mipsisa32-* | mipsisa32el-* \ - | mipsisa32r2-* | mipsisa32r2el-* \ - | mipsisa64-* | mipsisa64el-* \ - | mipsisa64r2-* | mipsisa64r2el-* \ - | mipsisa64sb1-* | mipsisa64sb1el-* \ - | mipsisa64sr71k-* | mipsisa64sr71kel-* \ - | mipsr5900-* | mipsr5900el-* \ - | mipstx39-* | mipstx39el-* \ - | mmix-* \ - | mt-* \ - | msp430-* \ - | nds32-* | nds32le-* | nds32be-* \ - | nios-* | nios2-* | nios2eb-* | nios2el-* \ - | none-* | np1-* | ns16k-* | ns32k-* \ - | open8-* \ - | orion-* \ - | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ - | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ - | pyramid-* \ - | rl78-* | romp-* | rs6000-* | rx-* \ - | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ - | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ - | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ - | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ - | tahoe-* \ - | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ - | tile*-* \ - | tron-* \ - | ubicom32-* \ - | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ - | vax-* \ - | we32k-* \ - | x86-* | x86_64-* | xc16x-* | xps100-* \ - | xstormy16-* | xtensa*-* \ - | ymp-* \ - | z8k-* | z80-*) - ;; - # Recognize the basic CPU types without company name, with glob match. - xtensa*) - basic_machine=$basic_machine-unknown + pmac | pmac-mpw) + cpu=powerpc + vendor=apple ;; + # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. - 386bsd) - basic_machine=i386-unknown - os=-bsd - ;; 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) - basic_machine=m68000-att + cpu=m68000 + vendor=att ;; 3b*) - basic_machine=we32k-att - ;; - a29khif) - basic_machine=a29k-amd - os=-udi - ;; - abacus) - basic_machine=abacus-unknown - ;; - adobe68k) - basic_machine=m68010-adobe - os=-scout - ;; - alliant | fx80) - basic_machine=fx80-alliant - ;; - altos | altos3068) - basic_machine=m68k-altos - ;; - am29k) - basic_machine=a29k-none - os=-bsd - ;; - amd64) - basic_machine=x86_64-pc - ;; - amd64-*) - basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - amdahl) - basic_machine=580-amdahl - os=-sysv - ;; - amiga | amiga-*) - basic_machine=m68k-unknown - ;; - amigaos | amigados) - basic_machine=m68k-unknown - os=-amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - os=-sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - os=-sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - os=-bsd - ;; - aros) - basic_machine=i386-pc - os=-aros - ;; - aux) - basic_machine=m68k-apple - os=-aux - ;; - balance) - basic_machine=ns32k-sequent - os=-dynix - ;; - blackfin) - basic_machine=bfin-unknown - os=-linux - ;; - blackfin-*) - basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + cpu=we32k + vendor=att ;; bluegene*) - basic_machine=powerpc-ibm - os=-cnk - ;; - c54x-*) - basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - c55x-*) - basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - c6x-*) - basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - c90) - basic_machine=c90-cray - os=-unicos - ;; - cegcc) - basic_machine=arm-unknown - os=-cegcc - ;; - convex-c1) - basic_machine=c1-convex - os=-bsd - ;; - convex-c2) - basic_machine=c2-convex - os=-bsd - ;; - convex-c32) - basic_machine=c32-convex - os=-bsd - ;; - convex-c34) - basic_machine=c34-convex - os=-bsd - ;; - convex-c38) - basic_machine=c38-convex - os=-bsd - ;; - cray | j90) - basic_machine=j90-cray - os=-unicos - ;; - craynv) - basic_machine=craynv-cray - os=-unicosmp - ;; - cr16 | cr16-*) - basic_machine=cr16-unknown - os=-elf - ;; - crds | unos) - basic_machine=m68k-crds - ;; - crisv32 | crisv32-* | etraxfs*) - basic_machine=crisv32-axis - ;; - cris | cris-* | etrax*) - basic_machine=cris-axis - ;; - crx) - basic_machine=crx-unknown - os=-elf - ;; - da30 | da30-*) - basic_machine=m68k-da30 - ;; - decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) - basic_machine=mips-dec + cpu=powerpc + vendor=ibm + os=cnk ;; decsystem10* | dec10*) - basic_machine=pdp10-dec - os=-tops10 + cpu=pdp10 + vendor=dec + os=tops10 ;; decsystem20* | dec20*) - basic_machine=pdp10-dec - os=-tops20 + cpu=pdp10 + vendor=dec + os=tops20 ;; delta | 3300 | motorola-3300 | motorola-delta \ | 3300-motorola | delta-motorola) - basic_machine=m68k-motorola - ;; - delta88) - basic_machine=m88k-motorola - os=-sysv3 - ;; - dicos) - basic_machine=i686-pc - os=-dicos + cpu=m68k + vendor=motorola ;; - djgpp) - basic_machine=i586-pc - os=-msdosdjgpp - ;; - dpx20 | dpx20-*) - basic_machine=rs6000-bull - os=-bosx - ;; - dpx2* | dpx2*-bull) - basic_machine=m68k-bull - os=-sysv3 - ;; - ebmon29k) - basic_machine=a29k-amd - os=-ebmon - ;; - elxsi) - basic_machine=elxsi-elxsi - os=-bsd + dpx2*) + cpu=m68k + vendor=bull + os=sysv3 ;; encore | umax | mmax) - basic_machine=ns32k-encore + cpu=ns32k + vendor=encore ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - os=-ose + elxsi) + cpu=elxsi + vendor=elxsi + os=${os:-bsd} ;; fx2800) - basic_machine=i860-alliant + cpu=i860 + vendor=alliant ;; genix) - basic_machine=ns32k-ns - ;; - gmicro) - basic_machine=tron-gmicro - os=-sysv - ;; - go32) - basic_machine=i386-pc - os=-go32 + cpu=ns32k + vendor=ns ;; h3050r* | hiux*) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - h8300hms) - basic_machine=h8300-hitachi - os=-hms - ;; - h8300xray) - basic_machine=h8300-hitachi - os=-xray - ;; - h8500hms) - basic_machine=h8500-hitachi - os=-hms - ;; - harris) - basic_machine=m88k-harris - os=-sysv3 - ;; - hp300-*) - basic_machine=m68k-hp - ;; - hp300bsd) - basic_machine=m68k-hp - os=-bsd - ;; - hp300hpux) - basic_machine=m68k-hp - os=-hpux + cpu=hppa1.1 + vendor=hitachi + os=hiuxwe2 ;; hp3k9[0-9][0-9] | hp9[0-9][0-9]) - basic_machine=hppa1.0-hp + cpu=hppa1.0 + vendor=hp ;; hp9k2[0-9][0-9] | hp9k31[0-9]) - basic_machine=m68000-hp + cpu=m68000 + vendor=hp ;; hp9k3[2-9][0-9]) - basic_machine=m68k-hp + cpu=m68k + vendor=hp ;; hp9k6[0-9][0-9] | hp6[0-9][0-9]) - basic_machine=hppa1.0-hp + cpu=hppa1.0 + vendor=hp ;; hp9k7[0-79][0-9] | hp7[0-79][0-9]) - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k78[0-9] | hp78[0-9]) # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[0-9][13679] | hp8[0-9][13679]) - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[0-9][0-9] | hp8[0-9][0-9]) - basic_machine=hppa1.0-hp - ;; - hppa-next) - os=-nextstep3 - ;; - hppaosf) - basic_machine=hppa1.1-hp - os=-osf - ;; - hppro) - basic_machine=hppa1.1-hp - os=-proelf - ;; - i370-ibm* | ibm*) - basic_machine=i370-ibm + cpu=hppa1.0 + vendor=hp ;; i*86v32) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv32 + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=sysv32 ;; i*86v4*) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv4 + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=sysv4 ;; i*86v) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=sysv ;; i*86sol2) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-solaris2 + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=solaris2 ;; - i386mach) - basic_machine=i386-mach - os=-mach - ;; - i386-vsta | vsta) - basic_machine=i386-unknown - os=-vsta + j90 | j90-cray) + cpu=j90 + vendor=cray + os=${os:-unicos} ;; iris | iris4d) - basic_machine=mips-sgi + cpu=mips + vendor=sgi case $os in - -irix*) + irix*) ;; *) - os=-irix4 + os=irix4 ;; esac ;; - isi68 | isi) - basic_machine=m68k-isi - os=-sysv - ;; - m68knommu) - basic_machine=m68k-unknown - os=-linux - ;; - m68knommu-*) - basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux - ;; - m88k-omron*) - basic_machine=m88k-omron - ;; - magnum | m3230) - basic_machine=mips-mips - os=-sysv - ;; - merlin) - basic_machine=ns32k-utek - os=-sysv - ;; - microblaze*) - basic_machine=microblaze-xilinx - ;; - mingw64) - basic_machine=x86_64-pc - os=-mingw64 - ;; - mingw32) - basic_machine=i386-pc - os=-mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - os=-mingw32ce - ;; miniframe) - basic_machine=m68000-convergent - ;; - *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) - basic_machine=m68k-atari - os=-mint - ;; - mips3*-*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` - ;; - mips3*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown - ;; - monitor) - basic_machine=m68k-rom68k - os=-coff - ;; - morphos) - basic_machine=powerpc-unknown - os=-morphos - ;; - msdos) - basic_machine=i386-pc - os=-msdos - ;; - ms1-*) - basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` - ;; - msys) - basic_machine=i386-pc - os=-msys - ;; - mvs) - basic_machine=i370-ibm - os=-mvs - ;; - nacl) - basic_machine=le32-unknown - os=-nacl - ;; - ncr3000) - basic_machine=i486-ncr - os=-sysv4 + cpu=m68000 + vendor=convergent ;; - netbsd386) - basic_machine=i386-unknown - os=-netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - os=-linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - os=-newsos - ;; - news1000) - basic_machine=m68030-sony - os=-newsos + *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) + cpu=m68k + vendor=atari + os=mint ;; news-3600 | risc-news) - basic_machine=mips-sony - os=-newsos - ;; - necv70) - basic_machine=v70-nec - os=-sysv + cpu=mips + vendor=sony + os=newsos ;; - next | m*-next ) - basic_machine=m68k-next + next | m*-next) + cpu=m68k + vendor=next case $os in - -nextstep* ) + openstep*) + ;; + nextstep*) ;; - -ns2*) - os=-nextstep2 + ns2*) + os=nextstep2 ;; *) - os=-nextstep3 + os=nextstep3 ;; esac ;; - nh3000) - basic_machine=m68k-harris - os=-cxux - ;; - nh[45]000) - basic_machine=m88k-harris - os=-cxux - ;; - nindy960) - basic_machine=i960-intel - os=-nindy - ;; - mon960) - basic_machine=i960-intel - os=-mon960 - ;; - nonstopux) - basic_machine=mips-compaq - os=-nonstopux - ;; np1) - basic_machine=np1-gould - ;; - neo-tandem) - basic_machine=neo-tandem - ;; - nse-tandem) - basic_machine=nse-tandem - ;; - nsr-tandem) - basic_machine=nsr-tandem + cpu=np1 + vendor=gould ;; op50n-* | op60c-*) - basic_machine=hppa1.1-oki - os=-proelf - ;; - openrisc | openrisc-*) - basic_machine=or32-unknown - ;; - os400) - basic_machine=powerpc-ibm - os=-os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - os=-ose - ;; - os68k) - basic_machine=m68k-none - os=-os68k + cpu=hppa1.1 + vendor=oki + os=proelf ;; pa-hitachi) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - paragon) - basic_machine=i860-intel - os=-osf - ;; - parisc) - basic_machine=hppa-unknown - os=-linux - ;; - parisc-*) - basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` - os=-linux + cpu=hppa1.1 + vendor=hitachi + os=hiuxwe2 ;; pbd) - basic_machine=sparc-tti + cpu=sparc + vendor=tti ;; pbb) - basic_machine=m68k-tti + cpu=m68k + vendor=tti ;; - pc532 | pc532-*) - basic_machine=ns32k-pc532 - ;; - pc98) - basic_machine=i386-pc - ;; - pc98-*) - basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentium | p5 | k5 | k6 | nexgen | viac3) - basic_machine=i586-pc - ;; - pentiumpro | p6 | 6x86 | athlon | athlon_*) - basic_machine=i686-pc - ;; - pentiumii | pentium2 | pentiumiii | pentium3) - basic_machine=i686-pc - ;; - pentium4) - basic_machine=i786-pc - ;; - pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) - basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentiumpro-* | p6-* | 6x86-* | athlon-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentium4-*) - basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + pc532) + cpu=ns32k + vendor=pc532 ;; pn) - basic_machine=pn-gould - ;; - power) basic_machine=power-ibm + cpu=pn + vendor=gould ;; - ppc | ppcbe) basic_machine=powerpc-unknown - ;; - ppc-* | ppcbe-*) - basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppcle | powerpclittle | ppc-le | powerpc-little) - basic_machine=powerpcle-unknown - ;; - ppcle-* | powerpclittle-*) - basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppc64) basic_machine=powerpc64-unknown - ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) - basic_machine=powerpc64le-unknown - ;; - ppc64le-* | powerpc64little-*) - basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + power) + cpu=power + vendor=ibm ;; ps2) - basic_machine=i386-ibm - ;; - pw32) - basic_machine=i586-unknown - os=-pw32 - ;; - rdos | rdos64) - basic_machine=x86_64-pc - os=-rdos - ;; - rdos32) - basic_machine=i386-pc - os=-rdos - ;; - rom68k) - basic_machine=m68k-rom68k - os=-coff + cpu=i386 + vendor=ibm ;; rm[46]00) - basic_machine=mips-siemens + cpu=mips + vendor=siemens ;; rtpc | rtpc-*) - basic_machine=romp-ibm - ;; - s390 | s390-*) - basic_machine=s390-ibm - ;; - s390x | s390x-*) - basic_machine=s390x-ibm + cpu=romp + vendor=ibm ;; - sa29200) - basic_machine=a29k-amd - os=-udi - ;; - sb1) - basic_machine=mipsisa64sb1-unknown + sde) + cpu=mipsisa32 + vendor=sde + os=${os:-elf} ;; - sb1el) - basic_machine=mipsisa64sb1el-unknown + simso-wrs) + cpu=sparclite + vendor=wrs + os=vxworks ;; - sde) - basic_machine=mipsisa32-sde - os=-elf + tower | tower-32) + cpu=m68k + vendor=ncr ;; - sei) - basic_machine=mips-sei - os=-seiux + vpp*|vx|vx-*) + cpu=f301 + vendor=fujitsu ;; - sequent) - basic_machine=i386-sequent + w65) + cpu=w65 + vendor=wdc ;; - sh) - basic_machine=sh-hitachi - os=-hms + w89k-*) + cpu=hppa1.1 + vendor=winbond + os=proelf ;; - sh5el) - basic_machine=sh5le-unknown + none) + cpu=none + vendor=none ;; - sh64) - basic_machine=sh64-unknown + leon|leon[3-9]) + cpu=sparc + vendor=$basic_machine ;; - sparclite-wrs | simso-wrs) - basic_machine=sparclite-wrs - os=-vxworks + leon-*|leon[3-9]-*) + cpu=sparc + vendor=`echo "$basic_machine" | sed 's/-.*//'` ;; - sps7) - basic_machine=m68k-bull - os=-sysv2 + + *-*) + # shellcheck disable=SC2162 + IFS="-" read cpu vendor <<EOF +$basic_machine +EOF ;; - spur) - basic_machine=spur-unknown + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + cpu=$basic_machine + vendor=pc ;; - st2000) - basic_machine=m68k-tandem + # These rules are duplicated from below for sake of the special case above; + # i.e. things that normalized to x86 arches should also default to "pc" + pc98) + cpu=i386 + vendor=pc ;; - stratus) - basic_machine=i860-stratus - os=-sysv4 + x64 | amd64) + cpu=x86_64 + vendor=pc ;; - strongarm-* | thumb-*) - basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + # Recognize the basic CPU types without company name. + *) + cpu=$basic_machine + vendor=unknown ;; - sun2) - basic_machine=m68000-sun +esac + +unset -v basic_machine + +# Decode basic machines in the full and proper CPU-Company form. +case $cpu-$vendor in + # Here we handle the default manufacturer of certain CPU types in canonical form. It is in + # some cases the only manufacturer, in others, it is the most popular. + craynv-unknown) + vendor=cray + os=${os:-unicosmp} ;; - sun2os3) - basic_machine=m68000-sun - os=-sunos3 + c90-unknown | c90-cray) + vendor=cray + os=${os:-unicos} ;; - sun2os4) - basic_machine=m68000-sun - os=-sunos4 + fx80-unknown) + vendor=alliant ;; - sun3os3) - basic_machine=m68k-sun - os=-sunos3 + romp-unknown) + vendor=ibm ;; - sun3os4) - basic_machine=m68k-sun - os=-sunos4 + mmix-unknown) + vendor=knuth ;; - sun4os3) - basic_machine=sparc-sun - os=-sunos3 + microblaze-unknown | microblazeel-unknown) + vendor=xilinx ;; - sun4os4) - basic_machine=sparc-sun - os=-sunos4 + rs6000-unknown) + vendor=ibm ;; - sun4sol2) - basic_machine=sparc-sun - os=-solaris2 + vax-unknown) + vendor=dec ;; - sun3 | sun3-*) - basic_machine=m68k-sun + pdp11-unknown) + vendor=dec ;; - sun4) - basic_machine=sparc-sun + we32k-unknown) + vendor=att ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun + cydra-unknown) + vendor=cydrome ;; - sv1) - basic_machine=sv1-cray - os=-unicos + i370-ibm*) + vendor=ibm ;; - symmetry) - basic_machine=i386-sequent - os=-dynix + orion-unknown) + vendor=highlevel ;; - t3e) - basic_machine=alphaev5-cray - os=-unicos + xps-unknown | xps100-unknown) + cpu=xps100 + vendor=honeywell ;; - t90) - basic_machine=t90-cray - os=-unicos + + # Here we normalize CPU types with a missing or matching vendor + dpx20-unknown | dpx20-bull) + cpu=rs6000 + vendor=bull + os=${os:-bosx} ;; - tile*) - basic_machine=$basic_machine-unknown - os=-linux-gnu + + # Here we normalize CPU types irrespective of the vendor + amd64-*) + cpu=x86_64 ;; - tx39) - basic_machine=mipstx39-unknown + blackfin-*) + cpu=bfin + os=linux ;; - tx39el) - basic_machine=mipstx39el-unknown + c54x-*) + cpu=tic54x ;; - toad1) - basic_machine=pdp10-xkl - os=-tops20 + c55x-*) + cpu=tic55x ;; - tower | tower-32) - basic_machine=m68k-ncr + c6x-*) + cpu=tic6x ;; - tpf) - basic_machine=s390x-ibm - os=-tpf + e500v[12]-*) + cpu=powerpc + os=$os"spe" ;; - udi29k) - basic_machine=a29k-amd - os=-udi + mips3*-*) + cpu=mips64 ;; - ultra3) - basic_machine=a29k-nyu - os=-sym1 + ms1-*) + cpu=mt ;; - v810 | necv810) - basic_machine=v810-nec - os=-none + m68knommu-*) + cpu=m68k + os=linux ;; - vaxv) - basic_machine=vax-dec - os=-sysv + m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) + cpu=s12z ;; - vms) - basic_machine=vax-dec - os=-vms + openrisc-*) + cpu=or32 ;; - vpp*|vx|vx-*) - basic_machine=f301-fujitsu + parisc-*) + cpu=hppa + os=linux ;; - vxworks960) - basic_machine=i960-wrs - os=-vxworks + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + cpu=i586 ;; - vxworks68) - basic_machine=m68k-wrs - os=-vxworks + pentiumpro-* | p6-* | 6x86-* | athlon-* | athalon_*-*) + cpu=i686 ;; - vxworks29k) - basic_machine=a29k-wrs - os=-vxworks + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + cpu=i686 ;; - w65*) - basic_machine=w65-wdc - os=-none + pentium4-*) + cpu=i786 ;; - w89k-*) - basic_machine=hppa1.1-winbond - os=-proelf + pc98-*) + cpu=i386 ;; - xbox) - basic_machine=i686-pc - os=-mingw32 + ppc-* | ppcbe-*) + cpu=powerpc ;; - xps | xps100) - basic_machine=xps100-honeywell + ppcle-* | powerpclittle-*) + cpu=powerpcle ;; - xscale-* | xscalee[bl]-*) - basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + ppc64-*) + cpu=powerpc64 ;; - ymp) - basic_machine=ymp-cray - os=-unicos + ppc64le-* | powerpc64little-*) + cpu=powerpc64le ;; - z8k-*-coff) - basic_machine=z8k-unknown - os=-sim + sb1-*) + cpu=mipsisa64sb1 ;; - z80-*-coff) - basic_machine=z80-unknown - os=-sim + sb1el-*) + cpu=mipsisa64sb1el ;; - none) - basic_machine=none-none - os=-none + sh5e[lb]-*) + cpu=`echo "$cpu" | sed 's/^\(sh.\)e\(.\)$/\1\2e/'` ;; - -# Here we handle the default manufacturer of certain CPU types. It is in -# some cases the only manufacturer, in others, it is the most popular. - w89k) - basic_machine=hppa1.1-winbond + spur-*) + cpu=spur ;; - op50n) - basic_machine=hppa1.1-oki + strongarm-* | thumb-*) + cpu=arm ;; - op60c) - basic_machine=hppa1.1-oki + tx39-*) + cpu=mipstx39 ;; - romp) - basic_machine=romp-ibm + tx39el-*) + cpu=mipstx39el ;; - mmix) - basic_machine=mmix-knuth + x64-*) + cpu=x86_64 ;; - rs6000) - basic_machine=rs6000-ibm + xscale-* | xscalee[bl]-*) + cpu=`echo "$cpu" | sed 's/^xscale/arm/'` ;; - vax) - basic_machine=vax-dec + + # Recognize the canonical CPU Types that limit and/or modify the + # company names they are paired with. + cr16-*) + os=${os:-elf} ;; - pdp10) - # there are many clones, so DEC is not a safe bet - basic_machine=pdp10-unknown + crisv32-* | etraxfs*-*) + cpu=crisv32 + vendor=axis ;; - pdp11) - basic_machine=pdp11-dec + cris-* | etrax*-*) + cpu=cris + vendor=axis ;; - we32k) - basic_machine=we32k-att + crx-*) + os=${os:-elf} ;; - sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) - basic_machine=sh-unknown + neo-tandem) + cpu=neo + vendor=tandem ;; - sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) - basic_machine=sparc-sun + nse-tandem) + cpu=nse + vendor=tandem ;; - cydra) - basic_machine=cydra-cydrome + nsr-tandem) + cpu=nsr + vendor=tandem ;; - orion) - basic_machine=orion-highlevel + nsv-tandem) + cpu=nsv + vendor=tandem ;; - orion105) - basic_machine=clipper-highlevel + nsx-tandem) + cpu=nsx + vendor=tandem ;; - mac | mpw | mac-mpw) - basic_machine=m68k-apple + s390-*) + cpu=s390 + vendor=ibm ;; - pmac | pmac-mpw) - basic_machine=powerpc-apple + s390x-*) + cpu=s390x + vendor=ibm ;; - *-unknown) - # Make sure to match an already-canonicalized machine name. + tile*-*) + os=${os:-linux-gnu} ;; + *) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 + # Recognize the canonical CPU types that are allowed with any + # company name. + case $cpu in + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | abacus \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ + | alphapca5[67] | alpha64pca5[67] \ + | am33_2.0 \ + | amdgcn \ + | arc | arceb \ + | arm | arm[lb]e | arme[lb] | armv* \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ + | bfin | bpf | bs2000 \ + | c[123]* | c30 | [cjt]90 | c4x \ + | c8051 | clipper | craynv | csky | cydra \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | elxsi | epiphany \ + | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | h8300 | h8500 \ + | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i*86 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle \ + | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ + | m88110 | m88k | maxq | mb | mcore | mep | metag \ + | microblaze | microblazeel \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64eb | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mmix \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ + | none | np1 | ns16k | ns32k | nvptx \ + | open8 \ + | or1k* \ + | or32 \ + | orion \ + | picochip \ + | pdp10 | pdp11 | pj | pjl | pn | power \ + | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ + | pru \ + | pyramid \ + | riscv | riscv32 | riscv64 \ + | rl78 | romp | rs6000 | rx \ + | score \ + | sh | shl \ + | sh[1234] | sh[24]a | sh[24]ae[lb] | sh[23]e | she[lb] | sh[lb]e \ + | sh[1234]e[lb] | sh[12345][lb]e | sh[23]ele | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet \ + | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v | sv1 | sx* \ + | spu \ + | tahoe \ + | tic30 | tic4x | tic54x | tic55x | tic6x | tic80 \ + | tron \ + | ubicom32 \ + | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ + | vax \ + | visium \ + | w65 \ + | wasm32 | wasm64 \ + | we32k \ + | x86 | x86_64 | xc16x | xgate | xps100 \ + | xstormy16 | xtensa* \ + | ymp \ + | z8k | z80) + ;; + + *) + echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + exit 1 + ;; + esac ;; esac # Here we canonicalize certain aliases for manufacturers. -case $basic_machine in - *-digital*) - basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` +case $vendor in + digital*) + vendor=dec ;; - *-commodore*) - basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + commodore*) + vendor=cbm ;; *) ;; @@ -1321,197 +1275,244 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x"$os" != x"" ] +if [ x$os != x ] then case $os in - # First match some system type aliases - # that might get confused with valid system types. - # -solaris* is a basic system type, with this one exception. - -auroraux) - os=-auroraux + # First match some system type aliases that might get confused + # with valid system types. + # solaris* is a basic system type, with this one exception. + auroraux) + os=auroraux ;; - -solaris1 | -solaris1.*) - os=`echo $os | sed -e 's|solaris1|sunos4|'` + bluegene*) + os=cnk ;; - -solaris) - os=-solaris2 + solaris1 | solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; - -svr4*) - os=-sysv4 + solaris) + os=solaris2 ;; - -unixware*) - os=-sysv4.2uw + unixware*) + os=sysv4.2uw ;; - -gnu/linux*) + gnu/linux*) os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` ;; - # First accept the basic system types. + # es1800 is here to avoid being matched by es* (a different OS) + es1800*) + os=ose + ;; + # Some version numbers need modification + chorusos*) + os=chorusos + ;; + isc) + os=isc2.2 + ;; + sco6) + os=sco5v6 + ;; + sco5) + os=sco3.2v5 + ;; + sco4) + os=sco3.2v4 + ;; + sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + ;; + sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + scout) + # Don't match below + ;; + sco*) + os=sco3.2v2 + ;; + psos*) + os=psos + ;; + # Now accept the basic system types. # The portable systems comes first. - # Each alternative MUST END IN A *, to match a version number. - # -sysv* is not here because it comes later, after sysvr4. - -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ - | -sym* | -kopensolaris* | -plan9* \ - | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ - | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ - | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ - | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ - | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ - | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ - | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* | -cegcc* \ - | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ - | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* \ - | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ - | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ - | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ - | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ - | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) + # Each alternative MUST end in a * to match a version number. + # sysv* is not here because it comes later, after sysvr4. + gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | kopensolaris* | plan9* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* | twizzler* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | knetbsd* | mirbsd* | netbsd* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* | os108* \ + | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \ + | linux-newlib* | linux-musl* | linux-uclibc* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* \ + | morphos* | superux* | rtmk* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ + | nsk* | powerunix* | genode*) # Remember, each alternative MUST END IN *, to match a version number. ;; - -qnx*) - case $basic_machine in - x86-* | i*86-*) + qnx*) + case $cpu in + x86 | i*86) ;; *) - os=-nto$os + os=nto-$os ;; esac ;; - -nto-qnx*) + hiux*) + os=hiuxwe2 ;; - -nto*) - os=`echo $os | sed -e 's|nto|nto-qnx|'` + nto-qnx*) ;; - -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ - | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; - -mac*) - os=`echo $os | sed -e 's|mac|macos|'` + sim | xray | os68k* | v88r* \ + | windows* | osx | abug | netware* | os9* \ + | macos* | mpw* | magic* | mmixware* | mon960* | lnews*) ;; - -linux-dietlibc) - os=-linux-dietlibc + linux-dietlibc) + os=linux-dietlibc ;; - -linux*) + linux*) os=`echo $os | sed -e 's|linux|linux-gnu|'` ;; - -sunos5*) - os=`echo $os | sed -e 's|sunos5|solaris2|'` + lynx*178) + os=lynxos178 ;; - -sunos6*) - os=`echo $os | sed -e 's|sunos6|solaris3|'` + lynx*5) + os=lynxos5 ;; - -opened*) - os=-openedition + lynx*) + os=lynxos ;; - -os400*) - os=-os400 + mac*) + os=`echo "$os" | sed -e 's|mac|macos|'` ;; - -wince*) - os=-wince + opened*) + os=openedition ;; - -osfrose*) - os=-osfrose + os400*) + os=os400 ;; - -osf*) - os=-osf + sunos5*) + os=`echo "$os" | sed -e 's|sunos5|solaris2|'` ;; - -utek*) - os=-bsd + sunos6*) + os=`echo "$os" | sed -e 's|sunos6|solaris3|'` ;; - -dynix*) - os=-bsd + wince*) + os=wince ;; - -acis*) - os=-aos + utek*) + os=bsd ;; - -atheos*) - os=-atheos + dynix*) + os=bsd ;; - -syllable*) - os=-syllable + acis*) + os=aos ;; - -386bsd) - os=-bsd + atheos*) + os=atheos ;; - -ctix* | -uts*) - os=-sysv + syllable*) + os=syllable ;; - -nova*) - os=-rtmk-nova + 386bsd) + os=bsd ;; - -ns2 ) - os=-nextstep2 + ctix* | uts*) + os=sysv ;; - -nsk*) - os=-nsk + nova*) + os=rtmk-nova + ;; + ns2) + os=nextstep2 ;; # Preserve the version number of sinix5. - -sinix5.*) + sinix5.*) os=`echo $os | sed -e 's|sinix|sysv|'` ;; - -sinix*) - os=-sysv4 + sinix*) + os=sysv4 ;; - -tpf*) - os=-tpf + tpf*) + os=tpf ;; - -triton*) - os=-sysv3 + triton*) + os=sysv3 ;; - -oss*) - os=-sysv3 + oss*) + os=sysv3 ;; - -svr4) - os=-sysv4 + svr4*) + os=sysv4 ;; - -svr3) - os=-sysv3 + svr3) + os=sysv3 ;; - -sysvr4) - os=-sysv4 + sysvr4) + os=sysv4 ;; - # This must come after -sysvr4. - -sysv*) + # This must come after sysvr4. + sysv*) ;; - -ose*) - os=-ose + ose*) + os=ose ;; - -es1800*) - os=-ose + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + os=mint ;; - -xenix) - os=-xenix + zvmoe) + os=zvmoe ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) - os=-mint + dicos*) + os=dicos ;; - -aros*) - os=-aros + pikeos*) + # Until real need of OS specific support for + # particular features comes up, bare metal + # configurations are quite functional. + case $cpu in + arm*) + os=eabi + ;; + *) + os=elf + ;; + esac ;; - -zvmoe) - os=-zvmoe + nacl*) ;; - -dicos*) - os=-dicos + ios) ;; - -nacl*) + none) ;; - -none) + *-eabi) ;; *) - # Get rid of the `-' at the beginning of $os. - os=`echo $os | sed 's/[^-]*-//'` - echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 exit 1 ;; esac @@ -1527,261 +1528,265 @@ else # will signal an error saying that MANUFACTURER isn't an operating # system, and we'll never get to this point. -case $basic_machine in +case $cpu-$vendor in score-*) - os=-elf + os=elf ;; spu-*) - os=-elf + os=elf ;; *-acorn) - os=-riscix1.2 + os=riscix1.2 ;; arm*-rebel) - os=-linux + os=linux ;; arm*-semi) - os=-aout + os=aout ;; c4x-* | tic4x-*) - os=-coff + os=coff + ;; + c8051-*) + os=elf + ;; + clipper-intergraph) + os=clix ;; hexagon-*) - os=-elf + os=elf ;; tic54x-*) - os=-coff + os=coff ;; tic55x-*) - os=-coff + os=coff ;; tic6x-*) - os=-coff + os=coff ;; # This must come before the *-dec entry. pdp10-*) - os=-tops20 + os=tops20 ;; pdp11-*) - os=-none + os=none ;; *-dec | vax-*) - os=-ultrix4.2 + os=ultrix4.2 ;; m68*-apollo) - os=-domain + os=domain ;; i386-sun) - os=-sunos4.0.2 + os=sunos4.0.2 ;; m68000-sun) - os=-sunos3 + os=sunos3 ;; m68*-cisco) - os=-aout + os=aout ;; mep-*) - os=-elf + os=elf ;; mips*-cisco) - os=-elf + os=elf ;; mips*-*) - os=-elf - ;; - or1k-*) - os=-elf + os=elf ;; or32-*) - os=-coff + os=coff ;; *-tti) # must be before sparc entry or we get the wrong os. - os=-sysv3 + os=sysv3 ;; sparc-* | *-sun) - os=-sunos4.1.1 + os=sunos4.1.1 ;; - *-be) - os=-beos + pru-*) + os=elf ;; - *-haiku) - os=-haiku + *-be) + os=beos ;; *-ibm) - os=-aix + os=aix ;; *-knuth) - os=-mmixware + os=mmixware ;; *-wec) - os=-proelf + os=proelf ;; *-winbond) - os=-proelf + os=proelf ;; *-oki) - os=-proelf + os=proelf ;; *-hp) - os=-hpux + os=hpux ;; *-hitachi) - os=-hiux + os=hiux ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) - os=-sysv + os=sysv ;; *-cbm) - os=-amigaos + os=amigaos ;; *-dg) - os=-dgux + os=dgux ;; *-dolphin) - os=-sysv3 + os=sysv3 ;; m68k-ccur) - os=-rtu + os=rtu ;; m88k-omron*) - os=-luna + os=luna ;; - *-next ) - os=-nextstep + *-next) + os=nextstep ;; *-sequent) - os=-ptx + os=ptx ;; *-crds) - os=-unos + os=unos ;; *-ns) - os=-genix + os=genix ;; i370-*) - os=-mvs - ;; - *-next) - os=-nextstep3 + os=mvs ;; *-gould) - os=-sysv + os=sysv ;; *-highlevel) - os=-bsd + os=bsd ;; *-encore) - os=-bsd + os=bsd ;; *-sgi) - os=-irix + os=irix ;; *-siemens) - os=-sysv4 + os=sysv4 ;; *-masscomp) - os=-rtu + os=rtu ;; f30[01]-fujitsu | f700-fujitsu) - os=-uxpv + os=uxpv ;; *-rom68k) - os=-coff + os=coff ;; *-*bug) - os=-coff + os=coff ;; *-apple) - os=-macos + os=macos ;; *-atari*) - os=-mint + os=mint + ;; + *-wrs) + os=vxworks ;; *) - os=-none + os=none ;; esac fi # Here we handle the case where we know the os, and the CPU type, but not the # manufacturer. We pick the logical manufacturer. -vendor=unknown -case $basic_machine in - *-unknown) +case $vendor in + unknown) case $os in - -riscix*) + riscix*) vendor=acorn ;; - -sunos*) + sunos*) vendor=sun ;; - -cnk*|-aix*) + cnk*|-aix*) vendor=ibm ;; - -beos*) + beos*) vendor=be ;; - -hpux*) + hpux*) vendor=hp ;; - -mpeix*) + mpeix*) vendor=hp ;; - -hiux*) + hiux*) vendor=hitachi ;; - -unos*) + unos*) vendor=crds ;; - -dgux*) + dgux*) vendor=dg ;; - -luna*) + luna*) vendor=omron ;; - -genix*) + genix*) vendor=ns ;; - -mvs* | -opened*) + clix*) + vendor=intergraph + ;; + mvs* | opened*) vendor=ibm ;; - -os400*) + os400*) vendor=ibm ;; - -ptx*) + ptx*) vendor=sequent ;; - -tpf*) + tpf*) vendor=ibm ;; - -vxsim* | -vxworks* | -windiss*) + vxsim* | vxworks* | windiss*) vendor=wrs ;; - -aux*) + aux*) vendor=apple ;; - -hms*) + hms*) vendor=hitachi ;; - -mpw* | -macos*) + mpw* | macos*) vendor=apple ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) vendor=atari ;; - -vos*) + vos*) vendor=stratus ;; esac - basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` ;; esac -echo $basic_machine$os +echo "$cpu-$vendor-$os" exit # Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" diff --git a/configure.ac b/configure.ac index 07ce8002..d665498d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ dnl Process this file with autoconf to produce a configure script. -AC_INIT([rsync],[3.1.3],[http://rsync.samba.org/bugzilla.html]) +AC_INIT([rsync],[3.2.0],[http://rsync.samba.org/bugzilla.html]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_SRCDIR([byteorder.h]) @@ -41,12 +41,15 @@ fi dnl Checks for programs. AC_PROG_CC AC_PROG_CPP +AC_PROG_CXX +AC_PROG_AWK AC_PROG_EGREP AC_PROG_INSTALL AC_PROG_MKDIR_P AC_PROG_CC_STDC AC_SUBST(SHELL) AC_PATH_PROG([PERL], [perl]) +AC_PATH_PROG([PYTHON3], [python3]) AC_DEFINE([_GNU_SOURCE], 1, [Define _GNU_SOURCE so that we get all necessary prototypes]) @@ -61,6 +64,40 @@ if test x"$enable_profile" = x"yes"; then CFLAGS="$CFLAGS -pg" fi +AC_MSG_CHECKING([if md2man can create man pages]) +if test x"$ac_cv_path_PYTHON3" = x; then + AC_MSG_RESULT(no - python3 not found) + md2man_works=no +else + md2man_out=`"$srcdir/md2man" --test "$srcdir/rsync.1.md" 2>&1` + if test $? = 0; then + AC_MSG_RESULT(yes) + md2man_works=yes + else + AC_MSG_RESULT(no) + md2man_works=no + echo "$md2man_out" + fi +fi + +AC_MSG_CHECKING([if we require man-page building]) +AC_ARG_ENABLE([md2man], + AS_HELP_STRING([--disable-md2man],[disable md2man for man page creation])) +if test x"$enable_md2man" != x"no"; then + if test -f "$srcdir/rsync.1"; then + AC_MSG_RESULT(optional) + else + AC_MSG_RESULT(required) + if test x"$md2man_works" = x"no"; then + AC_MSG_ERROR(You need python3 and the cmarkgfm OR commonmark python3 lib in order to build man pages. +You can specify --disable-md2man if you want to skip building them.) + fi + fi + MAKE_MAN=man +else + AC_MSG_RESULT(no) + MAKE_MAN='' +fi # Specifically, this turns on panic_action handling. AC_ARG_ENABLE(maintainer-mode, @@ -69,7 +106,6 @@ if test x"$enable_maintainer_mode" = x"yes"; then CFLAGS="$CFLAGS -DMAINTAINER_MODE" fi - # This is needed for our included version of popt. Kind of silly, but # I don't want our version too far out of sync. CFLAGS="$CFLAGS -DHAVE_CONFIG_H" @@ -134,13 +170,6 @@ else fi AC_DEFINE_UNQUOTED(RSYNC_RSH, "$RSYNC_RSH", [default -e command]) -AC_CHECK_PROG(HAVE_YODL2MAN, yodl2man, 1, 0) -if test x$HAVE_YODL2MAN = x1; then - MAKE_MAN=man -else - MAKE_MAN=man-copy -fi - # Some programs on solaris are only found in /usr/xpg4/bin (or work better than others versions). AC_PATH_PROG(SHELL_PATH, sh, /bin/sh, [/usr/xpg4/bin$PATH_SEPARATOR$PATH]) AC_PATH_PROG(FAKEROOT_PATH, fakeroot, /usr/bin/fakeroot, [/usr/xpg4/bin$PATH_SEPARATOR$PATH]) @@ -164,6 +193,69 @@ fi AC_DEFINE_UNQUOTED(NOBODY_USER, "nobody", [unprivileged user--e.g. nobody]) AC_DEFINE_UNQUOTED(NOBODY_GROUP, "$NOBODY_GROUP", [unprivileged group for unprivileged user]) +# SIMD optimizations +SIMD= + +AC_MSG_CHECKING([whether to enable SIMD optimizations]) +AC_ARG_ENABLE(simd, + AS_HELP_STRING([--disable-simd],[disable SIMD optimizations (requires c++)])) + +if test x"$enable_simd" != x"no"; then + # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required + if test x"$build_cpu" = x"x86_64"; then + CXX_OK= + if test x"$CXX" != x""; then + CXX_VERSION=`$CXX --version 2>/dev/null | head -n 1` + case "$CXX_VERSION" in + g++*) + CXX_VERSION=`$CXX -dumpversion | sed 's/\..*//g'` + if test "$CXX_VERSION" -ge "5"; then + CXX_OK=yes + fi + ;; + *clang*) + # $CXX -dumpversion would have been ideal, but is broken on older clang + CXX_VERSION=`echo "$CXX_VERSION" | sed 's/.*version //g' | sed 's/\..*//g'` + if test "$CXX_VERSION" -ge "7"; then + CXX_OK=yes + fi + ;; + *) + CXX_VERSION='Unknown' + ;; + esac + else + CXX='No c++' + CXX_VERSION='Unknown' + fi + if test x"$CXX_OK" = x"yes"; then + # AC_MSG_RESULT() is called below. + SIMD="$SIMD x86_64" + else + AC_MSG_RESULT(error) + AC_MSG_ERROR([Failed to find g++ >=5 or clang++ >=7 for SIMD optimizations. +Specify --disable-simd to continue without it. ($CXX, $CXX_VERSION)]) + fi + elif test x"$enable_simd" = x"yes"; then + AC_MSG_RESULT(unavailable) + AC_MSG_ERROR(The SIMD optimizations are currently x86_64 only. +Omit --enable-simd to continue without it.) + fi +fi + +if test x"$SIMD" != x""; then + SIMD=`echo "$SIMD" | sed 's/^ *//'` + AC_MSG_RESULT([yes ($SIMD)]) + AC_DEFINE(HAVE_SIMD, 1, [Define to 1 to enable SIMD optimizations]) + SIMD=`echo "\\\$(SIMD_$SIMD)" | sed 's/ /) $(SIMD_/g'` + # We only use c++ for its target attribute dispatching, disable unneeded bulky features + CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti" +else + AC_MSG_RESULT(no) +fi + +AC_SUBST(SIMD) + # arrgh. libc in some old debian version screwed up the largefile # stuff, getting byte range locking wrong AC_CACHE_CHECK([for broken largefile support],rsync_cv_HAVE_BROKEN_LARGEFILE,[ @@ -173,6 +265,9 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include <fcntl.h> #include <sys/types.h> #include <sys/wait.h> +#if HAVE_UNISTD_H +#include <unistd.h> +#endif int main(void) { @@ -197,7 +292,7 @@ int main(void) } wait(&status); unlink(tpl); - exit(WEXITSTATUS(status)); + return WEXITSTATUS(status); } ]])],[rsync_cv_HAVE_BROKEN_LARGEFILE=yes],[rsync_cv_HAVE_BROKEN_LARGEFILE=no],[rsync_cv_HAVE_BROKEN_LARGEFILE=cross])]) if test x"$rsync_cv_HAVE_BROKEN_LARGEFILE" != x"yes"; then @@ -209,7 +304,7 @@ ipv6lib=none ipv6trylibc=yes AC_ARG_ENABLE(ipv6, - AS_HELP_STRING([--disable-ipv6],[do not even try to use IPv6])) + AS_HELP_STRING([--disable-ipv6],[turn off IPv6 support])) if test x"$enable_ipv6" != x"no"; then AC_MSG_CHECKING([ipv6 stack type]) for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta cygwin; do @@ -347,8 +442,92 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \ netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \ sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \ popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \ - zlib.h) -AC_HEADER_MAJOR + zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h) +AC_HEADER_MAJOR_FIXED + +AC_MSG_CHECKING([whether to enable use of openssl crypto library]) +AC_ARG_ENABLE([openssl], + AS_HELP_STRING([--disable-openssl],[disable openssl crypto library])) +AH_TEMPLATE([USE_OPENSSL], +[Undefine if you do not want to use openssl crypto library. By default this is defined.]) +if test x"$enable_openssl" != x"no"; then + if test x"$ac_cv_header_openssl_md4_h" = x"yes" && test x"$ac_cv_header_openssl_md5_h" = x"yes"; then + AC_MSG_RESULT(yes) + AC_SEARCH_LIBS(MD5_Init, crypto, + [AC_DEFINE(USE_OPENSSL)], + [AC_MSG_ERROR(Failed to find MD5_Init function in openssl crypto lib. +Use --disable-openssl to continue without openssl crypto lib support.)]) + else + AC_MSG_RESULT(no) + AC_MSG_ERROR(Failed to find openssl/md4.h and openssl/md5.h for openssl crypto lib support. +Use --disable-openssl to continue without it.) + fi +else + AC_MSG_RESULT(no) +fi + +AC_MSG_CHECKING([whether to enable xxhash checksum support]) +AC_ARG_ENABLE([xxhash], + AS_HELP_STRING([--disable-xxhash],[disable xxhash checksums])) +AH_TEMPLATE([SUPPORT_XXHASH], +[Undefine if you do not want xxhash checksums. By default this is defined.]) +if test x"$enable_xxhash" != x"no"; then + if test x"$ac_cv_header_xxhash_h" = x"yes"; then + AC_MSG_RESULT(yes) + AC_SEARCH_LIBS(XXH64_createState, xxhash, + [AC_DEFINE(SUPPORT_XXHASH)], + [AC_MSG_ERROR(Failed to find XXH64_createState function in xxhash lib. +Use --disable-xxhash to continue without xxhash checksums.)]) + else + AC_MSG_RESULT(no) + AC_MSG_ERROR(Failed to find xxhash.h for xxhash checksum support. +Use --disable-xxhash to continue without it.) + fi +else + AC_MSG_RESULT(no) +fi + +AC_MSG_CHECKING([whether to enable zstd compression]) +AC_ARG_ENABLE([zstd], + AC_HELP_STRING([--disable-zstd], [disable zstd compression])) +AH_TEMPLATE([SUPPORT_ZSTD], +[Undefine if you do not want zstd compression. By default this is defined.]) +if test x"$enable_zstd" != x"no"; then + if test x"$ac_cv_header_zstd_h" = x"yes"; then + AC_MSG_RESULT(yes) + AC_SEARCH_LIBS(ZSTD_minCLevel, zstd, + [AC_DEFINE(SUPPORT_ZSTD)], + [AC_MSG_ERROR(Failed to find ZSTD_minCLevel function in zstd lib. +Use --disable-zstd to continue without zstd compression.)]) + else + AC_MSG_RESULT(no) + AC_MSG_ERROR(Failed to find zstd.h for zstd compression support. +Use --disable-zstd to continue without it.) + fi +else + AC_MSG_RESULT(no) +fi + +AC_MSG_CHECKING([whether to enable LZ4 compression]) +AC_ARG_ENABLE([lz4], + AC_HELP_STRING([--disable-lz4], [disable LZ4 compression])) +AH_TEMPLATE([SUPPORT_LZ4], +[Undefine if you do not want LZ4 compression. By default this is defined.]) +if test x"$enable_lz4" != x"no"; then + if test x"$ac_cv_header_lz4_h" = x"yes"; then + AC_MSG_RESULT(yes) + AC_SEARCH_LIBS(LZ4_compress_default, lz4, + [AC_DEFINE(SUPPORT_LZ4)], + [AC_MSG_ERROR(Failed to find LZ4_compress_default function in lz4 lib. +Use --disable-lz4 to continue without lz4 compression.)]) + else + AC_MSG_RESULT(no) + AC_MSG_ERROR(Failed to find lz4.h for lz4 compression support. +Use --disable-lz4 to continue without it.) + fi +else + AC_MSG_RESULT(no) +fi AC_CACHE_CHECK([if makedev takes 3 args],rsync_cv_MAKEDEV_TAKES_3_ARGS,[ AC_RUN_IFELSE([AC_LANG_SOURCE([[ @@ -366,7 +545,7 @@ int main(void) { dev_t dev = makedev(0, 5, 7); if (major(dev) != 5 || minor(dev) != 7) - exit(1); + return 1; return 0; } ]])],[rsync_cv_MAKEDEV_TAKES_3_ARGS=yes],[rsync_cv_MAKEDEV_TAKES_3_ARGS=no],[rsync_cv_MAKEDEV_TAKES_3_ARGS=no])]) @@ -386,6 +565,7 @@ AC_CHECK_SIZEOF(int64_t) AC_CHECK_SIZEOF(off_t) AC_CHECK_SIZEOF(off64_t) AC_CHECK_SIZEOF(time_t) +AC_CHECK_SIZEOF(char*) AC_C_INLINE @@ -480,7 +660,7 @@ size_t iconv(); #endif ]], [[]])],[am_cv_proto_iconv_arg1=""],[am_cv_proto_iconv_arg1="const"]) am_cv_proto_iconv="extern size_t iconv (iconv_t cd, $am_cv_proto_iconv_arg1 char * *inbuf, size_t *inbytesleft, char * *outbuf, size_t *outbytesleft);"]) - am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed -e 's/( /(/'` + am_cv_proto_iconv=`echo "[$]am_cv_proto_iconv" | tr -s ' ' | sed 's/( /(/'` AC_MSG_RESULT([$]{ac_t:- }[$]am_cv_proto_iconv) AC_DEFINE_UNQUOTED(ICONV_CONST, $am_cv_proto_iconv_arg1, @@ -594,12 +774,13 @@ AC_FUNC_UTIME_NULL AC_FUNC_ALLOCA AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ fchmod fstat ftruncate strchr readlink link utime utimes lutimes strftime \ + chflags getattrlist \ memmove lchown vsnprintf snprintf vasprintf asprintf setsid strpbrk \ strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \ extattr_get_link sigaction sigprocmask setattrlist getgrouplist \ - initgroups utimensat posix_fallocate attropen setvbuf usleep) + initgroups utimensat posix_fallocate attropen setvbuf nanosleep usleep) dnl cygwin iconv.h defines iconv_open as libiconv_open if test x"$ac_cv_func_iconv_open" != x"yes"; then @@ -647,6 +828,7 @@ AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/syscall.h> +#include <unistd.h> #include <sys/types.h>]], [[syscall(SYS_fallocate, 0, 0, (loff_t)0, (loff_t)0);]])],[rsync_cv_have_sys_fallocate=yes],[rsync_cv_have_sys_fallocate=no])]) if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) @@ -685,7 +867,7 @@ AC_ARG_ENABLE(iconv, AS_HELP_STRING([--disable-iconv],[disable rsync's --iconv option]), [], [enable_iconv=$enable_iconv_open]) AH_TEMPLATE([ICONV_OPTION], -[Define if you want the --iconv option. Specifing a value will set the +[Define if you want the --iconv option. Specifying a value will set the default iconv setting (a NULL means no --iconv processing by default).]) if test x"$enable_iconv" != x"no"; then if test x"$enable_iconv" = x"yes"; then @@ -703,12 +885,12 @@ AC_CACHE_CHECK([whether chown() modifies symlinks],rsync_cv_chown_modifies_symli #endif #include <stdlib.h> #include <errno.h> - main() { +int main(void) { char const *dangling_symlink = "conftest.dangle"; unlink(dangling_symlink); if (symlink("conftest.no-such", dangling_symlink) < 0) abort(); - if (chown(dangling_symlink, getuid(), getgid()) < 0 && errno == ENOENT) exit(1); - exit(0); + if (chown(dangling_symlink, getuid(), getgid()) < 0 && errno == ENOENT) return 1; + return 0; }]])],[rsync_cv_chown_modifies_symlink=yes],[rsync_cv_chown_modifies_symlink=no],[rsync_cv_chown_modifies_symlink=no])]) if test $rsync_cv_chown_modifies_symlink = yes; then AC_DEFINE(CHOWN_MODIFIES_SYMLINK, 1, [Define to 1 if chown modifies symlinks.]) @@ -722,12 +904,12 @@ AC_CACHE_CHECK([whether link() can hard-link symlinks],rsync_cv_can_hardlink_sym #include <stdlib.h> #include <errno.h> #define FILENAME "conftest.dangle" - main() { +int main(void) { unlink(FILENAME); if (symlink("conftest.no-such", FILENAME) < 0) abort(); unlink(FILENAME "2"); - if (link(FILENAME, FILENAME "2") < 0) exit(1); - exit(0); + if (link(FILENAME, FILENAME "2") < 0) return 1; + return 0; }]])],[rsync_cv_can_hardlink_symlink=yes],[rsync_cv_can_hardlink_symlink=no],[rsync_cv_can_hardlink_symlink=no])]) if test $rsync_cv_can_hardlink_symlink = yes; then AC_DEFINE(CAN_HARDLINK_SYMLINK, 1, [Define to 1 if link() can hard-link symlinks.]) @@ -738,15 +920,18 @@ AC_CACHE_CHECK([whether link() can hard-link special files],rsync_cv_can_hardlin #if HAVE_UNISTD_H # include <unistd.h> #endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif #include <stdlib.h> #include <errno.h> #define FILENAME "conftest.fifi" - main() { +int main(void) { unlink(FILENAME); if (mkfifo(FILENAME, 0777) < 0) abort(); unlink(FILENAME "2"); - if (link(FILENAME, FILENAME "2") < 0) exit(1); - exit(0); + if (link(FILENAME, FILENAME "2") < 0) return 1; + return 0; }]])],[rsync_cv_can_hardlink_special=yes],[rsync_cv_can_hardlink_special=no],[rsync_cv_can_hardlink_special=no])]) if test $rsync_cv_can_hardlink_special = yes; then AC_DEFINE(CAN_HARDLINK_SPECIAL, 1, [Define to 1 if link() can hard-link special files.]) @@ -757,9 +942,9 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include <sys/types.h> #include <sys/socket.h> -main() { +int main(void) { int fd[2]; - exit((socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != -1) ? 0 : 1); + return (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != -1) ? 0 : 1; }]])],[rsync_cv_HAVE_SOCKETPAIR=yes],[rsync_cv_HAVE_SOCKETPAIR=no],[rsync_cv_HAVE_SOCKETPAIR=cross])]) if test x"$rsync_cv_HAVE_SOCKETPAIR" = x"yes"; then AC_DEFINE(HAVE_SOCKETPAIR, 1, [Define to 1 if you have the "socketpair" function]) @@ -824,23 +1009,23 @@ fi AC_CACHE_CHECK([for broken readdir],rsync_cv_HAVE_BROKEN_READDIR,[ AC_RUN_IFELSE([AC_LANG_SOURCE([[#include <sys/types.h> #include <dirent.h> -main() { struct dirent *di; DIR *d = opendir("."); di = readdir(d); +int main(void) { struct dirent *di; DIR *d = opendir("."); di = readdir(d); if (di && di->d_name[-2] == '.' && di->d_name[-1] == 0 && -di->d_name[0] == 0) exit(0); exit(1);} ]])],[rsync_cv_HAVE_BROKEN_READDIR=yes],[rsync_cv_HAVE_BROKEN_READDIR=no],[rsync_cv_HAVE_BROKEN_READDIR=cross])]) +di->d_name[0] == 0) return 0; return 1;} ]])],[rsync_cv_HAVE_BROKEN_READDIR=yes],[rsync_cv_HAVE_BROKEN_READDIR=no],[rsync_cv_HAVE_BROKEN_READDIR=cross])]) if test x"$rsync_cv_HAVE_BROKEN_READDIR" = x"yes"; then AC_DEFINE(HAVE_BROKEN_READDIR, 1, [Define to 1 if readdir() is broken]) fi AC_CACHE_CHECK([for utimbuf],rsync_cv_HAVE_STRUCT_UTIMBUF,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> -#include <utime.h>]], [[struct utimbuf tbuf; tbuf.actime = 0; tbuf.modtime = 1; exit(utime("foo.c",&tbuf));]])],[rsync_cv_HAVE_STRUCT_UTIMBUF=yes],[rsync_cv_HAVE_STRUCT_UTIMBUF=no])]) +#include <utime.h>]], [[struct utimbuf tbuf; tbuf.actime = 0; tbuf.modtime = 1; return utime("foo.c",&tbuf);]])],[rsync_cv_HAVE_STRUCT_UTIMBUF=yes],[rsync_cv_HAVE_STRUCT_UTIMBUF=no])]) if test x"$rsync_cv_HAVE_STRUCT_UTIMBUF" = x"yes"; then AC_DEFINE(HAVE_STRUCT_UTIMBUF, 1, [Define to 1 if you have the "struct utimbuf" type]) fi AC_CACHE_CHECK([if gettimeofday takes tz argument],rsync_cv_HAVE_GETTIMEOFDAY_TZ,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/time.h> -#include <unistd.h>]], [[struct timeval tv; exit(gettimeofday(&tv, NULL));]])],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=yes],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=no])]) +#include <unistd.h>]], [[struct timeval tv; return gettimeofday(&tv, NULL);]])],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=yes],[rsync_cv_HAVE_GETTIMEOFDAY_TZ=no])]) if test x"$rsync_cv_HAVE_GETTIMEOFDAY_TZ" != x"no"; then AC_DEFINE(HAVE_GETTIMEOFDAY_TZ, 1, [Define to 1 if gettimeofday() takes a time-zone arg]) fi @@ -849,10 +1034,13 @@ AC_CACHE_CHECK([for C99 vsnprintf],rsync_cv_HAVE_C99_VSNPRINTF,[ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include <sys/types.h> #include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> void foo(const char *format, ...) { va_list ap; int len; - char buf[5]; + static char buf[] = "12345678901234567890"; va_start(ap, format); len = vsnprintf(0, 0, format, ap); @@ -860,10 +1048,8 @@ void foo(const char *format, ...) { if (len != 5) exit(1); if (snprintf(buf, 3, "hello") != 5 || strcmp(buf, "he") != 0) exit(1); - - exit(0); } -main() { foo("hello"); } +int main(void) { foo("hello"); return 0; } ]])],[rsync_cv_HAVE_C99_VSNPRINTF=yes],[rsync_cv_HAVE_C99_VSNPRINTF=no],[rsync_cv_HAVE_C99_VSNPRINTF=cross])]) if test x"$rsync_cv_HAVE_C99_VSNPRINTF" = x"yes"; then AC_DEFINE(HAVE_C99_VSNPRINTF, 1, [Define to 1 if vsprintf has a C99-compatible return value]) @@ -875,15 +1061,15 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[#include <stdlib.h> #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> -main() { +int main(void) { struct stat st; char tpl[20]="/tmp/test.XXXXXX"; int fd = mkstemp(tpl); - if (fd == -1) exit(1); + if (fd == -1) return 1; unlink(tpl); - if (fstat(fd, &st) != 0) exit(1); - if ((st.st_mode & 0777) != 0600) exit(1); - exit(0); + if (fstat(fd, &st) != 0) return 1; + if ((st.st_mode & 0777) != 0600) return 1; + return 0; }]])],[rsync_cv_HAVE_SECURE_MKSTEMP=yes],[rsync_cv_HAVE_SECURE_MKSTEMP=no],[rsync_cv_HAVE_SECURE_MKSTEMP=cross])]) if test x"$rsync_cv_HAVE_SECURE_MKSTEMP" = x"yes"; then case $host_os in @@ -905,7 +1091,10 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include <stdio.h> #include <sys/stat.h> #include <errno.h> -main() { int rc, ec; char *fn = "fifo-test"; +#if HAVE_UNISTD_H +# include <unistd.h> +#endif +int main(void) { int rc, ec; char *fn = "fifo-test"; unlink(fn); rc = mknod(fn,S_IFIFO,0600); ec = errno; unlink(fn); if (rc) {printf("(%d %d) ",rc,ec); return ec;} return 0;}]])],[rsync_cv_MKNOD_CREATES_FIFOS=yes],[rsync_cv_MKNOD_CREATES_FIFOS=no],[rsync_cv_MKNOD_CREATES_FIFOS=cross])]) @@ -918,7 +1107,10 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include <stdio.h> #include <sys/stat.h> #include <errno.h> -main() { int rc, ec; char *fn = "sock-test"; +#if HAVE_UNISTD_H +# include <unistd.h> +#endif +int main(void) { int rc, ec; char *fn = "sock-test"; unlink(fn); rc = mknod(fn,S_IFSOCK,0600); ec = errno; unlink(fn); if (rc) {printf("(%d %d) ",rc,ec); return ec;} return 0;}]])],[rsync_cv_MKNOD_CREATES_SOCKETS=yes],[rsync_cv_MKNOD_CREATES_SOCKETS=no],[rsync_cv_MKNOD_CREATES_SOCKETS=cross])]) @@ -932,7 +1124,7 @@ fi AC_CACHE_CHECK([whether -c -o works],rsync_cv_DASHC_WORKS_WITH_DASHO,[ rm -rf conftest* cat > conftest.$ac_ext <<EOF -int main() { return 0; } +int main(void) { return 0; } EOF ${CC-cc} -c -o conftest..o conftest.$ac_ext if test -f conftest..o; then @@ -959,9 +1151,6 @@ AC_SUBST(BUILD_POPT) AC_SUBST(BUILD_ZLIB) AC_SUBST(MAKE_MAN) -AC_PATH_PROG([STUNNEL], [stunnel], [stunnel], [$PATH$PATH_SEPARATOR/usr/sbin$PATH_SEPARATOR/sbin]) -AC_PATH_PROG([STUNNEL4], [stunnel4], [$STUNNEL], [$PATH$PATH_SEPARATOR/usr/sbin$PATH_SEPARATOR/sbin]) - AC_CHECK_FUNCS(_acl __acl _facl __facl) ################################################# # check for ACL support @@ -979,7 +1168,7 @@ else AC_DEFINE(HAVE_UNIXWARE_ACLS, 1, [true if you have UnixWare ACLs]) AC_DEFINE(SUPPORT_ACLS, 1, [Define to 1 to add support for ACLs]) ;; - solaris*|*cygwin*) + solaris*) AC_MSG_RESULT(Using solaris ACLs) AC_DEFINE(HAVE_SOLARIS_ACLS, 1, [true if you have solaris ACLs]) AC_DEFINE(SUPPORT_ACLS, 1) @@ -1111,8 +1300,3 @@ AC_OUTPUT AC_MSG_RESULT() AC_MSG_RESULT([ rsync ${RSYNC_VERSION} configuration successful]) AC_MSG_RESULT() -if test x$HAVE_YODL2MAN != x1; then - AC_MSG_RESULT([ Note that yodl2man was not found, so pre-existing manpage files will be]) - AC_MSG_RESULT([ used w/o change (if available) -- no .yo file changes will be used.]) - AC_MSG_RESULT() -fi diff --git a/configure.sh b/configure.sh index 1c22db90..58500715 100755 --- a/configure.sh +++ b/configure.sh @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for rsync 3.1.3. +# Generated by GNU Autoconf 2.69 for rsync 3.2.0. # # Report bugs to <http://rsync.samba.org/bugzilla.html>. # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='rsync' PACKAGE_TARNAME='rsync' -PACKAGE_VERSION='3.1.3' -PACKAGE_STRING='rsync 3.1.3' +PACKAGE_VERSION='3.2.0' +PACKAGE_STRING='rsync 3.2.0' PACKAGE_BUGREPORT='http://rsync.samba.org/bugzilla.html' PACKAGE_URL='' @@ -625,8 +625,6 @@ ac_includes_default="\ ac_header_list= ac_subst_vars='LTLIBOBJS -STUNNEL4 -STUNNEL MAKE_MAN BUILD_ZLIB BUILD_POPT @@ -635,10 +633,11 @@ OBJ_RESTORE OBJ_SAVE ALLOCA LIBOBJS +SIMD FAKEROOT_PATH SHELL_PATH -HAVE_YODL2MAN HAVE_REMSH +PYTHON3 PERL MKDIR_P INSTALL_DATA @@ -646,6 +645,10 @@ INSTALL_SCRIPT INSTALL_PROGRAM EGREP GREP +AWK +ac_ct_CXX +CXXFLAGS +CXX CPP OBJEXT EXEEXT @@ -682,6 +685,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -706,6 +710,7 @@ ac_user_opts=' enable_option_checking enable_debug enable_profile +enable_md2man enable_maintainer_mode with_included_popt with_included_zlib @@ -714,9 +719,14 @@ with_rsync_path with_rsyncd_conf with_rsh with_nobody_group +enable_simd enable_largefile enable_ipv6 enable_locale +enable_openssl +enable_xxhash +enable_zstd +enable_lz4 enable_iconv_open enable_iconv enable_acl_support @@ -730,7 +740,10 @@ CFLAGS LDFLAGS LIBS CPPFLAGS -CPP' +CPP +CXX +CXXFLAGS +CCC' # Initialize some variables set by options. @@ -769,6 +782,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1021,6 +1035,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1158,7 +1181,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1271,7 +1294,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures rsync 3.1.3 to adapt to many kinds of systems. +\`configure' configures rsync 3.2.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1311,6 +1334,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1336,7 +1360,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of rsync 3.1.3:";; + short | recursive ) echo "Configuration of rsync 3.2.0:";; esac cat <<\_ACEOF @@ -1346,11 +1370,17 @@ Optional Features: --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --disable-debug disable debugging symbols and features --enable-profile turn on CPU profiling + --disable-md2man disable md2man for man page creation --enable-maintainer-mode turn on extra debug features + --disable-simd disable SIMD optimizations (requires c++) --disable-largefile omit support for large files - --disable-ipv6 do not even try to use IPv6 + --disable-ipv6 turn off IPv6 support --disable-locale disable locale features + --disable-openssl disable openssl crypto library + --disable-xxhash disable xxhash checksums + --disable-zstd disable zstd compression + --disable-lz4 disable LZ4 compression --disable-iconv-open disable all use of iconv_open() function --disable-iconv disable rsync's --iconv option --disable-acl-support disable ACL support @@ -1379,6 +1409,8 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I<include dir> if you have headers in a nonstandard directory <include dir> CPP C preprocessor + CXX C++ compiler command + CXXFLAGS C++ compiler flags Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -1446,7 +1478,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -rsync configure 3.1.3 +rsync configure 3.2.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1535,6 +1567,44 @@ fi } # ac_fn_c_try_cpp +# ac_fn_cxx_try_compile LINENO +# ---------------------------- +# Try to compile conftest.$ac_ext, and return whether this succeeded. +ac_fn_cxx_try_compile () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + rm -f conftest.$ac_objext + if { { ac_try="$ac_compile" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compile") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest.$ac_objext; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_compile + # ac_fn_c_try_run LINENO # ---------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. Assumes @@ -2109,7 +2179,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by rsync $as_me 3.1.3, which was +It was created by rsync $as_me 3.2.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3530,6 +3600,305 @@ ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu +if test -z "$CXX"; then + if test -n "$CCC"; then + CXX=$CCC + else + if test -n "$ac_tool_prefix"; then + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$CXX"; then + ac_cv_prog_CXX="$CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_CXX="$ac_tool_prefix$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CXX=$ac_cv_prog_CXX +if test -n "$CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +$as_echo "$CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$CXX" && break + done +fi +if test -z "$CXX"; then + ac_ct_CXX=$CXX + for ac_prog in g++ c++ gpp aCC CC cxx cc++ cl.exe FCC KCC RCC xlC_r xlC +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_CXX"; then + ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_CXX="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CXX=$ac_cv_prog_ac_ct_CXX +if test -n "$ac_ct_CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CXX" >&5 +$as_echo "$ac_ct_CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$ac_ct_CXX" && break +done + + if test "x$ac_ct_CXX" = x; then + CXX="g++" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CXX=$ac_ct_CXX + fi +fi + + fi +fi +# Provide some information about the compiler. +$as_echo "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +for ac_option in --version -v -V -qversion; do + { { ac_try="$ac_compiler $ac_option >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_compiler $ac_option >&5") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + sed '10a\ +... rest of stderr output deleted ... + 10q' conftest.err >conftest.er1 + cat conftest.er1 >&5 + fi + rm -f conftest.er1 conftest.err + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } +done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C++ compiler" >&5 +$as_echo_n "checking whether we are using the GNU C++ compiler... " >&6; } +if ${ac_cv_cxx_compiler_gnu+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_compiler_gnu=yes +else + ac_compiler_gnu=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_cv_cxx_compiler_gnu=$ac_compiler_gnu + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_compiler_gnu" >&5 +$as_echo "$ac_cv_cxx_compiler_gnu" >&6; } +if test $ac_compiler_gnu = yes; then + GXX=yes +else + GXX= +fi +ac_test_CXXFLAGS=${CXXFLAGS+set} +ac_save_CXXFLAGS=$CXXFLAGS +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CXX accepts -g" >&5 +$as_echo_n "checking whether $CXX accepts -g... " >&6; } +if ${ac_cv_prog_cxx_g+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_save_cxx_werror_flag=$ac_cxx_werror_flag + ac_cxx_werror_flag=yes + ac_cv_prog_cxx_g=no + CXXFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_prog_cxx_g=yes +else + CXXFLAGS="" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + +else + ac_cxx_werror_flag=$ac_save_cxx_werror_flag + CXXFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO"; then : + ac_cv_prog_cxx_g=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ac_cxx_werror_flag=$ac_save_cxx_werror_flag +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_g" >&5 +$as_echo "$ac_cv_prog_cxx_g" >&6; } +if test "$ac_test_CXXFLAGS" = set; then + CXXFLAGS=$ac_save_CXXFLAGS +elif test $ac_cv_prog_cxx_g = yes; then + if test "$GXX" = yes; then + CXXFLAGS="-g -O2" + else + CXXFLAGS="-g" + fi +else + if test "$GXX" = yes; then + CXXFLAGS="-O2" + else + CXXFLAGS= + fi +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +for ac_prog in gawk mawk nawk awk +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_AWK+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$AWK"; then + ac_cv_prog_AWK="$AWK" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_AWK="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +AWK=$ac_cv_prog_AWK +if test -n "$AWK"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 +$as_echo "$AWK" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$AWK" && break +done + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 $as_echo_n "checking for grep that handles long lines and -e... " >&6; } if ${ac_cv_path_GREP+:} false; then : @@ -4128,6 +4497,46 @@ $as_echo "no" >&6; } fi +# Extract the first word of "python3", so it can be a program name with args. +set dummy python3; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTHON3+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTHON3 in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTHON3="$PYTHON3" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PYTHON3="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTHON3=$ac_cv_path_PYTHON3 +if test -n "$PYTHON3"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON3" >&5 +$as_echo "$PYTHON3" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + $as_echo "#define _GNU_SOURCE 1" >>confdefs.h @@ -4147,6 +4556,51 @@ if test x"$enable_profile" = x"yes"; then CFLAGS="$CFLAGS -pg" fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if md2man can create man pages" >&5 +$as_echo_n "checking if md2man can create man pages... " >&6; } +if test x"$ac_cv_path_PYTHON3" = x; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no - python3 not found" >&5 +$as_echo "no - python3 not found" >&6; } + md2man_works=no +else + md2man_out=`"$srcdir/md2man" --test "$srcdir/rsync.1.md" 2>&1` + if test $? = 0; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + md2man_works=yes + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + md2man_works=no + echo "$md2man_out" + fi +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if we require man-page building" >&5 +$as_echo_n "checking if we require man-page building... " >&6; } +# Check whether --enable-md2man was given. +if test "${enable_md2man+set}" = set; then : + enableval=$enable_md2man; +fi + +if test x"$enable_md2man" != x"no"; then + if test -f "$srcdir/rsync.1"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: optional" >&5 +$as_echo "optional" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: required" >&5 +$as_echo "required" >&6; } + if test x"$md2man_works" = x"no"; then + as_fn_error $? "You need python3 and the cmarkgfm OR commonmark python3 lib in order to build man pages. +You can specify --disable-md2man if you want to skip building them." "$LINENO" 5 + fi + fi + MAKE_MAN=man +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + MAKE_MAN='' +fi # Specifically, this turns on panic_action handling. # Check whether --enable-maintainer-mode was given. @@ -4158,7 +4612,6 @@ if test x"$enable_maintainer_mode" = x"yes"; then CFLAGS="$CFLAGS -DMAINTAINER_MODE" fi - # This is needed for our included version of popt. Kind of silly, but # I don't want our version too far out of sync. CFLAGS="$CFLAGS -DHAVE_CONFIG_H" @@ -4302,50 +4755,6 @@ cat >>confdefs.h <<_ACEOF _ACEOF -# Extract the first word of "yodl2man", so it can be a program name with args. -set dummy yodl2man; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_HAVE_YODL2MAN+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$HAVE_YODL2MAN"; then - ac_cv_prog_HAVE_YODL2MAN="$HAVE_YODL2MAN" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_HAVE_YODL2MAN="1" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - - test -z "$ac_cv_prog_HAVE_YODL2MAN" && ac_cv_prog_HAVE_YODL2MAN="0" -fi -fi -HAVE_YODL2MAN=$ac_cv_prog_HAVE_YODL2MAN -if test -n "$HAVE_YODL2MAN"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAVE_YODL2MAN" >&5 -$as_echo "$HAVE_YODL2MAN" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -if test x$HAVE_YODL2MAN = x1; then - MAKE_MAN=man -else - MAKE_MAN=man-copy -fi - # Some programs on solaris are only found in /usr/xpg4/bin (or work better than others versions). # Extract the first word of "sh", so it can be a program name with args. set dummy sh; ac_word=$2 @@ -4462,6 +4871,79 @@ cat >>confdefs.h <<_ACEOF _ACEOF +# SIMD optimizations +SIMD= + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to enable SIMD optimizations" >&5 +$as_echo_n "checking whether to enable SIMD optimizations... " >&6; } +# Check whether --enable-simd was given. +if test "${enable_simd+set}" = set; then : + enableval=$enable_simd; +fi + + +if test x"$enable_simd" != x"no"; then + # For x86-64 SIMD, g++ >=5 or clang++ >=7 is required + if test x"$build_cpu" = x"x86_64"; then + CXX_OK= + if test x"$CXX" != x""; then + CXX_VERSION=`$CXX --version 2>/dev/null | head -n 1` + case "$CXX_VERSION" in + g++*) + CXX_VERSION=`$CXX -dumpversion | sed 's/\..*//g'` + if test "$CXX_VERSION" -ge "5"; then + CXX_OK=yes + fi + ;; + *clang*) + # $CXX -dumpversion would have been ideal, but is broken on older clang + CXX_VERSION=`echo "$CXX_VERSION" | sed 's/.*version //g' | sed 's/\..*//g'` + if test "$CXX_VERSION" -ge "7"; then + CXX_OK=yes + fi + ;; + *) + CXX_VERSION='Unknown' + ;; + esac + else + CXX='No c++' + CXX_VERSION='Unknown' + fi + if test x"$CXX_OK" = x"yes"; then + # AC_MSG_RESULT() is called below. + SIMD="$SIMD x86_64" + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: error" >&5 +$as_echo "error" >&6; } + as_fn_error $? "Failed to find g++ >=5 or clang++ >=7 for SIMD optimizations. +Specify --disable-simd to continue without it. ($CXX, $CXX_VERSION)" "$LINENO" 5 + fi + elif test x"$enable_simd" = x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unavailable" >&5 +$as_echo "unavailable" >&6; } + as_fn_error $? "The SIMD optimizations are currently x86_64 only. +Omit --enable-simd to continue without it." "$LINENO" 5 + fi +fi + +if test x"$SIMD" != x""; then + SIMD=`echo "$SIMD" | sed 's/^ *//'` + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes ($SIMD)" >&5 +$as_echo "yes ($SIMD)" >&6; } + +$as_echo "#define HAVE_SIMD 1" >>confdefs.h + + SIMD=`echo "\\\$(SIMD_$SIMD)" | sed 's/ /) $(SIMD_/g'` + # We only use c++ for its target attribute dispatching, disable unneeded bulky features + CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + # arrgh. libc in some old debian version screwed up the largefile # stuff, getting byte range locking wrong @@ -4482,6 +4964,9 @@ else #include <fcntl.h> #include <sys/types.h> #include <sys/wait.h> +#if HAVE_UNISTD_H +#include <unistd.h> +#endif int main(void) { @@ -4506,7 +4991,7 @@ int main(void) } wait(&status); unlink(tpl); - exit(WEXITSTATUS(status)); + return WEXITSTATUS(status); } _ACEOF @@ -5600,7 +6085,7 @@ for ac_header in sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \ netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \ sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \ popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \ - zlib.h + zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" @@ -5630,7 +6115,11 @@ return makedev(0, 0); } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_header_sys_types_h_makedev=yes + if grep sys/sysmacros.h conftest.err >/dev/null; then + ac_cv_header_sys_types_h_makedev=no + else + ac_cv_header_sys_types_h_makedev=yes + fi else ac_cv_header_sys_types_h_makedev=no fi @@ -5664,6 +6153,338 @@ fi fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to enable use of openssl crypto library" >&5 +$as_echo_n "checking whether to enable use of openssl crypto library... " >&6; } +# Check whether --enable-openssl was given. +if test "${enable_openssl+set}" = set; then : + enableval=$enable_openssl; +fi + + +if test x"$enable_openssl" != x"no"; then + if test x"$ac_cv_header_openssl_md4_h" = x"yes" && test x"$ac_cv_header_openssl_md5_h" = x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing MD5_Init" >&5 +$as_echo_n "checking for library containing MD5_Init... " >&6; } +if ${ac_cv_search_MD5_Init+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char MD5_Init (); +int +main () +{ +return MD5_Init (); + ; + return 0; +} +_ACEOF +for ac_lib in '' crypto; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_MD5_Init=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_MD5_Init+:} false; then : + break +fi +done +if ${ac_cv_search_MD5_Init+:} false; then : + +else + ac_cv_search_MD5_Init=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_MD5_Init" >&5 +$as_echo "$ac_cv_search_MD5_Init" >&6; } +ac_res=$ac_cv_search_MD5_Init +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + $as_echo "#define USE_OPENSSL 1" >>confdefs.h + +else + as_fn_error $? "Failed to find MD5_Init function in openssl crypto lib. +Use --disable-openssl to continue without openssl crypto lib support." "$LINENO" 5 +fi + + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "Failed to find openssl/md4.h and openssl/md5.h for openssl crypto lib support. +Use --disable-openssl to continue without it." "$LINENO" 5 + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to enable xxhash checksum support" >&5 +$as_echo_n "checking whether to enable xxhash checksum support... " >&6; } +# Check whether --enable-xxhash was given. +if test "${enable_xxhash+set}" = set; then : + enableval=$enable_xxhash; +fi + + +if test x"$enable_xxhash" != x"no"; then + if test x"$ac_cv_header_xxhash_h" = x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing XXH64_createState" >&5 +$as_echo_n "checking for library containing XXH64_createState... " >&6; } +if ${ac_cv_search_XXH64_createState+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char XXH64_createState (); +int +main () +{ +return XXH64_createState (); + ; + return 0; +} +_ACEOF +for ac_lib in '' xxhash; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_XXH64_createState=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_XXH64_createState+:} false; then : + break +fi +done +if ${ac_cv_search_XXH64_createState+:} false; then : + +else + ac_cv_search_XXH64_createState=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_XXH64_createState" >&5 +$as_echo "$ac_cv_search_XXH64_createState" >&6; } +ac_res=$ac_cv_search_XXH64_createState +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + $as_echo "#define SUPPORT_XXHASH 1" >>confdefs.h + +else + as_fn_error $? "Failed to find XXH64_createState function in xxhash lib. +Use --disable-xxhash to continue without xxhash checksums." "$LINENO" 5 +fi + + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "Failed to find xxhash.h for xxhash checksum support. +Use --disable-xxhash to continue without it." "$LINENO" 5 + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to enable zstd compression" >&5 +$as_echo_n "checking whether to enable zstd compression... " >&6; } +# Check whether --enable-zstd was given. +if test "${enable_zstd+set}" = set; then : + enableval=$enable_zstd; +fi + + +if test x"$enable_zstd" != x"no"; then + if test x"$ac_cv_header_zstd_h" = x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing ZSTD_minCLevel" >&5 +$as_echo_n "checking for library containing ZSTD_minCLevel... " >&6; } +if ${ac_cv_search_ZSTD_minCLevel+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char ZSTD_minCLevel (); +int +main () +{ +return ZSTD_minCLevel (); + ; + return 0; +} +_ACEOF +for ac_lib in '' zstd; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_ZSTD_minCLevel=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_ZSTD_minCLevel+:} false; then : + break +fi +done +if ${ac_cv_search_ZSTD_minCLevel+:} false; then : + +else + ac_cv_search_ZSTD_minCLevel=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZSTD_minCLevel" >&5 +$as_echo "$ac_cv_search_ZSTD_minCLevel" >&6; } +ac_res=$ac_cv_search_ZSTD_minCLevel +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + $as_echo "#define SUPPORT_ZSTD 1" >>confdefs.h + +else + as_fn_error $? "Failed to find ZSTD_minCLevel function in zstd lib. +Use --disable-zstd to continue without zstd compression." "$LINENO" 5 +fi + + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "Failed to find zstd.h for zstd compression support. +Use --disable-zstd to continue without it." "$LINENO" 5 + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to enable LZ4 compression" >&5 +$as_echo_n "checking whether to enable LZ4 compression... " >&6; } +# Check whether --enable-lz4 was given. +if test "${enable_lz4+set}" = set; then : + enableval=$enable_lz4; +fi + + +if test x"$enable_lz4" != x"no"; then + if test x"$ac_cv_header_lz4_h" = x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing LZ4_compress_default" >&5 +$as_echo_n "checking for library containing LZ4_compress_default... " >&6; } +if ${ac_cv_search_LZ4_compress_default+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char LZ4_compress_default (); +int +main () +{ +return LZ4_compress_default (); + ; + return 0; +} +_ACEOF +for ac_lib in '' lz4; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_LZ4_compress_default=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_LZ4_compress_default+:} false; then : + break +fi +done +if ${ac_cv_search_LZ4_compress_default+:} false; then : + +else + ac_cv_search_LZ4_compress_default=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_LZ4_compress_default" >&5 +$as_echo "$ac_cv_search_LZ4_compress_default" >&6; } +ac_res=$ac_cv_search_LZ4_compress_default +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + $as_echo "#define SUPPORT_LZ4 1" >>confdefs.h + +else + as_fn_error $? "Failed to find LZ4_compress_default function in lz4 lib. +Use --disable-lz4 to continue without lz4 compression." "$LINENO" 5 +fi + + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "Failed to find lz4.h for lz4 compression support. +Use --disable-lz4 to continue without it." "$LINENO" 5 + fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if makedev takes 3 args" >&5 $as_echo_n "checking if makedev takes 3 args... " >&6; } if ${rsync_cv_MAKEDEV_TAKES_3_ARGS+:} false; then : @@ -5690,7 +6511,7 @@ int main(void) { dev_t dev = makedev(0, 5, 7); if (major(dev) != 5 || minor(dev) != 7) - exit(1); + return 1; return 0; } @@ -6109,6 +6930,39 @@ cat >>confdefs.h <<_ACEOF _ACEOF +# The cast to long int works around a bug in the HP C Compiler +# version HP92453-01 B.11.11.23709.GP, which incorrectly rejects +# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# This bug is HP SR number 8606223364. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking size of char*" >&5 +$as_echo_n "checking size of char*... " >&6; } +if ${ac_cv_sizeof_charp+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (char*))" "ac_cv_sizeof_charp" "$ac_includes_default"; then : + +else + if test "$ac_cv_type_charp" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute sizeof (char*) +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_sizeof_charp=0 + fi +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_charp" >&5 +$as_echo "$ac_cv_sizeof_charp" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define SIZEOF_CHARP $ac_cv_sizeof_charp +_ACEOF + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for inline" >&5 $as_echo_n "checking for inline... " >&6; } @@ -6963,7 +7817,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext am_cv_proto_iconv="extern size_t iconv (iconv_t cd, $am_cv_proto_iconv_arg1 char * *inbuf, size_t *inbytesleft, char * *outbuf, size_t *outbytesleft);" fi - am_cv_proto_iconv=`echo "$am_cv_proto_iconv" | tr -s ' ' | sed -e 's/( /(/'` + am_cv_proto_iconv=`echo "$am_cv_proto_iconv" | tr -s ' ' | sed 's/( /(/'` { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${ac_t:- }$am_cv_proto_iconv" >&5 $as_echo "${ac_t:- @@ -7706,12 +8560,13 @@ fi for ac_func in waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ fchmod fstat ftruncate strchr readlink link utime utimes lutimes strftime \ + chflags getattrlist \ memmove lchown vsnprintf snprintf vasprintf asprintf setsid strpbrk \ strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \ extattr_get_link sigaction sigprocmask setattrlist getgrouplist \ - initgroups utimensat posix_fallocate attropen setvbuf usleep + initgroups utimensat posix_fallocate attropen setvbuf nanosleep usleep do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -7835,6 +8690,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include <sys/syscall.h> +#include <unistd.h> #include <sys/types.h> int main () @@ -7980,12 +8836,12 @@ else #endif #include <stdlib.h> #include <errno.h> - main() { +int main(void) { char const *dangling_symlink = "conftest.dangle"; unlink(dangling_symlink); if (symlink("conftest.no-such", dangling_symlink) < 0) abort(); - if (chown(dangling_symlink, getuid(), getgid()) < 0 && errno == ENOENT) exit(1); - exit(0); + if (chown(dangling_symlink, getuid(), getgid()) < 0 && errno == ENOENT) return 1; + return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -8024,12 +8880,12 @@ else #include <stdlib.h> #include <errno.h> #define FILENAME "conftest.dangle" - main() { +int main(void) { unlink(FILENAME); if (symlink("conftest.no-such", FILENAME) < 0) abort(); unlink(FILENAME "2"); - if (link(FILENAME, FILENAME "2") < 0) exit(1); - exit(0); + if (link(FILENAME, FILENAME "2") < 0) return 1; + return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -8065,15 +8921,18 @@ else #if HAVE_UNISTD_H # include <unistd.h> #endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif #include <stdlib.h> #include <errno.h> #define FILENAME "conftest.fifi" - main() { +int main(void) { unlink(FILENAME); if (mkfifo(FILENAME, 0777) < 0) abort(); unlink(FILENAME "2"); - if (link(FILENAME, FILENAME "2") < 0) exit(1); - exit(0); + if (link(FILENAME, FILENAME "2") < 0) return 1; + return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -8109,9 +8968,9 @@ else #include <sys/types.h> #include <sys/socket.h> -main() { +int main(void) { int fd[2]; - exit((socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != -1) ? 0 : 1); + return (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != -1) ? 0 : 1; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -8341,9 +9200,9 @@ else /* end confdefs.h. */ #include <sys/types.h> #include <dirent.h> -main() { struct dirent *di; DIR *d = opendir("."); di = readdir(d); +int main(void) { struct dirent *di; DIR *d = opendir("."); di = readdir(d); if (di && di->d_name[-2] == '.' && di->d_name[-1] == 0 && -di->d_name[0] == 0) exit(0); exit(1);} +di->d_name[0] == 0) return 0; return 1;} _ACEOF if ac_fn_c_try_run "$LINENO"; then : rsync_cv_HAVE_BROKEN_READDIR=yes @@ -8376,7 +9235,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext int main () { -struct utimbuf tbuf; tbuf.actime = 0; tbuf.modtime = 1; exit(utime("foo.c",&tbuf)); +struct utimbuf tbuf; tbuf.actime = 0; tbuf.modtime = 1; return utime("foo.c",&tbuf); ; return 0; } @@ -8409,7 +9268,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext int main () { -struct timeval tv; exit(gettimeofday(&tv, NULL)); +struct timeval tv; return gettimeofday(&tv, NULL); ; return 0; } @@ -8443,10 +9302,13 @@ else #include <sys/types.h> #include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> void foo(const char *format, ...) { va_list ap; int len; - char buf[5]; + static char buf[] = "12345678901234567890"; va_start(ap, format); len = vsnprintf(0, 0, format, ap); @@ -8454,10 +9316,8 @@ void foo(const char *format, ...) { if (len != 5) exit(1); if (snprintf(buf, 3, "hello") != 5 || strcmp(buf, "he") != 0) exit(1); - - exit(0); } -main() { foo("hello"); } +int main(void) { foo("hello"); return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -8494,15 +9354,15 @@ else #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> -main() { +int main(void) { struct stat st; char tpl[20]="/tmp/test.XXXXXX"; int fd = mkstemp(tpl); - if (fd == -1) exit(1); + if (fd == -1) return 1; unlink(tpl); - if (fstat(fd, &st) != 0) exit(1); - if ((st.st_mode & 0777) != 0600) exit(1); - exit(0); + if (fstat(fd, &st) != 0) return 1; + if ((st.st_mode & 0777) != 0600) return 1; + return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : @@ -8547,7 +9407,10 @@ else #include <stdio.h> #include <sys/stat.h> #include <errno.h> -main() { int rc, ec; char *fn = "fifo-test"; +#if HAVE_UNISTD_H +# include <unistd.h> +#endif +int main(void) { int rc, ec; char *fn = "fifo-test"; unlink(fn); rc = mknod(fn,S_IFIFO,0600); ec = errno; unlink(fn); if (rc) {printf("(%d %d) ",rc,ec); return ec;} return 0;} @@ -8585,7 +9448,10 @@ else #include <stdio.h> #include <sys/stat.h> #include <errno.h> -main() { int rc, ec; char *fn = "sock-test"; +#if HAVE_UNISTD_H +# include <unistd.h> +#endif +int main(void) { int rc, ec; char *fn = "sock-test"; unlink(fn); rc = mknod(fn,S_IFSOCK,0600); ec = errno; unlink(fn); if (rc) {printf("(%d %d) ",rc,ec); return ec;} return 0;} @@ -8619,7 +9485,7 @@ else rm -rf conftest* cat > conftest.$ac_ext <<EOF -int main() { return 0; } +int main(void) { return 0; } EOF ${CC-cc} -c -o conftest..o conftest.$ac_ext if test -f conftest..o; then @@ -8649,89 +9515,6 @@ fi -# Extract the first word of "stunnel", so it can be a program name with args. -set dummy stunnel; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_path_STUNNEL+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $STUNNEL in - [\\/]* | ?:[\\/]*) - ac_cv_path_STUNNEL="$STUNNEL" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH$PATH_SEPARATOR/usr/sbin$PATH_SEPARATOR/sbin -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_path_STUNNEL="$as_dir/$ac_word$ac_exec_ext" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - - test -z "$ac_cv_path_STUNNEL" && ac_cv_path_STUNNEL="stunnel" - ;; -esac -fi -STUNNEL=$ac_cv_path_STUNNEL -if test -n "$STUNNEL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STUNNEL" >&5 -$as_echo "$STUNNEL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -# Extract the first word of "stunnel4", so it can be a program name with args. -set dummy stunnel4; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_path_STUNNEL4+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $STUNNEL4 in - [\\/]* | ?:[\\/]*) - ac_cv_path_STUNNEL4="$STUNNEL4" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH$PATH_SEPARATOR/usr/sbin$PATH_SEPARATOR/sbin -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_path_STUNNEL4="$as_dir/$ac_word$ac_exec_ext" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - - test -z "$ac_cv_path_STUNNEL4" && ac_cv_path_STUNNEL4="$STUNNEL" - ;; -esac -fi -STUNNEL4=$ac_cv_path_STUNNEL4 -if test -n "$STUNNEL4"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STUNNEL4" >&5 -$as_echo "$STUNNEL4" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - for ac_func in _acl __acl _facl __facl do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` @@ -8770,7 +9553,7 @@ $as_echo "#define HAVE_UNIXWARE_ACLS 1" >>confdefs.h $as_echo "#define SUPPORT_ACLS 1" >>confdefs.h ;; - solaris*|*cygwin*) + solaris*) { $as_echo "$as_me:${as_lineno-$LINENO}: result: Using solaris ACLs" >&5 $as_echo "Using solaris ACLs" >&6; } @@ -9627,7 +10410,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by rsync $as_me 3.1.3, which was +This file was extended by rsync $as_me 3.2.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -9689,7 +10472,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -rsync config.status 3.1.3 +rsync config.status 3.2.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -9701,6 +10484,7 @@ ac_pwd='$ac_pwd' srcdir='$srcdir' INSTALL='$INSTALL' MKDIR_P='$MKDIR_P' +AWK='$AWK' test -n "\$AWK" || AWK=awk _ACEOF @@ -10417,11 +11201,3 @@ $as_echo "" >&6; } $as_echo " rsync ${RSYNC_VERSION} configuration successful" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: " >&5 $as_echo "" >&6; } -if test x$HAVE_YODL2MAN != x1; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: Note that yodl2man was not found, so pre-existing manpage files will be" >&5 -$as_echo " Note that yodl2man was not found, so pre-existing manpage files will be" >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: result: used w/o change (if available) -- no .yo file changes will be used." >&5 -$as_echo " used w/o change (if available) -- no .yo file changes will be used." >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: result: " >&5 -$as_echo "" >&6; } -fi diff --git a/connection.c b/connection.c index 62818610..1039115c 100644 --- a/connection.c +++ b/connection.c @@ -2,6 +2,7 @@ * Support the max connections option. * * Copyright (C) 1998 Andrew Tridgell + * Copyright (C) 2006-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/define-from-md.awk b/define-from-md.awk new file mode 100755 index 00000000..9f7f2bf3 --- /dev/null +++ b/define-from-md.awk @@ -0,0 +1,41 @@ +#!/usr/bin/awk -f + +# The caller must pass args: -v hfile=NAME rsync.1.md + +BEGIN { + heading = "/* DO NOT EDIT THIS FILE! It is auto-generated from a list of values in " ARGV[1] "! */" + if (hfile ~ /compress/) { + define = "#define DEFAULT_DONT_COMPRESS" + prefix = "*." + } else { + define = "#define DEFAULT_CVSIGNORE" + prefix = "" + } + value_list = "" +} + +/^ > [^ ]+$/ { + gsub(/`/, "") + if (value_list != "") value_list = value_list " " + value_list = value_list prefix $2 + next +} + +value_list ~ /\.gz / && hfile ~ /compress/ { + exit +} + +value_list ~ /SCCS / && hfile ~ /cvsignore/ { + exit +} + +value_list = "" + +END { + if (value_list != "") + print heading "\n\n" define " \"" value_list "\"" > hfile + else { + print "Failed to find a value list in " ARGV[1] " for " hfile + exit 1 + } +} @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -89,8 +89,8 @@ static enum delret delete_dir_contents(char *fname, uint16 flags) if (fp->flags & FLAG_MOUNT_DIR && S_ISDIR(fp->mode)) { if (DEBUG_GTE(DEL, 1)) { rprintf(FINFO, - "mount point, %s, pins parent directory\n", - f_name(fp, NULL)); + "mount point, %s, pins parent directory\n", + f_name(fp, NULL)); } ret = DR_NOT_EMPTY; continue; diff --git a/doc/rsync.sgml b/doc/rsync.sgml index 76a50c26..0f900590 100644 --- a/doc/rsync.sgml +++ b/doc/rsync.sgml @@ -126,7 +126,7 @@ <para><option>-P</option> Display a progress indicator while files are transferred. This should - normally be ommitted if rsync is not run on a terminal. + normally be omitted if rsync is not run on a terminal. </para> </section> @@ -348,4 +348,4 @@ running rsync giving the network directory. <para><ulink url="http://www.ccp14.ac.uk/ccp14admin/rsync/"></ulink></para> </appendix> -</book>
\ No newline at end of file +</book> @@ -2,7 +2,7 @@ * Error codes returned by rsync. * * Copyright (C) 1998-2000 Andrew Tridgell - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -4,7 +4,7 @@ * Copyright (C) 1996-2001 Andrew Tridgell <tridge@samba.org> * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,6 +21,7 @@ */ #include "rsync.h" +#include "default-cvsignore.h" extern int am_server; extern int am_sender; @@ -1051,16 +1052,6 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr, return rule; } -static char default_cvsignore[] = - /* These default ignored items come from the CVS manual. */ - "RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS" - " .make.state .nse_depinfo *~ #* .#* ,* _$* *$" - " *.old *.bak *.BAK *.orig *.rej .del-*" - " *.a *.olb *.o *.obj *.so *.exe" - " *.Z *.elc *.ln core" - /* The rest we added to suit ourself. */ - " .svn/ .git/ .hg/ .bzr/"; - static void get_cvs_excludes(uint32 rflags) { static int initialized = 0; @@ -1070,7 +1061,7 @@ static void get_cvs_excludes(uint32 rflags) return; initialized = 1; - parse_filter_str(&cvs_filter_list, default_cvsignore, + parse_filter_str(&cvs_filter_list, DEFAULT_CVSIGNORE, rule_template(rflags | (protocol_version >= 30 ? FILTRULE_PERISHABLE : 0)), 0); @@ -1286,6 +1277,8 @@ char *get_rule_prefix(filter_rule *rule, const char *pat, int for_xfer, } if (rule->rflags & FILTRULE_EXCLUDE_SELF) *op++ = 'e'; + if (rule->rflags & FILTRULE_XATTR) + *op++ = 'x'; if (rule->rflags & FILTRULE_SENDER_SIDE && (!for_xfer || protocol_version >= 29)) *op++ = 's'; @@ -1404,8 +1397,7 @@ void recv_filter_list(int f_in) char line[BIGPATHBUFLEN]; int xflags = protocol_version >= 29 ? 0 : XFLG_OLD_PREFIXES; int receiver_wants_list = prune_empty_dirs - || (delete_mode - && (!delete_excluded || protocol_version >= 29)); + || (delete_mode && (!delete_excluded || protocol_version >= 29)); unsigned int len; if (!local_server && (am_sender || receiver_wants_list)) { @@ -3,7 +3,7 @@ * * Copyright (C) 1998 Andrew Tridgell * Copyright (C) 2002 Martin Pool - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,12 +26,12 @@ #define ENODATA EAGAIN #endif -/* We want all reads to be aligned on 1K boundries. */ -#define ALIGN_BOUNDRY 1024 +/* We want all reads to be aligned on 1K boundaries. */ +#define ALIGN_BOUNDARY 1024 /* How far past the boundary is an offset? */ -#define ALIGNED_OVERSHOOT(oft) ((oft) & (ALIGN_BOUNDRY-1)) +#define ALIGNED_OVERSHOOT(oft) ((oft) & (ALIGN_BOUNDARY-1)) /* Round up a length to the next boundary */ -#define ALIGNED_LENGTH(len) ((((len) - 1) | (ALIGN_BOUNDRY-1)) + 1) +#define ALIGNED_LENGTH(len) ((((len) - 1) | (ALIGN_BOUNDARY-1)) + 1) extern int sparse_files; @@ -44,6 +44,8 @@ int sparse_end(int f, OFF_T size) { int ret; + sparse_past_write = 0; + if (!sparse_seek) return 0; @@ -322,7 +324,9 @@ int unmap_file(struct map_struct *map) map->p = NULL; } ret = map->status; - memset(map, 0, sizeof map[0]); +#if 0 /* I don't think we really need this. */ + force_memzero(map, sizeof map[0]); +#endif free(map); return ret; @@ -4,7 +4,7 @@ * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2002-2018 Wayne Davison + * Copyright (C) 2002-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -55,6 +55,7 @@ extern int preserve_specials; extern int delete_during; extern int missing_args; extern int eol_nulls; +extern int atimes_ndx; extern int relative_paths; extern int implied_dirs; extern int ignore_perishable; @@ -100,6 +101,7 @@ int flist_cnt = 0; /* how many (non-tmp) file list objects exist */ int file_total = 0; /* total of all active items over all file-lists */ int file_old_total = 0; /* total of active items that will soon be gone */ int flist_eof = 0; /* all the file-lists are now known */ +int xfer_flags_as_varint = 0; #define NORMAL_NAME 0 #define SLASH_ENDING_NAME 1 @@ -140,7 +142,6 @@ void init_flist(void) rprintf(FINFO, "FILE_STRUCT_LEN=%d, EXTRA_LEN=%d\n", (int)FILE_STRUCT_LEN, (int)EXTRA_LEN); } - parse_checksum_choice(); /* Sets checksum_type && xfersum_type */ flist_csum_len = csum_len_for_type(checksum_type, 1); show_filelist_progress = INFO_GTE(FLIST, 1) && xfer_dirs && !am_server && !inc_recurse; @@ -379,7 +380,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, #endif int ndx, int first_ndx) { - static time_t modtime; + static time_t modtime, atime; static mode_t mode; #ifdef SUPPORT_HARD_LINKS static int64 dev; @@ -479,14 +480,20 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, modtime = file->modtime; if (NSEC_BUMP(file) && protocol_version >= 31) xflags |= XMIT_MOD_NSEC; + if (atimes_ndx && !S_ISDIR(mode)) { + if (F_ATIME(file) == atime) + xflags |= XMIT_SAME_ATIME; + else + atime = F_ATIME(file); + } #ifdef SUPPORT_HARD_LINKS if (tmp_dev != -1) { if (protocol_version >= 30) { struct ht_int64_node *np = idev_find(tmp_dev, tmp_ino); - first_hlink_ndx = (int32)(long)np->data - 1; + first_hlink_ndx = (int32)(long)np->data; /* is -1 when new */ if (first_hlink_ndx < 0) { - np->data = (void*)(long)(first_ndx + ndx + 1); + np->data = (void*)(long)(first_ndx + ndx); xflags |= XMIT_HLINK_FIRST; } if (DEBUG_GTE(HLINK, 1)) { @@ -521,11 +528,14 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, if (l2 > 255) xflags |= XMIT_LONG_NAME; - /* We must make sure we don't send a zero flag byte or the - * other end will terminate the flist transfer. Note that - * the use of XMIT_TOP_DIR on a non-dir has no meaning, so - * it's harmless way to add a bit to the first flag byte. */ - if (protocol_version >= 28) { + /* We must avoid sending a flag value of 0 (or an initial byte of + * 0 for the older xflags protocol) or it will signal the end of + * the list. Note that the use of XMIT_TOP_DIR on a non-dir has + * no meaning, so it's a harmless way to add a bit to the first + * flag byte. */ + if (xfer_flags_as_varint) + write_varint(f, xflags ? xflags : XMIT_EXTENDED_FLAGS); + else if (protocol_version >= 28) { if (!xflags && !S_ISDIR(mode)) xflags |= XMIT_TOP_DIR; if ((xflags & 0xFF00) || !xflags) { @@ -565,6 +575,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, write_varint(f, F_MOD_NSEC(file)); if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); + if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME)) + write_varlong(f, atime, 4); if (preserve_uid && !(xflags & XMIT_SAME_UID)) { if (protocol_version < 30) write_int(f, uid); @@ -652,7 +664,7 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, static struct file_struct *recv_file_entry(int f, struct file_list *flist, int xflags) { - static int64 modtime; + static int64 modtime, atime; static mode_t mode; #ifdef SUPPORT_HARD_LINKS static int64 dev; @@ -759,8 +771,10 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x struct file_struct *first = flist->files[first_hlink_ndx - flist->ndx_start]; file_length = F_LENGTH(first); modtime = first->modtime; - modtime_nsec = F_MOD_NSEC(first); + modtime_nsec = F_MOD_NSEC_or_0(first); mode = first->mode; + if (atimes_ndx && !S_ISDIR(mode)) + atime = F_ATIME(first); if (preserve_uid) uid = F_OWNER(first); if (preserve_gid) @@ -799,6 +813,16 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x modtime_nsec = 0; if (!(xflags & XMIT_SAME_MODE)) mode = from_wire_mode(read_int(f)); + if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME)) { + atime = read_varlong(f, 4); +#if SIZEOF_TIME_T < SIZEOF_INT64 + if (!am_generator && (int64)(time_t)atime != atime) { + rprintf(FERROR_XFER, + "Access time value of %s truncated on receiver.\n", + lastname); + } +#endif + } if (chmod_modes && !S_ISLNK(mode) && mode) mode = tweak_mode(mode, chmod_modes); @@ -944,7 +968,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x #ifdef CAN_SET_NSEC if (modtime_nsec) { file->flags |= FLAG_MOD_NSEC; - OPT_EXTRA(file, 0)->unum = modtime_nsec; + F_MOD_NSEC(file) = modtime_nsec; } #endif file->len32 = (uint32)file_length; @@ -955,7 +979,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x exit_cleanup(RERR_UNSUPPORTED); #else file->flags |= FLAG_LENGTH64; - OPT_EXTRA(file, NSEC_BUMP(file))->unum = (uint32)(file_length >> 32); + F_HIGH_LEN(file) = (uint32)(file_length >> 32); #endif } #endif @@ -966,6 +990,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x F_GROUP(file) = gid; file->flags |= gid_flags; } + if (atimes_ndx && !S_ISDIR(mode)) + F_ATIME(file) = atime; if (unsort_ndx) F_NDX(file) = flist->used + flist->ndx_start; @@ -1075,10 +1101,10 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x ino = read_longint(f); } np = idev_find(dev, ino); - ndx = (int32)(long)np->data - 1; + ndx = (int32)(long)np->data; /* is -1 when new */ if (ndx < 0) { - ndx = cnt++; np->data = (void*)(long)cnt; + ndx = cnt++; } F_HL_GNUM(file) = ndx; } @@ -1346,14 +1372,14 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, #ifdef ST_MTIME_NSEC if (st.ST_MTIME_NSEC && protocol_version >= 31) { file->flags |= FLAG_MOD_NSEC; - OPT_EXTRA(file, 0)->unum = st.ST_MTIME_NSEC; + F_MOD_NSEC(file) = st.ST_MTIME_NSEC; } #endif file->len32 = (uint32)st.st_size; #if SIZEOF_CAPITAL_OFF_T >= 8 if (st.st_size > 0xFFFFFFFFu && S_ISREG(st.st_mode)) { file->flags |= FLAG_LENGTH64; - OPT_EXTRA(file, NSEC_BUMP(file))->unum = (uint32)(st.st_size >> 32); + F_HIGH_LEN(file) = (uint32)(st.st_size >> 32); } #endif file->mode = st.st_mode; @@ -1363,6 +1389,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, F_GROUP(file) = st.st_gid; if (am_generator && st.st_uid == our_uid) file->flags |= FLAG_OWNED_BY_US; + if (atimes_ndx && !S_ISDIR(file->mode)) + F_ATIME(file) = st.st_atime; if (basename != thisname) file->dirname = lastdir; @@ -1392,6 +1420,20 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, return file; } +OFF_T get_device_size(int fd, const char *fname) +{ + OFF_T off = lseek(fd, 0, SEEK_END); + + if (off == (OFF_T) -1) { + rsyserr(FERROR, errno, "failed to get device size via seek: %s", fname); + return 0; + } + if (lseek(fd, 0, SEEK_SET) != 0) + rsyserr(FERROR, errno, "failed to seek device back to start: %s", fname); + + return off; +} + /* Only called for temporary file_struct entries created by make_file(). */ void unmake_file(struct file_struct *file) { @@ -1636,6 +1678,7 @@ static void add_dirs_to_tree(int parent_ndx, struct file_list *from_flist, int32 *parent_dp = parent_ndx < 0 ? NULL : F_DIR_NODE_P(dir_flist->sorted[parent_ndx]); + /* The sending side is adding entries to dir_flist in sorted order, so sorted & files are the same. */ flist_expand(dir_flist, dir_cnt); dir_flist->sorted = dir_flist->files; @@ -1703,6 +1746,8 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, interpret_stat_error(fbuf, True); return; } + if (errno == ENOTDIR && (flags & FLAG_PERHAPS_DIR)) + return; io_error |= IOERR_GENERAL; rsyserr(FERROR_XFER, errno, "opendir %s failed", full_fname(fbuf)); return; @@ -1941,6 +1986,18 @@ static void send1extra(int f, struct file_struct *file, struct file_list *flist) free(relname_list); } +static void write_end_of_flist(int f, int send_io_error) +{ + if (xfer_flags_as_varint) { + write_varint(f, 0); + write_varint(f, send_io_error ? io_error : 0); + } else if (send_io_error) { + write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST); + write_varint(f, io_error); + } else + write_byte(f, 0); +} + void send_extra_file_list(int f, int at_least) { struct file_list *flist; @@ -1970,7 +2027,7 @@ void send_extra_file_list(int f, int at_least) else dir_ndx = send_dir_ndx; write_ndx(f, NDX_FLIST_OFFSET - dir_ndx); - flist->parent_ndx = dir_ndx; + flist->parent_ndx = send_dir_ndx; /* the sending side must remember the sorted ndx value */ send1extra(f, file, flist); prev_flags = file->flags; @@ -1992,14 +2049,13 @@ void send_extra_file_list(int f, int at_least) } if (io_error == save_io_error || ignore_errors) - write_byte(f, 0); - else if (use_safe_inc_flist) { - write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST); - write_varint(f, io_error); - } else { + write_end_of_flist(f, 0); + else if (use_safe_inc_flist) + write_end_of_flist(f, 1); + else { if (delete_during) fatal_unsafe_io_error(); - write_byte(f, 0); + write_end_of_flist(f, 0); } if (need_unsorted_flist) { @@ -2328,14 +2384,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) /* Indicate end of file list */ if (io_error == 0 || ignore_errors) - write_byte(f, 0); - else if (use_safe_inc_flist) { - write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST); - write_varint(f, io_error); - } else { + write_end_of_flist(f, 0); + else if (use_safe_inc_flist) + write_end_of_flist(f, 1); + else { if (delete_during && inc_recurse) fatal_unsafe_io_error(); - write_byte(f, 0); + write_end_of_flist(f, 0); } #ifdef SUPPORT_HARD_LINKS @@ -2454,22 +2509,34 @@ struct file_list *recv_file_list(int f, int dir_ndx) dstart = 0; } - while ((flags = read_byte(f)) != 0) { + while (1) { struct file_struct *file; - if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS)) - flags |= read_byte(f) << 8; + if (xfer_flags_as_varint) { + if ((flags = read_varint(f)) == 0) { + int err = read_varint(f); + if (!ignore_errors) + io_error |= err; + break; + } + } else { + if ((flags = read_byte(f)) == 0) + break; + + if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS)) + flags |= read_byte(f) << 8; - if (flags == (XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST)) { - int err; - if (!use_safe_inc_flist) { - rprintf(FERROR, "Invalid flist flag: %x\n", flags); - exit_cleanup(RERR_PROTOCOL); + if (flags == (XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST)) { + int err; + if (!use_safe_inc_flist) { + rprintf(FERROR, "Invalid flist flag: %x\n", flags); + exit_cleanup(RERR_PROTOCOL); + } + err = read_varint(f); + if (!ignore_errors) + io_error |= err; + break; } - err = read_varint(f); - if (!ignore_errors) - io_error |= err; - break; } flist_expand(flist, 1); @@ -3228,6 +3295,7 @@ struct file_list *get_dirlist(char *dirname, int dlen, int flags) int save_xfer_dirs = xfer_dirs; int save_prune_empty_dirs = prune_empty_dirs; int senddir_fd = flags & GDL_IGNORE_FILTER_RULES ? -2 : -1; + int senddir_flags = FLAG_CONTENT_DIR; if (dlen < 0) { dlen = strlcpy(dirbuf, dirname, MAXPATHLEN); @@ -3238,9 +3306,12 @@ struct file_list *get_dirlist(char *dirname, int dlen, int flags) dirlist = flist_new(FLIST_TEMP, "get_dirlist"); + if (flags & GDL_PERHAPS_DIR) + senddir_flags |= FLAG_PERHAPS_DIR; + recurse = 0; xfer_dirs = 1; - send_directory(senddir_fd, dirlist, dirname, dlen, FLAG_CONTENT_DIR); + send_directory(senddir_fd, dirlist, dirname, dlen, senddir_flags); xfer_dirs = save_xfer_dirs; recurse = save_recurse; if (INFO_GTE(PROGRESS, 1)) diff --git a/generator.c b/generator.c index a112da6e..8f9d6035 100644 --- a/generator.c +++ b/generator.c @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -39,6 +39,7 @@ extern int preserve_acls; extern int preserve_xattrs; extern int preserve_links; extern int preserve_devices; +extern int write_devices; extern int preserve_specials; extern int preserve_hard_links; extern int preserve_executability; @@ -58,6 +59,7 @@ extern int human_readable; extern int ignore_existing; extern int ignore_non_existing; extern int want_xattr_optim; +extern int modify_window; extern int inplace; extern int append_mode; extern int make_backups; @@ -76,9 +78,7 @@ extern int fuzzy_basis; extern int always_checksum; extern int flist_csum_len; extern char *partial_dir; -extern int compare_dest; -extern int copy_dest; -extern int link_dest; +extern int alt_dest_type; extern int whole_file; extern int list_only; extern int read_batch; @@ -99,7 +99,7 @@ extern struct file_list *cur_flist, *first_flist, *dir_flist; extern filter_rule_list filter_list, daemon_filter_list; int maybe_ATTRS_REPORT = 0; -int maybe_ATTRS_SET_NANO = 0; +int maybe_ATTRS_ACCURATE_TIME = 0; static dev_t dev_zero; static int deldelay_size = 0, deldelay_cnt = 0; @@ -113,7 +113,7 @@ static int need_retouch_dir_perms; static const char *solo_file = NULL; enum nonregtype { - TYPE_DIR, TYPE_SPECIAL, TYPE_DEVICE, TYPE_SYMLINK + TYPE_DIR, TYPE_SPECIAL, TYPE_DEVICE, TYPE_SYMLINK }; /* Forward declarations. */ @@ -171,10 +171,8 @@ static int remember_delete(struct file_struct *file, const char *fname, int flag deldelay_buf[deldelay_cnt++] = '!'; while (1) { - len = snprintf(deldelay_buf + deldelay_cnt, - deldelay_size - deldelay_cnt, - "%x %s%c", - (int)file->mode, fname, '\0'); + len = snprintf(deldelay_buf + deldelay_cnt, deldelay_size - deldelay_cnt, + "%x %s%c", (int)file->mode, fname, '\0'); if ((deldelay_cnt += len) <= deldelay_size) break; deldelay_cnt -= len; @@ -211,8 +209,7 @@ static int read_delay_line(char *buf, int *flags_p) deldelay_size - deldelay_cnt); if (len == 0) { if (deldelay_cnt) { - rprintf(FERROR, - "ERROR: unexpected EOF in delete-delay file.\n"); + rprintf(FERROR, "ERROR: unexpected EOF in delete-delay file.\n"); } return -1; } @@ -276,6 +273,7 @@ static void do_delayed_deletions(char *delbuf) static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) { static int already_warned = 0; + static struct hashtable *dev_tbl; struct file_list *dirlist; char delbuf[MAXPATHLEN]; int dlen, i; @@ -304,10 +302,16 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) change_local_filter_dir(fbuf, dlen, F_DEPTH(file)); if (one_file_system) { - if (file->flags & FLAG_TOP_DIR) + if (!dev_tbl) + dev_tbl = hashtable_create(16, HT_KEY64); + if (file->flags & FLAG_TOP_DIR) { + hashtable_find(dev_tbl, *fs_dev+1, ""); filesystem_dev = *fs_dev; - else if (filesystem_dev != *fs_dev) - return; + } else if (filesystem_dev != *fs_dev) { + if (!hashtable_find(dev_tbl, *fs_dev+1, NULL)) + return; + filesystem_dev = *fs_dev; /* it's a prior top-dir dev */ + } } dirlist = get_dirlist(fbuf, dlen, 0); @@ -383,12 +387,12 @@ static void do_delete_pass(void) rprintf(FINFO, " \r"); } -static inline int time_diff(STRUCT_STAT *stp, struct file_struct *file) +static inline int mtime_differs(STRUCT_STAT *stp, struct file_struct *file) { #ifdef ST_MTIME_NSEC - return cmp_time(stp->st_mtime, stp->ST_MTIME_NSEC, file->modtime, F_MOD_NSEC(file)); + return !same_time(stp->st_mtime, stp->ST_MTIME_NSEC, file->modtime, F_MOD_NSEC_or_0(file)); #else - return cmp_time(stp->st_mtime, 0L, file->modtime, 0L); + return !same_time(stp->st_mtime, 0, file->modtime, 0); #endif } @@ -446,7 +450,7 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp) { if (S_ISLNK(file->mode)) { #ifdef CAN_SET_SYMLINK_TIMES - if (preserve_times & PRESERVE_LINK_TIMES && time_diff(&sxp->st, file)) + if (preserve_times & PRESERVE_LINK_TIMES && mtime_differs(&sxp->st, file)) return 0; #endif #ifdef CAN_CHMOD_SYMLINK @@ -466,7 +470,7 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp) return 0; #endif } else { - if (preserve_times && time_diff(&sxp->st, file)) + if (preserve_times && mtime_differs(&sxp->st, file)) return 0; if (perms_differ(file, sxp)) return 0; @@ -501,10 +505,13 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre if (iflags & ITEM_LOCAL_CHANGE) iflags |= symlink_timeset_failed_flags; } else if (keep_time - ? time_diff(&sxp->st, file) + ? mtime_differs(&sxp->st, file) : iflags & (ITEM_TRANSFER|ITEM_LOCAL_CHANGE) && !(iflags & ITEM_MATCHED) && (!(iflags & ITEM_XNAME_FOLLOWS) || *xname)) iflags |= ITEM_REPORT_TIME; + if (atimes_ndx && !S_ISDIR(file->mode) && !S_ISLNK(file->mode) + && !same_time(F_ATIME(file), 0, sxp->st.st_atime, 0)) + iflags |= ITEM_REPORT_ATIME; #if !defined HAVE_LCHMOD && !defined HAVE_SETATTRLIST if (S_ISLNK(file->mode)) { ; @@ -518,8 +525,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre iflags |= ITEM_REPORT_PERMS; if (uid_ndx && am_root && (uid_t)F_OWNER(file) != sxp->st.st_uid) iflags |= ITEM_REPORT_OWNER; - if (gid_ndx && !(file->flags & FLAG_SKIP_GROUP) - && sxp->st.st_gid != (gid_t)F_GROUP(file)) + if (gid_ndx && !(file->flags & FLAG_SKIP_GROUP) && sxp->st.st_gid != (gid_t)F_GROUP(file)) iflags |= ITEM_REPORT_GROUP; #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode)) { @@ -593,7 +599,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) if (ignore_times) return 0; - return time_diff(st, file) == 0; + return !mtime_differs(st, file); } @@ -638,14 +644,14 @@ static void sum_sizes_sqroot(struct sum_struct *sum, int64 len) if (c < 0 || c >= max_blength) blength = max_blength; else { - blength = 0; - do { - blength |= c; - if (len < (int64)blength * blength) - blength &= ~c; - c >>= 1; - } while (c >= 8); /* round to multiple of 8 */ - blength = MAX(blength, BLOCK_SIZE); + blength = 0; + do { + blength |= c; + if (len < (int64)blength * blength) + blength &= ~c; + c >>= 1; + } while (c >= 8); /* round to multiple of 8 */ + blength = MAX(blength, BLOCK_SIZE); } } @@ -770,7 +776,7 @@ static struct file_struct *find_fuzzy(struct file_struct *file, struct file_list if (!S_ISREG(fp->mode) || !F_LENGTH(fp) || fp->flags & FLAG_FILE_SENT) continue; - if (F_LENGTH(fp) == F_LENGTH(file) && cmp_time(fp->modtime, 0L, file->modtime, 0L) == 0) { + if (F_LENGTH(fp) == F_LENGTH(file) && same_time(fp->modtime, 0, file->modtime, 0)) { if (DEBUG_GTE(FUZZY, 2)) rprintf(FINFO, "fuzzy size/modtime match for %s\n", f_name(fp, NULL)); *fnamecmp_type_ptr = FNAMECMP_FUZZY + i; @@ -876,27 +882,22 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx, pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname); if (link_stat(cmpbuf, &sxp->st, 0) < 0 || !S_ISREG(sxp->st.st_mode)) continue; - switch (match_level) { - case 0: + if (match_level == 0) { best_match = j; match_level = 1; - /* FALL THROUGH */ - case 1: - if (!unchanged_file(cmpbuf, file, &sxp->st)) - continue; + } + if (!unchanged_file(cmpbuf, file, &sxp->st)) + continue; + if (match_level == 1) { best_match = j; match_level = 2; - /* FALL THROUGH */ - case 2: - if (!unchanged_attrs(cmpbuf, file, sxp)) { - free_stat_x(sxp); - continue; - } + } + if (unchanged_attrs(cmpbuf, file, sxp)) { best_match = j; match_level = 3; break; } - break; + free_stat_x(sxp); } while (basis_dir[++j] != NULL); if (!match_level) @@ -909,17 +910,19 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx, goto got_nothing_for_ya; } - if (match_level == 3 && !copy_dest) { + if (match_level == 3 && alt_dest_type != COPY_DEST) { if (find_exact_for_existing) { - if (link_dest && real_st.st_dev == sxp->st.st_dev && real_st.st_ino == sxp->st.st_ino) + if (alt_dest_type == LINK_DEST && real_st.st_dev == sxp->st.st_dev && real_st.st_ino == sxp->st.st_ino) return -1; if (do_unlink(fname) < 0 && errno != ENOENT) goto got_nothing_for_ya; } #ifdef SUPPORT_HARD_LINKS - if (link_dest) { + if (alt_dest_type == LINK_DEST) { if (!hard_link_one(file, fname, cmpbuf, 1)) goto try_a_copy; + if (atimes_ndx) + set_file_attrs(fname, file, sxp, NULL, 0); if (preserve_hard_links && F_IS_HLINKED(file)) finish_hard_link(file, fname, ndx, &sxp->st, itemizing, code, j); if (!maybe_ATTRS_REPORT && (INFO_GTE(NAME, 2) || stdout_format_has_i > 1)) { @@ -1085,7 +1088,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, if (match_level == 3) { #ifdef SUPPORT_HARD_LINKS - if (link_dest + if (alt_dest_type == LINK_DEST #ifndef CAN_HARDLINK_SYMLINK && !S_ISLNK(file->mode) #endif @@ -1106,7 +1109,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, match_level = 2; if (itemizing && stdout_format_has_i && (INFO_GTE(NAME, 2) || stdout_format_has_i > 1)) { - int chg = compare_dest && type != TYPE_DIR ? 0 + int chg = alt_dest_type == COMPARE_DEST && type != TYPE_DIR ? 0 : ITEM_LOCAL_CHANGE + (match_level == 3 ? ITEM_XNAME_FOLLOWS : 0); char *lp = match_level == 3 ? "" : NULL; itemize(cmpbuf, file, ndx, 0, sxp, chg + ITEM_MATCHED, 0, lp); @@ -1124,35 +1127,40 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, static void list_file_entry(struct file_struct *f) { char permbuf[PERMSTRING_SIZE]; - int64 len; - int colwidth = human_readable ? 14 : 11; + const char *mtime_str = timestring(f->modtime); + int size_width = human_readable ? 14 : 11; + int mtime_width = 1 + strlen(mtime_str); + int atime_width = atimes_ndx ? mtime_width : 0; if (!F_IS_ACTIVE(f)) { /* this can happen if duplicate names were removed */ return; } - permstring(permbuf, f->mode); - len = F_LENGTH(f); - /* TODO: indicate '+' if the entry has an ACL. */ -#ifdef SUPPORT_LINKS - if (preserve_links && S_ISLNK(f->mode)) { - rprintf(FINFO, "%s %*s %s %s -> %s\n", - permbuf, colwidth, human_num(len), - timestring(f->modtime), f_name(f, NULL), - F_SYMLINK(f)); - } else -#endif if (missing_args == 2 && f->mode == 0) { rprintf(FINFO, "%-*s %s\n", - colwidth + 31, "*missing", + 10 + 1 + size_width + mtime_width + atime_width, "*missing", f_name(f, NULL)); } else { - rprintf(FINFO, "%s %*s %s %s\n", - permbuf, colwidth, human_num(len), - timestring(f->modtime), f_name(f, NULL)); + const char *atime_str = atimes_ndx && !S_ISDIR(f->mode) ? timestring(F_ATIME(f)) : ""; + const char *arrow, *lnk; + + permstring(permbuf, f->mode); + +#ifdef SUPPORT_LINKS + if (preserve_links && S_ISLNK(f->mode)) { + arrow = " -> "; + lnk = F_SYMLINK(f); + } else +#endif + arrow = lnk = ""; + + rprintf(FINFO, "%s %*s %s%*s %s%s%s\n", + permbuf, size_width, human_num(F_LENGTH(f)), + timestring(f->modtime), atime_width, atime_str, + f_name(f, NULL), arrow, lnk); } } @@ -1215,7 +1223,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, return; } - maybe_ATTRS_SET_NANO = always_checksum ? ATTRS_SET_NANO : 0; + maybe_ATTRS_ACCURATE_TIME = always_checksum ? ATTRS_ACCURATE_TIME : 0; if (skip_dir) { if (is_below(file, skip_dir)) { @@ -1271,10 +1279,16 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, && (*dn != '.' || dn[1]) /* Avoid an issue with --relative and the "." dir. */ && (!prior_dir_file || strcmp(dn, f_name(prior_dir_file, NULL)) != 0) && flist_find_name(cur_flist, dn, 1) < 0) { - rprintf(FERROR, - "ABORTING due to invalid path from sender: %s/%s\n", - dn, file->basename); - exit_cleanup(RERR_PROTOCOL); + /* The --delete-missing-args option can actually put invalid entries into + * the file list, so if that option was specified, we'll just complain about + * it and allow it. */ + if (missing_args == 2 && file->mode == 0) + rprintf(FERROR, "WARNING: parent dir is absent in the file list: %s\n", dn); + else { + rprintf(FERROR, "ABORTING due to invalid path from sender: %s/%s\n", + dn, file->basename); + exit_cleanup(RERR_PROTOCOL); + } } if (relative_paths && !implied_dirs && do_stat(dn, &sx.st) < 0) { @@ -1303,21 +1317,6 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, } parent_dirname = dn; - if (need_fuzzy_dirlist && S_ISREG(file->mode)) { - int i; - strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf); - for (i = 0; i < fuzzy_basis; i++) { - if (i && pathjoin(fnamecmpbuf, MAXPATHLEN, basis_dir[i-1], dn) >= MAXPATHLEN) - continue; - fuzzy_dirlist[i] = get_dirlist(fnamecmpbuf, -1, GDL_IGNORE_FILTER_RULES); - if (fuzzy_dirlist[i] && fuzzy_dirlist[i]->used == 0) { - flist_free(fuzzy_dirlist[i]); - fuzzy_dirlist[i] = NULL; - } - } - need_fuzzy_dirlist = 0; - } - statret = link_stat(fname, &sx.st, keep_dirlinks && is_dir); stat_errno = errno; } @@ -1377,7 +1376,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, added_perms = 0; if (is_dir < 0) { if (!(preserve_times & PRESERVE_DIR_TIMES)) - return; + goto cleanup; /* In inc_recurse mode we want to make sure any missing * directories get created while we're still processing * the parent dir (which allows us to touch the parent @@ -1412,12 +1411,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, if (file->flags & FLAG_DIR_CREATED) statret = -1; if (!preserve_perms) { /* See comment in non-dir code below. */ - file->mode = dest_mode(file->mode, sx.st.st_mode, - dflt_perms, statret == 0); + file->mode = dest_mode(file->mode, sx.st.st_mode, dflt_perms, statret == 0); } if (statret != 0 && basis_dir[0] != NULL) { - int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, - itemizing, code); + int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, itemizing, code); if (j == -2) { itemizing = 0; code = FNONE; @@ -1439,8 +1436,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, "recv_generator: mkdir %s failed", full_fname(fname)); skipping_dir_contents: - rprintf(FERROR, - "*** Skipping any contents from this failed directory ***\n"); + rprintf(FERROR, "*** Skipping any contents from this failed directory ***\n"); skip_dir = file; file->flags |= FLAG_MISSING_DIR; goto cleanup; @@ -1452,7 +1448,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, copy_xattrs(fnamecmpbuf, fname); #endif if (set_file_attrs(fname, file, real_ret ? NULL : &real_sx, NULL, 0) - && INFO_GTE(NAME, 1) && code != FNONE && f_out != -1) + && INFO_GTE(NAME, 1) && code != FNONE && f_out != -1) rprintf(code, "%s/\n", fname); /* We need to ensure that the dirs in the transfer have both @@ -1495,8 +1491,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, * mode based on the local permissions and some heuristics. */ if (!preserve_perms) { int exists = statret == 0 && !S_ISDIR(sx.st.st_mode); - file->mode = dest_mode(file->mode, sx.st.st_mode, dflt_perms, - exists); + file->mode = dest_mode(file->mode, sx.st.st_mode, dflt_perms, exists); } #ifdef SUPPORT_HARD_LINKS @@ -1519,7 +1514,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, "ignoring unsafe symlink \"%s\" -> \"%s\"\n", fname, sl); } - return; + goto cleanup; } if (statret == 0) { char lnk[MAXPATHLEN]; @@ -1541,15 +1536,14 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, goto cleanup; } } else if (basis_dir[0] != NULL) { - int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, - itemizing, code); + int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, itemizing, code); if (j == -2) { #ifndef CAN_HARDLINK_SYMLINK - if (link_dest) { + if (alt_dest_type == LINK_DEST) { /* Resort to --copy-dest behavior. */ } else #endif - if (!copy_dest) + if (alt_dest_type != COPY_DEST) goto cleanup; itemizing = 0; code = FNONE; @@ -1617,15 +1611,14 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, goto cleanup; } } else if (basis_dir[0] != NULL) { - int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, - itemizing, code); + int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, itemizing, code); if (j == -2) { #ifndef CAN_HARDLINK_SPECIAL - if (link_dest) { + if (alt_dest_type == LINK_DEST) { /* Resort to --copy-dest behavior. */ } else #endif - if (!copy_dest) + if (alt_dest_type != COPY_DEST) goto cleanup; itemizing = 0; code = FNONE; @@ -1681,7 +1674,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, goto cleanup; } - if (update_only > 0 && statret == 0 && time_diff(&sx.st, file) > 0) { + if (update_only > 0 && statret == 0 && file->modtime - sx.st.st_mtime <= modify_window) { if (INFO_GTE(SKIP, 1)) rprintf(FINFO, "%s is newer\n", fname); #ifdef SUPPORT_HARD_LINKS @@ -1693,16 +1686,15 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, fnamecmp_type = FNAMECMP_FNAME; - if (statret == 0 && !S_ISREG(sx.st.st_mode)) { + if (statret == 0 && !(S_ISREG(sx.st.st_mode) || (write_devices && IS_DEVICE(sx.st.st_mode)))) { if (delete_item(fname, sx.st.st_mode, del_opts | DEL_FOR_FILE) != 0) goto cleanup; statret = -1; stat_errno = ENOENT; } - if (basis_dir[0] != NULL && (statret != 0 || !copy_dest)) { - int j = try_dests_reg(file, fname, ndx, fnamecmpbuf, &sx, - statret == 0, itemizing, code); + if (basis_dir[0] != NULL && (statret != 0 || alt_dest_type != COPY_DEST)) { + int j = try_dests_reg(file, fname, ndx, fnamecmpbuf, &sx, statret == 0, itemizing, code); if (j == -2) { if (remove_source_files == 1) goto return_with_success; @@ -1720,14 +1712,30 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, real_ret = statret; if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL - && link_stat(partialptr, &partial_st, 0) == 0 - && S_ISREG(partial_st.st_mode)) { + && link_stat(partialptr, &partial_st, 0) == 0 + && S_ISREG(partial_st.st_mode)) { if (statret != 0) goto prepare_to_open; } else partialptr = NULL; if (statret != 0 && fuzzy_basis) { + if (need_fuzzy_dirlist && S_ISREG(file->mode)) { + const char *dn = file->dirname ? file->dirname : "."; + int i; + strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf); + for (i = 0; i < fuzzy_basis; i++) { + if (i && pathjoin(fnamecmpbuf, MAXPATHLEN, basis_dir[i-1], dn) >= MAXPATHLEN) + continue; + fuzzy_dirlist[i] = get_dirlist(fnamecmpbuf, -1, GDL_IGNORE_FILTER_RULES | GDL_PERHAPS_DIR); + if (fuzzy_dirlist[i] && fuzzy_dirlist[i]->used == 0) { + flist_free(fuzzy_dirlist[i]); + fuzzy_dirlist[i] = NULL; + } + } + need_fuzzy_dirlist = 0; + } + /* Sets fnamecmp_type to FNAMECMP_FUZZY or above. */ fuzzy_file = find_fuzzy(file, fuzzy_dirlist, &fnamecmp_type); if (fuzzy_file) { @@ -1765,7 +1773,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, do_unlink(partialptr); handle_partial_dir(partialptr, PDIR_DELETE); } - set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT | maybe_ATTRS_SET_NANO); + set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT | maybe_ATTRS_ACCURATE_TIME); if (itemizing) itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL); #ifdef SUPPORT_HARD_LINKS @@ -1912,8 +1920,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, } else { if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy) < 0) { rprintf(FWARNING, - "WARNING: file is too large for checksum sending: %s\n", - fnamecmp); + "WARNING: file is too large for checksum sending: %s\n", + fnamecmp); write_sum_head(f_out, NULL); } close(fd); @@ -2068,8 +2076,13 @@ static void touch_up_dirs(struct file_list *flist, int ndx) do_chmod(fname, file->mode); if (need_retouch_dir_times) { STRUCT_STAT st; - if (link_stat(fname, &st, 0) == 0 && time_diff(&st, file)) - set_modtime(fname, file->modtime, F_MOD_NSEC(file), file->mode); + if (link_stat(fname, &st, 0) == 0 && mtime_differs(&st, file)) { + st.st_mtime = file->modtime; +#ifdef ST_MTIME_NSEC + st.ST_MTIME_NSEC = F_MOD_NSEC_or_0(file); +#endif + set_times(fname, &st); + } } if (counter >= loopchk_limit) { if (allowed_lull) @@ -15,8 +15,7 @@ fprintf(stderr, "Unable to stat `%s'\n", *argv); exit(1); } - printf("%ld/%ld\n", (long)major(st.st_dev), - (long)minor(st.st_dev)); + printf("%ld/%ld\n", (long)major(st.st_dev), (long)minor(st.st_dev)); } return 0; diff --git a/getgroups.c b/getgroups.c index 190054b5..a96e04d4 100644 --- a/getgroups.c +++ b/getgroups.c @@ -3,7 +3,7 @@ * `id -G` on Linux, but it's too hard to find a portable equivalent. * * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3 as diff --git a/hashtable.c b/hashtable.c index 2d06a66e..17133dd2 100644 --- a/hashtable.c +++ b/hashtable.c @@ -1,7 +1,7 @@ /* * Routines to provide a memory-efficient hashtable. * - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -66,9 +66,19 @@ void hashtable_destroy(struct hashtable *tbl) free(tbl); } -/* This returns the node for the indicated key, either newly created or - * already existing. Returns NULL if not allocating and not found. */ -void *hashtable_find(struct hashtable *tbl, int64 key, int allocate_if_missing) +/* Returns the node that holds the indicated key if it exists. When it does not + * exist, it returns either NULL (when data_when_new is NULL), or it returns a + * new node with its node->data set to the indicated value. + * + * If your code doesn't know the data value for a new node in advance (usually + * because it doesn't know if a node is new or not) you should pass in a unique + * (non-0) value that you can use to check if the returned node is new. You can + * then overwrite the data with any value you want (even 0) since it only needs + * to be different than whatever data_when_new value you use later on. + * + * This return is a void* just because it might be pointing at a ht_int32_node + * or a ht_int64_node, and that makes the caller's assignment a little easier. */ +void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new) { int key64 = tbl->key64; struct ht_int32_node *node; @@ -79,7 +89,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, int allocate_if_missing) exit_cleanup(RERR_MESSAGEIO); } - if (allocate_if_missing && tbl->entries > HASH_LOAD_LIMIT(tbl->size)) { + if (data_when_new && tbl->entries > HASH_LOAD_LIMIT(tbl->size)) { void *old_nodes = tbl->nodes; int size = tbl->size * 2; int i; @@ -99,8 +109,12 @@ void *hashtable_find(struct hashtable *tbl, int64 key, int allocate_if_missing) int64 move_key = HT_KEY(move_node, key64); if (move_key == 0) continue; - node = hashtable_find(tbl, move_key, 1); - node->data = move_node->data; + if (move_node->data) + hashtable_find(tbl, move_key, move_node->data); + else { + node = hashtable_find(tbl, move_key, ""); + node->data = 0; + } } free(old_nodes); @@ -155,7 +169,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, int allocate_if_missing) if (nkey == key) return node; if (nkey == 0) { - if (!allocate_if_missing) + if (!data_when_new) return NULL; break; } @@ -167,6 +181,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, int allocate_if_missing) ((struct ht_int64_node*)node)->key = key; else node->key = (int32)key; + node->data = data_when_new; tbl->entries++; return node; } @@ -453,16 +468,27 @@ uint32_t hashlittle(const void *key, size_t length) switch(length) /* all the case statements fall through */ { case 12: c+=((uint32_t)k[11])<<24; + /* FALLTHROUGH */ case 11: c+=((uint32_t)k[10])<<16; + /* FALLTHROUGH */ case 10: c+=((uint32_t)k[9])<<8; + /* FALLTHROUGH */ case 9 : c+=k[8]; + /* FALLTHROUGH */ case 8 : b+=((uint32_t)k[7])<<24; + /* FALLTHROUGH */ case 7 : b+=((uint32_t)k[6])<<16; + /* FALLTHROUGH */ case 6 : b+=((uint32_t)k[5])<<8; + /* FALLTHROUGH */ case 5 : b+=k[4]; + /* FALLTHROUGH */ case 4 : a+=((uint32_t)k[3])<<24; + /* FALLTHROUGH */ case 3 : a+=((uint32_t)k[2])<<16; + /* FALLTHROUGH */ case 2 : a+=((uint32_t)k[1])<<8; + /* FALLTHROUGH */ case 1 : a+=k[0]; break; case 0 : return c; diff --git a/help-from-md.awk b/help-from-md.awk new file mode 100755 index 00000000..19007974 --- /dev/null +++ b/help-from-md.awk @@ -0,0 +1,40 @@ +#!/usr/bin/awk -f + +# The caller must pass args: -v hfile=help-NAME.h NAME.NUM.md + +BEGIN { + heading = "/* DO NOT EDIT THIS FILE! It is auto-generated from the option list in " ARGV[1] "! */" + findcomment = hfile + sub("\\.", "\\.", findcomment) + findcomment = "\\[comment\\].*" findcomment + backtick_cnt = 0 + prints = "" +} + +/^```/ { + backtick_cnt++ + next +} + +foundcomment { + if (backtick_cnt > 1) exit + if (backtick_cnt == 1) { + gsub(/"/, "\\\"") + prints = prints "\n rprintf(F,\"" $0 "\\n\");" + } + next +} + +$0 ~ findcomment { + foundcomment = 1 + backtick_cnt = 0 +} + +END { + if (foundcomment && backtick_cnt > 1) + print heading "\n" prints > hfile + else { + print "Failed to find " hfile " section in " ARGV[1] + exit 1 + } +} @@ -4,7 +4,7 @@ * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ extern int list_only; extern int am_sender; extern int inc_recurse; extern int do_xfers; -extern int link_dest; +extern int alt_dest_type; extern int preserve_acls; extern int preserve_xattrs; extern int protocol_version; @@ -48,6 +48,8 @@ extern struct file_list *cur_flist; * we can avoid the pool of dev+inode data. For incremental recursion mode, * the receiver will use a ndx hash to remember old pathnames. */ +static void *data_when_new = ""; + static struct hashtable *dev_tbl; static struct hashtable *prior_hlinks; @@ -57,32 +59,29 @@ static struct file_list *hlink_flist; void init_hard_links(void) { if (am_sender || protocol_version < 30) - dev_tbl = hashtable_create(16, 1); + dev_tbl = hashtable_create(16, HT_KEY64); else if (inc_recurse) - prior_hlinks = hashtable_create(1024, 0); + prior_hlinks = hashtable_create(1024, HT_KEY32); } struct ht_int64_node *idev_find(int64 dev, int64 ino) { static struct ht_int64_node *dev_node = NULL; - struct hashtable *tbl; /* Note that some OSes have a dev == 0, so increment to avoid storing a 0. */ if (!dev_node || dev_node->key != dev+1) { /* We keep a separate hash table of inodes for every device. */ - dev_node = hashtable_find(dev_tbl, dev+1, 1); - if (!(tbl = dev_node->data)) { - tbl = dev_node->data = hashtable_create(512, 1); + dev_node = hashtable_find(dev_tbl, dev+1, data_when_new); + if (dev_node->data == data_when_new) { + dev_node->data = hashtable_create(512, HT_KEY64); if (DEBUG_GTE(HLINK, 3)) { - rprintf(FINFO, - "[%s] created hashtable for dev %s\n", - who_am_i(), big_num(dev)); + rprintf(FINFO, "[%s] created hashtable for dev %s\n", + who_am_i(), big_num(dev)); } } - } else - tbl = dev_node->data; + } - return hashtable_find(tbl, ino, 1); + return hashtable_find(dev_node->data, ino, (void*)-1L); } void idev_destroy(void) @@ -118,15 +117,14 @@ static void match_gnums(int32 *ndx_list, int ndx_count) struct ht_int32_node *node = NULL; int32 gnum, gnum_next; - qsort(ndx_list, ndx_count, sizeof ndx_list[0], - (int (*)()) hlink_compare_gnum); + qsort(ndx_list, ndx_count, sizeof ndx_list[0], (int (*)()) hlink_compare_gnum); for (from = 0; from < ndx_count; from++) { file = hlink_flist->sorted[ndx_list[from]]; gnum = F_HL_GNUM(file); if (inc_recurse) { - node = hashtable_find(prior_hlinks, gnum, 1); - if (!node->data) { + node = hashtable_find(prior_hlinks, gnum, data_when_new); + if (node->data == data_when_new) { if (!(node->data = new_array0(char, 5))) out_of_memory("match_gnums"); assert(gnum >= hlink_flist->ndx_start); @@ -269,7 +267,7 @@ static char *check_prior(struct file_struct *file, int gnum, } if (inc_recurse - && (node = hashtable_find(prior_hlinks, gnum, 0)) != NULL) { + && (node = hashtable_find(prior_hlinks, gnum, NULL)) != NULL) { assert(node->data != NULL); if (CVAL(node->data, 0) != 0) { *prev_ndx_p = -1; @@ -396,7 +394,7 @@ int hard_link_check(struct file_struct *file, int ndx, char *fname, pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname); if (link_stat(cmpbuf, &alt_sx.st, 0) < 0) continue; - if (link_dest) { + if (alt_dest_type == LINK_DEST) { if (prev_st.st_dev != alt_sx.st.st_dev || prev_st.st_ino != alt_sx.st.st_ino) continue; @@ -528,7 +526,7 @@ void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx, if (inc_recurse) { int gnum = F_HL_GNUM(file); - struct ht_int32_node *node = hashtable_find(prior_hlinks, gnum, 0); + struct ht_int32_node *node = hashtable_find(prior_hlinks, gnum, NULL); if (node == NULL) { rprintf(FERROR, "Unable to find a hlink node for %d (%s)\n", gnum, f_name(file, prev_name)); exit_cleanup(RERR_MESSAGEIO); @@ -1,6 +1,6 @@ /* Inline functions for rsync. * - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -1,6 +1,6 @@ /* Inline functions for rsync. * - * Copyright (C) 2008-2018 Wayne Davison + * Copyright (C) 2008-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -4,7 +4,7 @@ * Copyright (C) 1996-2001 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -44,6 +44,7 @@ extern int am_generator; extern int msgs2stderr; extern int inc_recurse; extern int io_error; +extern int batch_fd; extern int eol_nulls; extern int flist_eof; extern int file_total; @@ -67,7 +68,6 @@ extern iconv_t ic_send, ic_recv; int csum_length = SHORT_SUM_LENGTH; /* initial value */ int allowed_lull = 0; -int batch_fd = -1; int msgdone_cnt = 0; int forward_flist_data = 0; BOOL flist_receiving_enabled = False; @@ -251,8 +251,7 @@ static size_t safe_read(int fd, char *buf, size_t len) cnt = select(fd+1, &r_fds, NULL, &e_fds, &tv); if (cnt <= 0) { if (cnt < 0 && errno == EBADF) { - rsyserr(FERROR, errno, "safe_read select failed [%s]", - who_am_i()); + rsyserr(FERROR, errno, "safe_read select failed"); exit_cleanup(RERR_FILEIO); } check_timeout(1, MSK_ALLOW_FLUSH); @@ -271,8 +270,7 @@ static size_t safe_read(int fd, char *buf, size_t len) if (n < 0) { if (errno == EINTR) continue; - rsyserr(FERROR, errno, "safe_read failed to read %ld bytes [%s]", - (long)len, who_am_i()); + rsyserr(FERROR, errno, "safe_read failed to read %ld bytes", (long)len); exit_cleanup(RERR_STREAMIO); } if ((got += (size_t)n) == len) @@ -315,8 +313,8 @@ static void safe_write(int fd, const char *buf, size_t len) if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) { write_failed: rsyserr(FERROR, errno, - "safe_write failed to write %ld bytes to %s [%s]", - (long)len, what_fd_is(fd), who_am_i()); + "safe_write failed to write %ld bytes to %s", + (long)len, what_fd_is(fd)); exit_cleanup(RERR_STREAMIO); } } else { @@ -337,8 +335,7 @@ static void safe_write(int fd, const char *buf, size_t len) cnt = select(fd + 1, NULL, &w_fds, NULL, &tv); if (cnt <= 0) { if (cnt < 0 && errno == EBADF) { - rsyserr(FERROR, errno, "safe_write select failed on %s [%s]", - what_fd_is(fd), who_am_i()); + rsyserr(FERROR, errno, "safe_write select failed on %s", what_fd_is(fd)); exit_cleanup(RERR_FILEIO); } if (io_timeout) @@ -815,7 +812,7 @@ static char *perform_io(size_t needed, int flags) msgs2stderr = 1; iobuf.out_fd = -2; iobuf.out.len = iobuf.msg.len = iobuf.raw_flushing_ends_before = 0; - rsyserr(FERROR_SOCKET, errno, "[%s] write error", who_am_i()); + rsyserr(FERROR_SOCKET, errno, "write error"); drain_multiplex_messages(); exit_cleanup(RERR_SOCKETIO); } @@ -915,7 +912,7 @@ void noop_io_until_death(void) { char buf[1024]; - if (!iobuf.in.buf || !iobuf.out.buf || iobuf.in_fd < 0 || iobuf.out_fd < 0 || kluge_around_eof) + if (!iobuf.in.buf || !iobuf.out.buf || iobuf.in_fd < 0 || iobuf.out_fd < 0 || kluge_around_eof || msgs2stderr) return; kluge_around_eof = 2; @@ -954,8 +951,17 @@ int send_msg(enum msgcode code, const char *buf, size_t len, int convert) } else #endif needed = len + 4 + 3; - if (iobuf.msg.len + needed > iobuf.msg.size) - perform_io(needed, PIO_NEED_MSGROOM); + if (iobuf.msg.len + needed > iobuf.msg.size) { + if (!am_receiver) + perform_io(needed, PIO_NEED_MSGROOM); + else { /* We allow the receiver to increase their iobuf.msg size to avoid a deadlock. */ + size_t old_size = iobuf.msg.size; + restore_iobuf_size(&iobuf.msg); + realloc_xbuf(&iobuf.msg, iobuf.msg.size * 2); + if (iobuf.msg.pos + iobuf.msg.len > old_size) + memcpy(iobuf.msg.buf + old_size, iobuf.msg.buf, iobuf.msg.pos + iobuf.msg.len - old_size); + } + } pos = iobuf.msg.pos + iobuf.msg.len; /* Must be set after any flushing. */ if (pos >= iobuf.msg.size) @@ -1113,8 +1119,7 @@ static void check_for_d_option_error(const char *msg) } if (saw_d) { - rprintf(FWARNING, - "*** Try using \"--old-d\" if remote rsync is <= 2.6.3 ***\n"); + rprintf(FWARNING, "*** Try using \"--old-d\" if remote rsync is <= 2.6.3 ***\n"); } } @@ -1176,7 +1181,7 @@ int read_line(int fd, char *buf, size_t bufsiz, int flags) #ifdef ICONV_OPTION if (flags & RL_CONVERT && iconv_buf.size < bufsiz) - realloc_xbuf(&iconv_buf, bufsiz + 1024); + realloc_xbuf(&iconv_buf, ROUND_UP_1024(bufsiz) + 1024); #endif start: @@ -1982,13 +1987,14 @@ static void sleep_for_bwlimit(int bytes_written) total_written = (sleep_usec - elapsed_usec) * bwlimit / (ONE_SEC/1024); } -void io_flush(int flush_it_all) +void io_flush(int flush_type) { if (iobuf.out.len > iobuf.out_empty_len) { - if (flush_it_all) /* FULL_FLUSH: flush everything in the output buffers */ + if (flush_type == FULL_FLUSH) /* flush everything in the output buffers */ perform_io(iobuf.out.size - iobuf.out_empty_len, PIO_NEED_OUTROOM); - else /* NORMAL_FLUSH: flush at least 1 byte */ + else if (flush_type == NORMAL_FLUSH) /* flush at least 1 byte */ perform_io(iobuf.out.size - iobuf.out.len + 1, PIO_NEED_OUTROOM); + /* MSG_FLUSH: flush iobuf.msg only */ } if (iobuf.msg.len) perform_io(iobuf.msg.size, PIO_NEED_MSGROOM); @@ -2013,20 +2019,20 @@ void write_varint(int f, int32 x) { char b[5]; uchar bit; - int cnt = 4; + int cnt; SIVAL(b, 1, x); - while (cnt > 1 && b[cnt] == 0) - cnt--; + for (cnt = 4; cnt > 1 && b[cnt] == 0; cnt--) {} bit = ((uchar)1<<(7-cnt+1)); + if (CVAL(b, cnt) >= bit) { cnt++; *b = ~(bit-1); } else if (cnt > 1) *b = b[cnt] | ~(bit*2-1); else - *b = b[cnt]; + *b = b[1]; write_buf(f, b, cnt); } @@ -2368,7 +2374,7 @@ void start_write_batch(int fd) * is involved. */ write_int(batch_fd, protocol_version); if (protocol_version >= 30) - write_byte(batch_fd, compat_flags); + write_varint(batch_fd, compat_flags); write_int(batch_fd, checksum_seed); if (am_sender) @@ -1,5 +1,5 @@ /* - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -1,6 +1,6 @@ /* Inline functions for rsync. * - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,6 +23,12 @@ isDigit(const char *ptr) } static inline int +isHexDigit(const char *ptr) +{ + return isxdigit(*(unsigned char *)ptr); +} + +static inline int isPrint(const char *ptr) { return isprint(*(unsigned char *)ptr); diff --git a/latest-year.h b/latest-year.h new file mode 100644 index 00000000..ddbf2b5e --- /dev/null +++ b/latest-year.h @@ -0,0 +1 @@ +#define LATEST_YEAR "2020" diff --git a/lib/compat.c b/lib/compat.c index e0813fc5..26a2aa60 100644 --- a/lib/compat.c +++ b/lib/compat.c @@ -3,7 +3,7 @@ * * Copyright (C) 1998 Andrew Tridgell * Copyright (C) 2002 Martin Pool - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -79,7 +79,7 @@ static char number_separator; #ifndef HAVE_STRPBRK /** - * Find the first ocurrence in @p s of any character in @p accept. + * Find the first occurrence in @p s of any character in @p accept. * * Derived from glibc **/ diff --git a/lib/md5-asm-x86_64.s b/lib/md5-asm-x86_64.s new file mode 100644 index 00000000..a3126151 --- /dev/null +++ b/lib/md5-asm-x86_64.s @@ -0,0 +1,693 @@ +/* + * x86-64 optimized assembler MD5 implementation + * + * Author: Marc Bevand, 2004 + * + * This code was placed in the public domain by the author. The original + * publication can be found at: + * + * https://www.zorinaq.com/papers/md5-amd64.html + */ +/* + * No modifications were made aside from changing the function and file names. + * The MD5_CTX structure as expected here (from OpenSSL) is binary compatible + * with the md_context used by rsync, for the fields accessed. + * + * Benchmarks (in MB/s) C ASM + * - Intel Atom D2700 302 334 + * - Intel i7-7700hq 351 376 + * - AMD ThreadRipper 2950x 728 784 + * + * The original code was also incorporated into OpenSSL. It has since been + * modified there. Those changes have not been made here due to licensing + * incompatibilities. Benchmarks of those changes on the above CPUs did not + * show any significant difference in performance, though. + */ + +.text +.align 16 + +.globl md5_process_asm +.type md5_process_asm,@function +md5_process_asm: + push %rbp + push %rbx + push %r12 + push %r13 # not really useful (r13 is unused) + push %r14 + push %r15 + + # rdi = arg #1 (ctx, MD5_CTX pointer) + # rsi = arg #2 (ptr, data pointer) + # rdx = arg #3 (nbr, number of 16-word blocks to process) + mov %rdi, %rbp # rbp = ctx + shl $6, %rdx # rdx = nbr in bytes + lea (%rsi,%rdx), %rdi # rdi = end + mov 0*4(%rbp), %eax # eax = ctx->A + mov 1*4(%rbp), %ebx # ebx = ctx->B + mov 2*4(%rbp), %ecx # ecx = ctx->C + mov 3*4(%rbp), %edx # edx = ctx->D + # end is 'rdi' + # ptr is 'rsi' + # A is 'eax' + # B is 'ebx' + # C is 'ecx' + # D is 'edx' + + cmp %rdi, %rsi # cmp end with ptr + je 1f # jmp if ptr == end + + # BEGIN of loop over 16-word blocks +2: # save old values of A, B, C, D + mov %eax, %r8d + mov %ebx, %r9d + mov %ecx, %r14d + mov %edx, %r15d + mov 0*4(%rsi), %r10d /* (NEXT STEP) X[0] */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + xor %ecx, %r11d /* y ^ ... */ + lea -680876936(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r11d /* x & ... */ + xor %edx, %r11d /* z ^ ... */ + mov 1*4(%rsi),%r10d /* (NEXT STEP) X[1] */ + add %r11d, %eax /* dst += ... */ + rol $7, %eax /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %ebx, %eax /* dst += x */ + xor %ebx, %r11d /* y ^ ... */ + lea -389564586(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r11d /* x & ... */ + xor %ecx, %r11d /* z ^ ... */ + mov 2*4(%rsi),%r10d /* (NEXT STEP) X[2] */ + add %r11d, %edx /* dst += ... */ + rol $12, %edx /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %eax, %edx /* dst += x */ + xor %eax, %r11d /* y ^ ... */ + lea 606105819(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r11d /* x & ... */ + xor %ebx, %r11d /* z ^ ... */ + mov 3*4(%rsi),%r10d /* (NEXT STEP) X[3] */ + add %r11d, %ecx /* dst += ... */ + rol $17, %ecx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %edx, %ecx /* dst += x */ + xor %edx, %r11d /* y ^ ... */ + lea -1044525330(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r11d /* x & ... */ + xor %eax, %r11d /* z ^ ... */ + mov 4*4(%rsi),%r10d /* (NEXT STEP) X[4] */ + add %r11d, %ebx /* dst += ... */ + rol $22, %ebx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %ecx, %ebx /* dst += x */ + xor %ecx, %r11d /* y ^ ... */ + lea -176418897(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r11d /* x & ... */ + xor %edx, %r11d /* z ^ ... */ + mov 5*4(%rsi),%r10d /* (NEXT STEP) X[5] */ + add %r11d, %eax /* dst += ... */ + rol $7, %eax /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %ebx, %eax /* dst += x */ + xor %ebx, %r11d /* y ^ ... */ + lea 1200080426(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r11d /* x & ... */ + xor %ecx, %r11d /* z ^ ... */ + mov 6*4(%rsi),%r10d /* (NEXT STEP) X[6] */ + add %r11d, %edx /* dst += ... */ + rol $12, %edx /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %eax, %edx /* dst += x */ + xor %eax, %r11d /* y ^ ... */ + lea -1473231341(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r11d /* x & ... */ + xor %ebx, %r11d /* z ^ ... */ + mov 7*4(%rsi),%r10d /* (NEXT STEP) X[7] */ + add %r11d, %ecx /* dst += ... */ + rol $17, %ecx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %edx, %ecx /* dst += x */ + xor %edx, %r11d /* y ^ ... */ + lea -45705983(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r11d /* x & ... */ + xor %eax, %r11d /* z ^ ... */ + mov 8*4(%rsi),%r10d /* (NEXT STEP) X[8] */ + add %r11d, %ebx /* dst += ... */ + rol $22, %ebx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %ecx, %ebx /* dst += x */ + xor %ecx, %r11d /* y ^ ... */ + lea 1770035416(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r11d /* x & ... */ + xor %edx, %r11d /* z ^ ... */ + mov 9*4(%rsi),%r10d /* (NEXT STEP) X[9] */ + add %r11d, %eax /* dst += ... */ + rol $7, %eax /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %ebx, %eax /* dst += x */ + xor %ebx, %r11d /* y ^ ... */ + lea -1958414417(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r11d /* x & ... */ + xor %ecx, %r11d /* z ^ ... */ + mov 10*4(%rsi),%r10d /* (NEXT STEP) X[10] */ + add %r11d, %edx /* dst += ... */ + rol $12, %edx /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %eax, %edx /* dst += x */ + xor %eax, %r11d /* y ^ ... */ + lea -42063(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r11d /* x & ... */ + xor %ebx, %r11d /* z ^ ... */ + mov 11*4(%rsi),%r10d /* (NEXT STEP) X[11] */ + add %r11d, %ecx /* dst += ... */ + rol $17, %ecx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %edx, %ecx /* dst += x */ + xor %edx, %r11d /* y ^ ... */ + lea -1990404162(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r11d /* x & ... */ + xor %eax, %r11d /* z ^ ... */ + mov 12*4(%rsi),%r10d /* (NEXT STEP) X[12] */ + add %r11d, %ebx /* dst += ... */ + rol $22, %ebx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %ecx, %ebx /* dst += x */ + xor %ecx, %r11d /* y ^ ... */ + lea 1804603682(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r11d /* x & ... */ + xor %edx, %r11d /* z ^ ... */ + mov 13*4(%rsi),%r10d /* (NEXT STEP) X[13] */ + add %r11d, %eax /* dst += ... */ + rol $7, %eax /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %ebx, %eax /* dst += x */ + xor %ebx, %r11d /* y ^ ... */ + lea -40341101(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r11d /* x & ... */ + xor %ecx, %r11d /* z ^ ... */ + mov 14*4(%rsi),%r10d /* (NEXT STEP) X[14] */ + add %r11d, %edx /* dst += ... */ + rol $12, %edx /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %eax, %edx /* dst += x */ + xor %eax, %r11d /* y ^ ... */ + lea -1502002290(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r11d /* x & ... */ + xor %ebx, %r11d /* z ^ ... */ + mov 15*4(%rsi),%r10d /* (NEXT STEP) X[15] */ + add %r11d, %ecx /* dst += ... */ + rol $17, %ecx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %edx, %ecx /* dst += x */ + xor %edx, %r11d /* y ^ ... */ + lea 1236535329(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r11d /* x & ... */ + xor %eax, %r11d /* z ^ ... */ + mov 0*4(%rsi),%r10d /* (NEXT STEP) X[0] */ + add %r11d, %ebx /* dst += ... */ + rol $22, %ebx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %ecx, %ebx /* dst += x */ + mov 1*4(%rsi), %r10d /* (NEXT STEP) X[1] */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + mov %edx, %r12d /* (NEXT STEP) z' = %edx */ + not %r11d /* not z */ + lea -165796510(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r12d /* x & z */ + and %ecx, %r11d /* y & (not z) */ + mov 6*4(%rsi),%r10d /* (NEXT STEP) X[6] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %r12d, %eax /* dst += ... */ + mov %ecx, %r12d /* (NEXT STEP) z' = %ecx */ + rol $5, %eax /* dst <<< s */ + add %ebx, %eax /* dst += x */ + not %r11d /* not z */ + lea -1069501632(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r12d /* x & z */ + and %ebx, %r11d /* y & (not z) */ + mov 11*4(%rsi),%r10d /* (NEXT STEP) X[11] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %r12d, %edx /* dst += ... */ + mov %ebx, %r12d /* (NEXT STEP) z' = %ebx */ + rol $9, %edx /* dst <<< s */ + add %eax, %edx /* dst += x */ + not %r11d /* not z */ + lea 643717713(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r12d /* x & z */ + and %eax, %r11d /* y & (not z) */ + mov 0*4(%rsi),%r10d /* (NEXT STEP) X[0] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %r12d, %ecx /* dst += ... */ + mov %eax, %r12d /* (NEXT STEP) z' = %eax */ + rol $14, %ecx /* dst <<< s */ + add %edx, %ecx /* dst += x */ + not %r11d /* not z */ + lea -373897302(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r12d /* x & z */ + and %edx, %r11d /* y & (not z) */ + mov 5*4(%rsi),%r10d /* (NEXT STEP) X[5] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %r12d, %ebx /* dst += ... */ + mov %edx, %r12d /* (NEXT STEP) z' = %edx */ + rol $20, %ebx /* dst <<< s */ + add %ecx, %ebx /* dst += x */ + not %r11d /* not z */ + lea -701558691(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r12d /* x & z */ + and %ecx, %r11d /* y & (not z) */ + mov 10*4(%rsi),%r10d /* (NEXT STEP) X[10] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %r12d, %eax /* dst += ... */ + mov %ecx, %r12d /* (NEXT STEP) z' = %ecx */ + rol $5, %eax /* dst <<< s */ + add %ebx, %eax /* dst += x */ + not %r11d /* not z */ + lea 38016083(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r12d /* x & z */ + and %ebx, %r11d /* y & (not z) */ + mov 15*4(%rsi),%r10d /* (NEXT STEP) X[15] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %r12d, %edx /* dst += ... */ + mov %ebx, %r12d /* (NEXT STEP) z' = %ebx */ + rol $9, %edx /* dst <<< s */ + add %eax, %edx /* dst += x */ + not %r11d /* not z */ + lea -660478335(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r12d /* x & z */ + and %eax, %r11d /* y & (not z) */ + mov 4*4(%rsi),%r10d /* (NEXT STEP) X[4] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %r12d, %ecx /* dst += ... */ + mov %eax, %r12d /* (NEXT STEP) z' = %eax */ + rol $14, %ecx /* dst <<< s */ + add %edx, %ecx /* dst += x */ + not %r11d /* not z */ + lea -405537848(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r12d /* x & z */ + and %edx, %r11d /* y & (not z) */ + mov 9*4(%rsi),%r10d /* (NEXT STEP) X[9] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %r12d, %ebx /* dst += ... */ + mov %edx, %r12d /* (NEXT STEP) z' = %edx */ + rol $20, %ebx /* dst <<< s */ + add %ecx, %ebx /* dst += x */ + not %r11d /* not z */ + lea 568446438(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r12d /* x & z */ + and %ecx, %r11d /* y & (not z) */ + mov 14*4(%rsi),%r10d /* (NEXT STEP) X[14] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %r12d, %eax /* dst += ... */ + mov %ecx, %r12d /* (NEXT STEP) z' = %ecx */ + rol $5, %eax /* dst <<< s */ + add %ebx, %eax /* dst += x */ + not %r11d /* not z */ + lea -1019803690(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r12d /* x & z */ + and %ebx, %r11d /* y & (not z) */ + mov 3*4(%rsi),%r10d /* (NEXT STEP) X[3] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %r12d, %edx /* dst += ... */ + mov %ebx, %r12d /* (NEXT STEP) z' = %ebx */ + rol $9, %edx /* dst <<< s */ + add %eax, %edx /* dst += x */ + not %r11d /* not z */ + lea -187363961(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r12d /* x & z */ + and %eax, %r11d /* y & (not z) */ + mov 8*4(%rsi),%r10d /* (NEXT STEP) X[8] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %r12d, %ecx /* dst += ... */ + mov %eax, %r12d /* (NEXT STEP) z' = %eax */ + rol $14, %ecx /* dst <<< s */ + add %edx, %ecx /* dst += x */ + not %r11d /* not z */ + lea 1163531501(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r12d /* x & z */ + and %edx, %r11d /* y & (not z) */ + mov 13*4(%rsi),%r10d /* (NEXT STEP) X[13] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %r12d, %ebx /* dst += ... */ + mov %edx, %r12d /* (NEXT STEP) z' = %edx */ + rol $20, %ebx /* dst <<< s */ + add %ecx, %ebx /* dst += x */ + not %r11d /* not z */ + lea -1444681467(%eax,%r10d),%eax /* Const + dst + ... */ + and %ebx, %r12d /* x & z */ + and %ecx, %r11d /* y & (not z) */ + mov 2*4(%rsi),%r10d /* (NEXT STEP) X[2] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ecx, %r11d /* (NEXT STEP) z' = %ecx */ + add %r12d, %eax /* dst += ... */ + mov %ecx, %r12d /* (NEXT STEP) z' = %ecx */ + rol $5, %eax /* dst <<< s */ + add %ebx, %eax /* dst += x */ + not %r11d /* not z */ + lea -51403784(%edx,%r10d),%edx /* Const + dst + ... */ + and %eax, %r12d /* x & z */ + and %ebx, %r11d /* y & (not z) */ + mov 7*4(%rsi),%r10d /* (NEXT STEP) X[7] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %ebx, %r11d /* (NEXT STEP) z' = %ebx */ + add %r12d, %edx /* dst += ... */ + mov %ebx, %r12d /* (NEXT STEP) z' = %ebx */ + rol $9, %edx /* dst <<< s */ + add %eax, %edx /* dst += x */ + not %r11d /* not z */ + lea 1735328473(%ecx,%r10d),%ecx /* Const + dst + ... */ + and %edx, %r12d /* x & z */ + and %eax, %r11d /* y & (not z) */ + mov 12*4(%rsi),%r10d /* (NEXT STEP) X[12] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %eax, %r11d /* (NEXT STEP) z' = %eax */ + add %r12d, %ecx /* dst += ... */ + mov %eax, %r12d /* (NEXT STEP) z' = %eax */ + rol $14, %ecx /* dst <<< s */ + add %edx, %ecx /* dst += x */ + not %r11d /* not z */ + lea -1926607734(%ebx,%r10d),%ebx /* Const + dst + ... */ + and %ecx, %r12d /* x & z */ + and %edx, %r11d /* y & (not z) */ + mov 0*4(%rsi),%r10d /* (NEXT STEP) X[0] */ + or %r11d, %r12d /* (y & (not z)) | (x & z) */ + mov %edx, %r11d /* (NEXT STEP) z' = %edx */ + add %r12d, %ebx /* dst += ... */ + mov %edx, %r12d /* (NEXT STEP) z' = %edx */ + rol $20, %ebx /* dst <<< s */ + add %ecx, %ebx /* dst += x */ + mov 5*4(%rsi), %r10d /* (NEXT STEP) X[5] */ + mov %ecx, %r11d /* (NEXT STEP) y' = %ecx */ + lea -378558(%eax,%r10d),%eax /* Const + dst + ... */ + mov 8*4(%rsi),%r10d /* (NEXT STEP) X[8] */ + xor %edx, %r11d /* z ^ ... */ + xor %ebx, %r11d /* x ^ ... */ + add %r11d, %eax /* dst += ... */ + rol $4, %eax /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) y' = %ebx */ + add %ebx, %eax /* dst += x */ + lea -2022574463(%edx,%r10d),%edx /* Const + dst + ... */ + mov 11*4(%rsi),%r10d /* (NEXT STEP) X[11] */ + xor %ecx, %r11d /* z ^ ... */ + xor %eax, %r11d /* x ^ ... */ + add %r11d, %edx /* dst += ... */ + rol $11, %edx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) y' = %eax */ + add %eax, %edx /* dst += x */ + lea 1839030562(%ecx,%r10d),%ecx /* Const + dst + ... */ + mov 14*4(%rsi),%r10d /* (NEXT STEP) X[14] */ + xor %ebx, %r11d /* z ^ ... */ + xor %edx, %r11d /* x ^ ... */ + add %r11d, %ecx /* dst += ... */ + rol $16, %ecx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) y' = %edx */ + add %edx, %ecx /* dst += x */ + lea -35309556(%ebx,%r10d),%ebx /* Const + dst + ... */ + mov 1*4(%rsi),%r10d /* (NEXT STEP) X[1] */ + xor %eax, %r11d /* z ^ ... */ + xor %ecx, %r11d /* x ^ ... */ + add %r11d, %ebx /* dst += ... */ + rol $23, %ebx /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) y' = %ecx */ + add %ecx, %ebx /* dst += x */ + lea -1530992060(%eax,%r10d),%eax /* Const + dst + ... */ + mov 4*4(%rsi),%r10d /* (NEXT STEP) X[4] */ + xor %edx, %r11d /* z ^ ... */ + xor %ebx, %r11d /* x ^ ... */ + add %r11d, %eax /* dst += ... */ + rol $4, %eax /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) y' = %ebx */ + add %ebx, %eax /* dst += x */ + lea 1272893353(%edx,%r10d),%edx /* Const + dst + ... */ + mov 7*4(%rsi),%r10d /* (NEXT STEP) X[7] */ + xor %ecx, %r11d /* z ^ ... */ + xor %eax, %r11d /* x ^ ... */ + add %r11d, %edx /* dst += ... */ + rol $11, %edx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) y' = %eax */ + add %eax, %edx /* dst += x */ + lea -155497632(%ecx,%r10d),%ecx /* Const + dst + ... */ + mov 10*4(%rsi),%r10d /* (NEXT STEP) X[10] */ + xor %ebx, %r11d /* z ^ ... */ + xor %edx, %r11d /* x ^ ... */ + add %r11d, %ecx /* dst += ... */ + rol $16, %ecx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) y' = %edx */ + add %edx, %ecx /* dst += x */ + lea -1094730640(%ebx,%r10d),%ebx /* Const + dst + ... */ + mov 13*4(%rsi),%r10d /* (NEXT STEP) X[13] */ + xor %eax, %r11d /* z ^ ... */ + xor %ecx, %r11d /* x ^ ... */ + add %r11d, %ebx /* dst += ... */ + rol $23, %ebx /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) y' = %ecx */ + add %ecx, %ebx /* dst += x */ + lea 681279174(%eax,%r10d),%eax /* Const + dst + ... */ + mov 0*4(%rsi),%r10d /* (NEXT STEP) X[0] */ + xor %edx, %r11d /* z ^ ... */ + xor %ebx, %r11d /* x ^ ... */ + add %r11d, %eax /* dst += ... */ + rol $4, %eax /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) y' = %ebx */ + add %ebx, %eax /* dst += x */ + lea -358537222(%edx,%r10d),%edx /* Const + dst + ... */ + mov 3*4(%rsi),%r10d /* (NEXT STEP) X[3] */ + xor %ecx, %r11d /* z ^ ... */ + xor %eax, %r11d /* x ^ ... */ + add %r11d, %edx /* dst += ... */ + rol $11, %edx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) y' = %eax */ + add %eax, %edx /* dst += x */ + lea -722521979(%ecx,%r10d),%ecx /* Const + dst + ... */ + mov 6*4(%rsi),%r10d /* (NEXT STEP) X[6] */ + xor %ebx, %r11d /* z ^ ... */ + xor %edx, %r11d /* x ^ ... */ + add %r11d, %ecx /* dst += ... */ + rol $16, %ecx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) y' = %edx */ + add %edx, %ecx /* dst += x */ + lea 76029189(%ebx,%r10d),%ebx /* Const + dst + ... */ + mov 9*4(%rsi),%r10d /* (NEXT STEP) X[9] */ + xor %eax, %r11d /* z ^ ... */ + xor %ecx, %r11d /* x ^ ... */ + add %r11d, %ebx /* dst += ... */ + rol $23, %ebx /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) y' = %ecx */ + add %ecx, %ebx /* dst += x */ + lea -640364487(%eax,%r10d),%eax /* Const + dst + ... */ + mov 12*4(%rsi),%r10d /* (NEXT STEP) X[12] */ + xor %edx, %r11d /* z ^ ... */ + xor %ebx, %r11d /* x ^ ... */ + add %r11d, %eax /* dst += ... */ + rol $4, %eax /* dst <<< s */ + mov %ebx, %r11d /* (NEXT STEP) y' = %ebx */ + add %ebx, %eax /* dst += x */ + lea -421815835(%edx,%r10d),%edx /* Const + dst + ... */ + mov 15*4(%rsi),%r10d /* (NEXT STEP) X[15] */ + xor %ecx, %r11d /* z ^ ... */ + xor %eax, %r11d /* x ^ ... */ + add %r11d, %edx /* dst += ... */ + rol $11, %edx /* dst <<< s */ + mov %eax, %r11d /* (NEXT STEP) y' = %eax */ + add %eax, %edx /* dst += x */ + lea 530742520(%ecx,%r10d),%ecx /* Const + dst + ... */ + mov 2*4(%rsi),%r10d /* (NEXT STEP) X[2] */ + xor %ebx, %r11d /* z ^ ... */ + xor %edx, %r11d /* x ^ ... */ + add %r11d, %ecx /* dst += ... */ + rol $16, %ecx /* dst <<< s */ + mov %edx, %r11d /* (NEXT STEP) y' = %edx */ + add %edx, %ecx /* dst += x */ + lea -995338651(%ebx,%r10d),%ebx /* Const + dst + ... */ + mov 0*4(%rsi),%r10d /* (NEXT STEP) X[0] */ + xor %eax, %r11d /* z ^ ... */ + xor %ecx, %r11d /* x ^ ... */ + add %r11d, %ebx /* dst += ... */ + rol $23, %ebx /* dst <<< s */ + mov %ecx, %r11d /* (NEXT STEP) y' = %ecx */ + add %ecx, %ebx /* dst += x */ + mov 0*4(%rsi), %r10d /* (NEXT STEP) X[0] */ + mov $0xffffffff, %r11d + xor %edx, %r11d /* (NEXT STEP) not z' = not %edx*/ + lea -198630844(%eax,%r10d),%eax /* Const + dst + ... */ + or %ebx, %r11d /* x | ... */ + xor %ecx, %r11d /* y ^ ... */ + add %r11d, %eax /* dst += ... */ + mov 7*4(%rsi),%r10d /* (NEXT STEP) X[7] */ + mov $0xffffffff, %r11d + rol $6, %eax /* dst <<< s */ + xor %ecx, %r11d /* (NEXT STEP) not z' = not %ecx */ + add %ebx, %eax /* dst += x */ + lea 1126891415(%edx,%r10d),%edx /* Const + dst + ... */ + or %eax, %r11d /* x | ... */ + xor %ebx, %r11d /* y ^ ... */ + add %r11d, %edx /* dst += ... */ + mov 14*4(%rsi),%r10d /* (NEXT STEP) X[14] */ + mov $0xffffffff, %r11d + rol $10, %edx /* dst <<< s */ + xor %ebx, %r11d /* (NEXT STEP) not z' = not %ebx */ + add %eax, %edx /* dst += x */ + lea -1416354905(%ecx,%r10d),%ecx /* Const + dst + ... */ + or %edx, %r11d /* x | ... */ + xor %eax, %r11d /* y ^ ... */ + add %r11d, %ecx /* dst += ... */ + mov 5*4(%rsi),%r10d /* (NEXT STEP) X[5] */ + mov $0xffffffff, %r11d + rol $15, %ecx /* dst <<< s */ + xor %eax, %r11d /* (NEXT STEP) not z' = not %eax */ + add %edx, %ecx /* dst += x */ + lea -57434055(%ebx,%r10d),%ebx /* Const + dst + ... */ + or %ecx, %r11d /* x | ... */ + xor %edx, %r11d /* y ^ ... */ + add %r11d, %ebx /* dst += ... */ + mov 12*4(%rsi),%r10d /* (NEXT STEP) X[12] */ + mov $0xffffffff, %r11d + rol $21, %ebx /* dst <<< s */ + xor %edx, %r11d /* (NEXT STEP) not z' = not %edx */ + add %ecx, %ebx /* dst += x */ + lea 1700485571(%eax,%r10d),%eax /* Const + dst + ... */ + or %ebx, %r11d /* x | ... */ + xor %ecx, %r11d /* y ^ ... */ + add %r11d, %eax /* dst += ... */ + mov 3*4(%rsi),%r10d /* (NEXT STEP) X[3] */ + mov $0xffffffff, %r11d + rol $6, %eax /* dst <<< s */ + xor %ecx, %r11d /* (NEXT STEP) not z' = not %ecx */ + add %ebx, %eax /* dst += x */ + lea -1894986606(%edx,%r10d),%edx /* Const + dst + ... */ + or %eax, %r11d /* x | ... */ + xor %ebx, %r11d /* y ^ ... */ + add %r11d, %edx /* dst += ... */ + mov 10*4(%rsi),%r10d /* (NEXT STEP) X[10] */ + mov $0xffffffff, %r11d + rol $10, %edx /* dst <<< s */ + xor %ebx, %r11d /* (NEXT STEP) not z' = not %ebx */ + add %eax, %edx /* dst += x */ + lea -1051523(%ecx,%r10d),%ecx /* Const + dst + ... */ + or %edx, %r11d /* x | ... */ + xor %eax, %r11d /* y ^ ... */ + add %r11d, %ecx /* dst += ... */ + mov 1*4(%rsi),%r10d /* (NEXT STEP) X[1] */ + mov $0xffffffff, %r11d + rol $15, %ecx /* dst <<< s */ + xor %eax, %r11d /* (NEXT STEP) not z' = not %eax */ + add %edx, %ecx /* dst += x */ + lea -2054922799(%ebx,%r10d),%ebx /* Const + dst + ... */ + or %ecx, %r11d /* x | ... */ + xor %edx, %r11d /* y ^ ... */ + add %r11d, %ebx /* dst += ... */ + mov 8*4(%rsi),%r10d /* (NEXT STEP) X[8] */ + mov $0xffffffff, %r11d + rol $21, %ebx /* dst <<< s */ + xor %edx, %r11d /* (NEXT STEP) not z' = not %edx */ + add %ecx, %ebx /* dst += x */ + lea 1873313359(%eax,%r10d),%eax /* Const + dst + ... */ + or %ebx, %r11d /* x | ... */ + xor %ecx, %r11d /* y ^ ... */ + add %r11d, %eax /* dst += ... */ + mov 15*4(%rsi),%r10d /* (NEXT STEP) X[15] */ + mov $0xffffffff, %r11d + rol $6, %eax /* dst <<< s */ + xor %ecx, %r11d /* (NEXT STEP) not z' = not %ecx */ + add %ebx, %eax /* dst += x */ + lea -30611744(%edx,%r10d),%edx /* Const + dst + ... */ + or %eax, %r11d /* x | ... */ + xor %ebx, %r11d /* y ^ ... */ + add %r11d, %edx /* dst += ... */ + mov 6*4(%rsi),%r10d /* (NEXT STEP) X[6] */ + mov $0xffffffff, %r11d + rol $10, %edx /* dst <<< s */ + xor %ebx, %r11d /* (NEXT STEP) not z' = not %ebx */ + add %eax, %edx /* dst += x */ + lea -1560198380(%ecx,%r10d),%ecx /* Const + dst + ... */ + or %edx, %r11d /* x | ... */ + xor %eax, %r11d /* y ^ ... */ + add %r11d, %ecx /* dst += ... */ + mov 13*4(%rsi),%r10d /* (NEXT STEP) X[13] */ + mov $0xffffffff, %r11d + rol $15, %ecx /* dst <<< s */ + xor %eax, %r11d /* (NEXT STEP) not z' = not %eax */ + add %edx, %ecx /* dst += x */ + lea 1309151649(%ebx,%r10d),%ebx /* Const + dst + ... */ + or %ecx, %r11d /* x | ... */ + xor %edx, %r11d /* y ^ ... */ + add %r11d, %ebx /* dst += ... */ + mov 4*4(%rsi),%r10d /* (NEXT STEP) X[4] */ + mov $0xffffffff, %r11d + rol $21, %ebx /* dst <<< s */ + xor %edx, %r11d /* (NEXT STEP) not z' = not %edx */ + add %ecx, %ebx /* dst += x */ + lea -145523070(%eax,%r10d),%eax /* Const + dst + ... */ + or %ebx, %r11d /* x | ... */ + xor %ecx, %r11d /* y ^ ... */ + add %r11d, %eax /* dst += ... */ + mov 11*4(%rsi),%r10d /* (NEXT STEP) X[11] */ + mov $0xffffffff, %r11d + rol $6, %eax /* dst <<< s */ + xor %ecx, %r11d /* (NEXT STEP) not z' = not %ecx */ + add %ebx, %eax /* dst += x */ + lea -1120210379(%edx,%r10d),%edx /* Const + dst + ... */ + or %eax, %r11d /* x | ... */ + xor %ebx, %r11d /* y ^ ... */ + add %r11d, %edx /* dst += ... */ + mov 2*4(%rsi),%r10d /* (NEXT STEP) X[2] */ + mov $0xffffffff, %r11d + rol $10, %edx /* dst <<< s */ + xor %ebx, %r11d /* (NEXT STEP) not z' = not %ebx */ + add %eax, %edx /* dst += x */ + lea 718787259(%ecx,%r10d),%ecx /* Const + dst + ... */ + or %edx, %r11d /* x | ... */ + xor %eax, %r11d /* y ^ ... */ + add %r11d, %ecx /* dst += ... */ + mov 9*4(%rsi),%r10d /* (NEXT STEP) X[9] */ + mov $0xffffffff, %r11d + rol $15, %ecx /* dst <<< s */ + xor %eax, %r11d /* (NEXT STEP) not z' = not %eax */ + add %edx, %ecx /* dst += x */ + lea -343485551(%ebx,%r10d),%ebx /* Const + dst + ... */ + or %ecx, %r11d /* x | ... */ + xor %edx, %r11d /* y ^ ... */ + add %r11d, %ebx /* dst += ... */ + mov 0*4(%rsi),%r10d /* (NEXT STEP) X[0] */ + mov $0xffffffff, %r11d + rol $21, %ebx /* dst <<< s */ + xor %edx, %r11d /* (NEXT STEP) not z' = not %edx */ + add %ecx, %ebx /* dst += x */ + # add old values of A, B, C, D + add %r8d, %eax + add %r9d, %ebx + add %r14d, %ecx + add %r15d, %edx + + # loop control + add $64, %rsi # ptr += 64 + cmp %rdi, %rsi # cmp end with ptr + jb 2b # jmp if ptr < end + # END of loop over 16-word blocks +1: + mov %eax, 0*4(%rbp) # ctx->A = A + mov %ebx, 1*4(%rbp) # ctx->B = B + mov %ecx, 2*4(%rbp) # ctx->C = C + mov %edx, 3*4(%rbp) # ctx->D = D + + pop %r15 + pop %r14 + pop %r13 # not really useful (r13 is unused) + pop %r12 + pop %rbx + pop %rbp + ret +.L_md5_process_asm_end: +.size md5_process_asm,.L_md5_process_asm_end-md5_process_asm @@ -2,6 +2,7 @@ * RFC 1321 compliant MD5 implementation * * Copyright (C) 2001-2003 Christophe Devine + * Copyright (C) 2007-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,6 +20,7 @@ #include "rsync.h" +#ifndef USE_OPENSSL void md5_begin(md_context *ctx) { ctx->A = 0x67452301; @@ -146,6 +148,10 @@ static void md5_process(md_context *ctx, const uchar data[CSUM_CHUNK]) ctx->D += D; } +#if defined(HAVE_SIMD) && (CSUM_CHUNK == 64) +extern void md5_process_asm(md_context *ctx, const void *data, size_t num); +#endif + void md5_update(md_context *ctx, const uchar *input, uint32 length) { uint32 left, fill; @@ -170,11 +176,20 @@ void md5_update(md_context *ctx, const uchar *input, uint32 length) left = 0; } +#if defined(HAVE_SIMD) && (CSUM_CHUNK == 64) + if (length >= CSUM_CHUNK) { + uint32 chunks = length / CSUM_CHUNK; + md5_process_asm(ctx, input, chunks); + length -= chunks * CSUM_CHUNK; + input += chunks * CSUM_CHUNK; + } +#else while (length >= CSUM_CHUNK) { md5_process(ctx, input); length -= CSUM_CHUNK; input += CSUM_CHUNK; } +#endif if (length) memcpy(ctx->buffer + left, input, length); @@ -206,6 +221,9 @@ void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN]) SIVALu(digest, 8, ctx->C); SIVALu(digest, 12, ctx->D); } +#endif + +#ifdef TEST_MD5 void get_md5(uchar *out, const uchar *input, int n) { @@ -215,8 +233,6 @@ void get_md5(uchar *out, const uchar *input, int n) md5_result(&ctx, out); } -#ifdef TEST_MD5 - #include <stdlib.h> #include <stdio.h> diff --git a/lib/mdfour.c b/lib/mdfour.c index 7254686d..6203658d 100644 --- a/lib/mdfour.c +++ b/lib/mdfour.c @@ -4,7 +4,7 @@ * An implementation of MD4 designed for use in the SMB authentication protocol. * * Copyright (C) 1997-1998 Andrew Tridgell - * Copyright (C) 2005-2018 Wayne Davison + * Copyright (C) 2005-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -193,6 +193,8 @@ void mdfour_result(md_context *md, uchar digest[MD4_DIGEST_LEN]) copy4(digest+12, m->D); } +#ifdef TEST_MDFOUR + void mdfour(uchar digest[MD4_DIGEST_LEN], uchar *in, int length) { md_context md; @@ -201,7 +203,6 @@ void mdfour(uchar digest[MD4_DIGEST_LEN], uchar *in, int length) mdfour_result(&md, digest); } -#ifdef TEST_MDFOUR int protocol_version = 28; static void file_checksum1(char *fname) diff --git a/lib/mdigest.h b/lib/mdigest.h index e0e33ed3..c3d22baa 100644 --- a/lib/mdigest.h +++ b/lib/mdigest.h @@ -1,11 +1,24 @@ /* The include file for both the MD4 and MD5 routines. */ +#ifdef USE_OPENSSL +#include "openssl/md4.h" +#include "openssl/md5.h" +#endif + #define MD4_DIGEST_LEN 16 #define MD5_DIGEST_LEN 16 #define MAX_DIGEST_LEN MD5_DIGEST_LEN #define CSUM_CHUNK 64 +#define CSUM_NONE 0 +#define CSUM_MD4_ARCHAIC 1 +#define CSUM_MD4_BUSTED 2 +#define CSUM_MD4_OLD 3 +#define CSUM_MD4 4 +#define CSUM_MD5 5 +#define CSUM_XXH64 6 + typedef struct { uint32 A, B, C, D; uint32 totalN; /* bit count, lower 32 bits */ @@ -17,10 +30,13 @@ void mdfour_begin(md_context *md); void mdfour_update(md_context *md, const uchar *in, uint32 length); void mdfour_result(md_context *md, uchar digest[MD4_DIGEST_LEN]); -void get_mdfour(uchar digest[MD4_DIGEST_LEN], const uchar *in, int length); +#ifndef USE_OPENSSL +#define MD5_CTX md_context +#define MD5_Init md5_begin +#define MD5_Update md5_update +#define MD5_Final(digest, cptr) md5_result(cptr, digest) void md5_begin(md_context *ctx); void md5_update(md_context *ctx, const uchar *input, uint32 length); void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN]); - -void get_md5(uchar digest[MD5_DIGEST_LEN], const uchar *input, int n); +#endif diff --git a/lib/permstring.c b/lib/permstring.c index b83826c6..ab8e26cb 100644 --- a/lib/permstring.c +++ b/lib/permstring.c @@ -4,7 +4,7 @@ * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/lib/pool_alloc.c b/lib/pool_alloc.c index 5856d591..a70a3f1a 100644 --- a/lib/pool_alloc.c +++ b/lib/pool_alloc.c @@ -49,15 +49,15 @@ pool_create(size_t size, size_t quantum, void (*bomb)(const char *), int flags) { struct alloc_pool *pool; - if (!(pool = new0(struct alloc_pool))) - return NULL; - if ((MINALIGN & (MINALIGN - 1)) != 0) { if (bomb) (*bomb)("Compiler error: MINALIGN is not a power of 2\n"); return NULL; } + if (!(pool = new0(struct alloc_pool))) + return NULL; + if (!size) size = POOL_DEF_EXTENT; if (!quantum) diff --git a/lib/snprintf.c b/lib/snprintf.c index c17868f0..52fdd11f 100644 --- a/lib/snprintf.c +++ b/lib/snprintf.c @@ -34,7 +34,7 @@ * probably requires libm on most operating systems. Don't yet * support the exponent (e,E) and sigfig (g,G). Also, fmtint() * was pretty badly broken, it just wasn't being exercised in ways - * which showed it, so that's been fixed. Also, formated the code + * which showed it, so that's been fixed. Also, formatted the code * to mutt conventions, and removed dead code left over from the * original. Also, there is now a builtin-test, just compile with: * gcc -I.. -DTEST_SNPRINTF -o snprintf snprintf.c -lm @@ -77,7 +77,7 @@ * Fix incorrect zpadlen handling in fmtfp. * Thanks to Ollie Oldham <ollie.oldham@metro-optix.com> for spotting it. * few mods to make it easier to compile the tests. - * addedd the "Ollie" test to the floating point ones. + * added the "Ollie" test to the floating point ones. * * Martin Pool (mbp@samba.org) April 2003 * Remove NO_CONFIG_H so that the test case can be built within a source diff --git a/lib/sysacls.c b/lib/sysacls.c index 21beed72..3eb903cf 100644 --- a/lib/sysacls.c +++ b/lib/sysacls.c @@ -2,7 +2,7 @@ * Unix SMB/CIFS implementation. * Based on the Samba ACL support code. * Copyright (C) Jeremy Allison 2000. - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2020 Wayne Davison * * The permission functions have been changed to get/set all bits via * one call. Some functions that rsync doesn't need were also removed. @@ -450,7 +450,7 @@ SMB_ACL_T sys_acl_get_file(const char *path_p, SMB_ACL_TYPE_T type) * * Note: we assume that the acl() system call returned a * well formed ACL which is sorted so that all of the - * access ACL entries preceed any default ACL entries + * access ACL entries precede any default ACL entries */ for (naccess = 0; naccess < count; naccess++) { if (acl_d->acl[naccess].a_type & ACL_DEFAULT) @@ -932,7 +932,7 @@ SMB_ACL_T sys_acl_get_file(const char *path_p, SMB_ACL_TYPE_T type) * * Note: we assume that the acl() system call returned a * well formed ACL which is sorted so that all of the - * access ACL entries preceed any default ACL entries + * access ACL entries precede any default ACL entries */ for (naccess = 0; naccess < count; naccess++) { if (acl_d->acl[naccess].a_type & ACL_DEFAULT) @@ -1095,7 +1095,7 @@ struct hpux_acl_types { * structures. * Inputs: * - * acl_count - Count of ACLs in the array of ACL strucutres. + * acl_count - Count of ACLs in the array of ACL structures. * aclp - Array of ACL structures. * acl_type_count - Pointer to acl_types structure. Should already be * allocated. @@ -1256,7 +1256,7 @@ static int hpux_acl_sort(int acl_count, int calclass, struct acl *aclp) { #if !defined(HAVE_HPUX_ACLSORT) /* - * The aclsort() system call is availabe on the latest HPUX General + * The aclsort() system call is available on the latest HPUX General * Patch Bundles. So for HPUX, we developed our version of acl_sort * function. Because, we don't want to update to a new * HPUX GR bundle just for aclsort() call. @@ -1311,7 +1311,7 @@ or DEF_USER_OBJ or DEF_GROUP_OBJ or DEF_OTHER_OBJ\n")); * Sorting crieteria - First sort by ACL type. If there are multiple entries of * same ACL type, sort by ACL id. * - * I am using the trival kind of sorting method here because, performance isn't + * I am using the trivial kind of sorting method here because, performance isn't * really effected by the ACLs feature. More over there aren't going to be more * than 17 entries on HPUX. */ diff --git a/lib/sysacls.h b/lib/sysacls.h index b551d7b0..040fcdab 100644 --- a/lib/sysacls.h +++ b/lib/sysacls.h @@ -3,7 +3,7 @@ * Version 2.2.x * Portable SMB ACL interface * Copyright (C) Jeremy Allison 2000 - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/lib/sysxattrs.c b/lib/sysxattrs.c index 1abd4685..583b93e9 100644 --- a/lib/sysxattrs.c +++ b/lib/sysxattrs.c @@ -2,7 +2,7 @@ * Extended attribute support for rsync. * * Copyright (C) 2004 Red Hat, Inc. - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2019 Wayne Davison * Written by Jay Fenlason. * * This program is free software; you can redistribute it and/or modify diff --git a/lib/sysxattrs.h b/lib/sysxattrs.h index 428421a0..024bbd18 100644 --- a/lib/sysxattrs.h +++ b/lib/sysxattrs.h @@ -1,9 +1,9 @@ #ifdef SUPPORT_XATTRS -#if defined HAVE_ATTR_XATTR_H -#include <attr/xattr.h> -#elif defined HAVE_SYS_XATTR_H +#if defined HAVE_SYS_XATTR_H #include <sys/xattr.h> +#elif defined HAVE_ATTR_XATTR_H +#include <attr/xattr.h> #elif defined HAVE_SYS_EXTATTR_H #include <sys/extattr.h> #endif @@ -11,13 +11,12 @@ * * You should have received a copy of the GNU General Public License along * with this program; if not, visit the http://fsf.org website. - */ - -/* This is based on loadparm.c from Samba, written by Andrew Tridgell + * + * This is based on loadparm.c from Samba, written by Andrew Tridgell * and Karl Auer. Some of the changes are: * * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison <wayned@samba.org> + * Copyright (C) 2003-2020 Wayne Davison */ /* Load parameters. @@ -31,7 +30,7 @@ * 1) add it to the global_vars or local_vars structure definition * 2) add it to the parm_table * 3) add it to the list of available functions (eg: using FN_GLOBAL_STRING()) - * 4) initialise it in the Defaults static stucture + * 4) initialise it in the Defaults static structure * * Notes: * The configuration file is processed sequentially for speed. For this @@ -43,6 +42,7 @@ #include "rsync.h" #include "itypes.h" +#include "default-dont-compress.h" extern item_list dparam_list; @@ -53,10 +53,6 @@ extern item_list dparam_list; #define LOG_DAEMON 0 #endif -#define DEFAULT_DONT_COMPRESS "*.gz *.zip *.z *.rpm *.deb *.iso *.bz2" \ - " *.t[gb]z *.7z *.mp[34] *.mov *.avi *.ogg *.jpg *.jpeg *.png" \ - " *.lzo *.rzip *.lzma *.rar *.ace *.gpg *.xz *.txz *.lz *.tlz" - /* the following are used by loadparm for option lists */ typedef enum { P_BOOL, P_BOOLREV, P_CHAR, P_INTEGER, @@ -100,19 +96,31 @@ typedef struct { char *pid_file; char *socket_options; + /* Each _EXP var tracks if the associated char* var has been expanded yet or not. */ + BOOL bind_address_EXP; + BOOL daemon_chroot_EXP; + BOOL daemon_gid_EXP; + BOOL daemon_uid_EXP; + BOOL motd_file_EXP; + BOOL pid_file_EXP; + BOOL socket_options_EXP; + int listen_backlog; int rsync_port; + + BOOL proxy_protocol; } global_vars; /* This structure describes a single section. Their order must match the * initializers below, which you can accomplish by keeping each sub-section * sorted. (e.g. in vim, just visually select each subsection and use !sort.) - * NOTE: the char* variables MUST all remain at the start of the stuct! */ + * NOTE: the char* variables MUST all remain at the start of the struct! */ typedef struct { char *auth_users; char *charset; char *comment; char *dont_compress; + char *early_exec; char *exclude; char *exclude_from; char *filter; @@ -135,8 +143,35 @@ typedef struct { char *syslog_tag; char *temp_dir; char *uid; -/* NOTE: update this macro if the last char* variable changes! */ -#define LOCAL_STRING_COUNT() (offsetof(local_vars, uid) / sizeof (char*) + 1) + + /* Each _EXP var tracks if the associated char* var has been expanded yet or not. */ + BOOL auth_users_EXP; + BOOL charset_EXP; + BOOL comment_EXP; + BOOL dont_compress_EXP; + BOOL early_exec_EXP; + BOOL exclude_EXP; + BOOL exclude_from_EXP; + BOOL filter_EXP; + BOOL gid_EXP; + BOOL hosts_allow_EXP; + BOOL hosts_deny_EXP; + BOOL include_EXP; + BOOL include_from_EXP; + BOOL incoming_chmod_EXP; + BOOL lock_file_EXP; + BOOL log_file_EXP; + BOOL log_format_EXP; + BOOL name_EXP; + BOOL outgoing_chmod_EXP; + BOOL path_EXP; + BOOL postxfer_exec_EXP; + BOOL prexfer_exec_EXP; + BOOL refuse_options_EXP; + BOOL secrets_file_EXP; + BOOL syslog_tag_EXP; + BOOL temp_dir_EXP; + BOOL uid_EXP; int max_connections; int max_verbosity; @@ -183,8 +218,18 @@ static const all_vars Defaults = { /* pid_file; */ NULL, /* socket_options; */ NULL, + /* bind_address_EXP; */ False, + /* daemon_chroot_EXP; */ False, + /* daemon_gid_EXP; */ False, + /* daemon_uid_EXP; */ False, + /* motd_file_EXP; */ False, + /* pid_file_EXP; */ False, + /* socket_options_EXP; */ False, + /* listen_backlog; */ 5, /* rsync_port; */ 0, + + /* proxy_protocol; */ False, }, /* ==== local_vars ==== */ @@ -193,7 +238,8 @@ static const all_vars Defaults = { /* charset; */ NULL, /* comment; */ NULL, /* dont_compress; */ DEFAULT_DONT_COMPRESS, - /* exclude; */ NULL, + /* early_exec; */ NULL, + /* exclude; */ NULL, /* exclude_from; */ NULL, /* filter; */ NULL, /* gid; */ NULL, @@ -216,6 +262,34 @@ static const all_vars Defaults = { /* temp_dir; */ NULL, /* uid; */ NULL, + /* auth_users_EXP; */ False, + /* charset_EXP; */ False, + /* comment_EXP; */ False, + /* dont_compress_EXP; */ False, + /* early_exec_EXP; */ False, + /* exclude_EXP; */ False, + /* exclude_from_EXP; */ False, + /* filter_EXP; */ False, + /* gid_EXP; */ False, + /* hosts_allow_EXP; */ False, + /* hosts_deny_EXP; */ False, + /* include_EXP; */ False, + /* include_from_EXP; */ False, + /* incoming_chmod_EXP; */ False, + /* lock_file_EXP; */ False, + /* log_file_EXP; */ False, + /* log_format_EXP; */ False, + /* name_EXP; */ False, + /* outgoing_chmod_EXP; */ False, + /* path_EXP; */ False, + /* postxfer_exec_EXP; */ False, + /* prexfer_exec_EXP; */ False, + /* refuse_options_EXP; */ False, + /* secrets_file_EXP; */ False, + /* syslog_tag_EXP; */ False, + /* temp_dir_EXP; */ False, + /* uid_EXP; */ False, + /* max_connections; */ 0, /* max_verbosity; */ 1, /* syslog_facility; */ LOG_DAEMON, @@ -328,12 +402,14 @@ static struct parm_struct parm_table[] = {"motd file", P_STRING, P_GLOBAL,&Vars.g.motd_file, NULL,0}, {"pid file", P_STRING, P_GLOBAL,&Vars.g.pid_file, NULL,0}, {"port", P_INTEGER,P_GLOBAL,&Vars.g.rsync_port, NULL,0}, + {"proxy protocol", P_BOOL, P_LOCAL, &Vars.g.proxy_protocol, NULL,0}, {"socket options", P_STRING, P_GLOBAL,&Vars.g.socket_options, NULL,0}, {"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0}, {"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0}, {"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0}, {"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0}, + {"early exec", P_STRING, P_LOCAL, &Vars.l.early_exec, NULL,0}, {"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0}, {"exclude", P_STRING, P_LOCAL, &Vars.l.exclude, NULL,0}, {"fake super", P_BOOL, P_LOCAL, &Vars.l.fake_super, NULL,0}, @@ -379,7 +455,7 @@ static struct parm_struct parm_table[] = }; /* Initialise the Default all_vars structure. */ -static void reset_all_vars(void) +void reset_daemon_vars(void) { memcpy(&Vars, &Defaults, sizeof Vars); } @@ -391,7 +467,7 @@ static char *expand_vars(char *str) char *buf, *t, *f; int bufsize; - if (strchr(str, '%') == NULL) + if (!str || !strchr(str, '%')) return str; bufsize = strlen(str) + 2048; @@ -434,20 +510,23 @@ static char *expand_vars(char *str) return buf; } +/* NOTE: use this function and all the FN_{GLOBAL,LOCAL} ones WITHOUT a trailing semicolon! */ +#define RETURN_EXPANDED(val) {if (!val ## _EXP) {val = expand_vars(val); val ## _EXP = True;} return val ? val : "";} + /* In this section all the functions that are used to access the * parameters from the rest of the program are defined. */ -#define FN_GLOBAL_STRING(fn_name, ptr) \ - char *fn_name(void) {return expand_vars(*(char **)(ptr) ? *(char **)(ptr) : "");} -#define FN_GLOBAL_BOOL(fn_name, ptr) \ - BOOL fn_name(void) {return *(BOOL *)(ptr);} -#define FN_GLOBAL_CHAR(fn_name, ptr) \ - char fn_name(void) {return *(char *)(ptr);} -#define FN_GLOBAL_INTEGER(fn_name, ptr) \ - int fn_name(void) {return *(int *)(ptr);} +#define FN_GLOBAL_STRING(fn_name, val) \ + char *fn_name(void) RETURN_EXPANDED(Vars.g.val) +#define FN_GLOBAL_BOOL(fn_name, val) \ + BOOL fn_name(void) {return Vars.g.val;} +#define FN_GLOBAL_CHAR(fn_name, val) \ + char fn_name(void) {return Vars.g.val;} +#define FN_GLOBAL_INTEGER(fn_name, val) \ + int fn_name(void) {return Vars.g.val;} #define FN_LOCAL_STRING(fn_name, val) \ - char *fn_name(int i) {return expand_vars(LP_SNUM_OK(i) && iSECTION(i).val ? iSECTION(i).val : Vars.l.val ? Vars.l.val : "");} + char *fn_name(int i) {if (LP_SNUM_OK(i) && iSECTION(i).val) RETURN_EXPANDED(iSECTION(i).val) else RETURN_EXPANDED(Vars.l.val)} #define FN_LOCAL_BOOL(fn_name, val) \ BOOL fn_name(int i) {return LP_SNUM_OK(i)? iSECTION(i).val : Vars.l.val;} #define FN_LOCAL_CHAR(fn_name, val) \ @@ -455,21 +534,24 @@ static char *expand_vars(char *str) #define FN_LOCAL_INTEGER(fn_name, val) \ int fn_name(int i) {return LP_SNUM_OK(i)? iSECTION(i).val : Vars.l.val;} -FN_GLOBAL_STRING(lp_bind_address, &Vars.g.bind_address) -FN_GLOBAL_STRING(lp_daemon_chroot, &Vars.g.daemon_chroot) -FN_GLOBAL_STRING(lp_daemon_gid, &Vars.g.daemon_gid) -FN_GLOBAL_STRING(lp_daemon_uid, &Vars.g.daemon_uid) -FN_GLOBAL_STRING(lp_motd_file, &Vars.g.motd_file) -FN_GLOBAL_STRING(lp_pid_file, &Vars.g.pid_file) -FN_GLOBAL_STRING(lp_socket_options, &Vars.g.socket_options) +FN_GLOBAL_STRING(lp_bind_address, bind_address) +FN_GLOBAL_STRING(lp_daemon_chroot, daemon_chroot) +FN_GLOBAL_STRING(lp_daemon_gid, daemon_gid) +FN_GLOBAL_STRING(lp_daemon_uid, daemon_uid) +FN_GLOBAL_STRING(lp_motd_file, motd_file) +FN_GLOBAL_STRING(lp_pid_file, pid_file) +FN_GLOBAL_STRING(lp_socket_options, socket_options) -FN_GLOBAL_INTEGER(lp_listen_backlog, &Vars.g.listen_backlog) -FN_GLOBAL_INTEGER(lp_rsync_port, &Vars.g.rsync_port) +FN_GLOBAL_INTEGER(lp_listen_backlog, listen_backlog) +FN_GLOBAL_INTEGER(lp_rsync_port, rsync_port) + +FN_GLOBAL_BOOL(lp_proxy_protocol, proxy_protocol) FN_LOCAL_STRING(lp_auth_users, auth_users) FN_LOCAL_STRING(lp_charset, charset) FN_LOCAL_STRING(lp_comment, comment) FN_LOCAL_STRING(lp_dont_compress, dont_compress) +FN_LOCAL_STRING(lp_early_exec, early_exec) FN_LOCAL_STRING(lp_exclude, exclude) FN_LOCAL_STRING(lp_exclude_from, exclude_from) FN_LOCAL_STRING(lp_filter, filter) @@ -525,19 +607,10 @@ static inline void string_set(char **s, const char *v) out_of_memory("string_set"); } -/* Copy the local_vars, strdup'ing any strings. NOTE: this depends on - * the structure starting with a contiguous list of the char* variables, - * and having an accurate count in the LOCAL_STRING_COUNT() macro. */ +/* Copy local_vars into a new section. No need to strdup since we don't free. */ static void copy_section(local_vars *psectionDest, local_vars *psectionSource) { - int count = LOCAL_STRING_COUNT(); - char **strings = (char**)psectionDest; - memcpy(psectionDest, psectionSource, sizeof psectionDest[0]); - while (count--) { - if (strings[count] && !(strings[count] = strdup(strings[count]))) - out_of_memory("copy_section"); - } } /* Initialise a section to the defaults. */ @@ -680,10 +753,10 @@ static BOOL do_parameter(char *parmname, char *parmvalue) switch (parm_table[parmnum].type) { case P_PATH: case P_STRING: - /* delay expansion of vars */ + /* delay expansion of %VAR% strings */ break; default: - /* expand any %VARS% now */ + /* expand any %VAR% strings now */ parmvalue = expand_vars(parmvalue); break; } @@ -808,7 +881,7 @@ int lp_load(char *pszFname, int globals_only) { bInGlobalSection = True; - reset_all_vars(); + reset_daemon_vars(); /* We get sections first, so have to start 'behind' to make up. */ iSectionIndex = -1; @@ -3,7 +3,7 @@ * * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org> * Copyright (C) 2000-2001 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -47,7 +47,6 @@ extern int64 total_data_written; extern int64 total_data_read; extern mode_t orig_umask; extern char *auth_user; -extern char *checksum_choice; extern char *stdout_format; extern char *logfile_format; extern char *logfile_name; @@ -76,8 +75,8 @@ static int64 initial_data_written; static int64 initial_data_read; struct { - int code; - char const *name; + int code; + char const *name; } const rerr_names[] = { { RERR_SYNTAX , "syntax or usage error" }, { RERR_PROTOCOL , "protocol incompatibility" }, @@ -223,25 +222,26 @@ void logfile_reopen(void) } } -static void filtered_fwrite(FILE *f, const char *buf, int len, int use_isprint) +static void filtered_fwrite(FILE *f, const char *in_buf, int in_len, int use_isprint, char end_char) { - const char *s, *end = buf + len; - for (s = buf; s < end; s++) { - if ((s < end - 4 - && *s == '\\' && s[1] == '#' - && isDigit(s + 2) - && isDigit(s + 3) - && isDigit(s + 4)) - || (*s != '\t' - && ((use_isprint && !isPrint(s)) - || *(uchar*)s < ' '))) { - if (s != buf && fwrite(buf, s - buf, 1, f) != 1) + char outbuf[1024], *ob = outbuf; + const char *end = in_buf + in_len; + while (in_buf < end) { + if (ob - outbuf >= (int)sizeof outbuf - 10) { + if (fwrite(outbuf, ob - outbuf, 1, f) != 1) exit_cleanup(RERR_MESSAGEIO); - fprintf(f, "\\#%03o", *(uchar*)s); - buf = s + 1; + ob = outbuf; } + if ((in_buf < end - 4 && *in_buf == '\\' && in_buf[1] == '#' + && isDigit(in_buf + 2) && isDigit(in_buf + 3) && isDigit(in_buf + 4)) + || (*in_buf != '\t' && ((use_isprint && !isPrint(in_buf)) || *(uchar*)in_buf < ' '))) + ob += snprintf(ob, 6, "\\#%03o", *(uchar*)in_buf++); + else + *ob++ = *in_buf++; } - if (buf != end && fwrite(buf, end - buf, 1, f) != 1) + if (end_char) /* The "- 10" above means that there is always room for one more char here. */ + *ob++ = end_char; + if (ob != outbuf && fwrite(outbuf, ob - outbuf, 1, f) != 1) exit_cleanup(RERR_MESSAGEIO); } @@ -250,7 +250,7 @@ static void filtered_fwrite(FILE *f, const char *buf, int len, int use_isprint) * can happen with certain fatal conditions. */ void rwrite(enum logcode code, const char *buf, int len, int is_utf8) { - int trailing_CR_or_NL; + char trailing_CR_or_NL; FILE *f = msgs2stderr ? stderr : stdout; #ifdef ICONV_OPTION iconv_t ic = is_utf8 && ic_recv != (iconv_t)-1 ? ic_recv : ic_chck; @@ -264,14 +264,13 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8) exit_cleanup(RERR_MESSAGEIO); if (msgs2stderr) { - if (!am_daemon) { - if (code == FLOG) - return; - goto output_msg; - } - if (code == FCLIENT) - return; - code = FLOG; + /* A normal daemon can get msgs2stderr set if the socket is busted, so we + * change the message destination into an FLOG message in order to try to + * get some info about an abnormal-exit into the log file. An rsh daemon + * can have this set via user request, so we'll leave the code alone so + * that the msg gets logged and then sent to stderr after that. */ + if (am_daemon > 0 && code != FCLIENT) + code = FLOG; } else if (send_msgs_to_gen) { assert(!is_utf8); /* Pass the message to our sibling in native charset. */ @@ -307,10 +306,28 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8) } else if (code == FLOG) return; - if (quiet && code == FINFO) - return; + switch (code) { + case FERROR_XFER: + got_xfer_error = 1; + /* FALL THROUGH */ + case FERROR: + case FWARNING: + f = stderr; + break; + case FINFO: + if (quiet) + return; + break; + /*case FLOG:*/ + /*case FCLIENT:*/ + /*case FERROR_UTF8:*/ + /*case FERROR_SOCKET:*/ + default: + fprintf(stderr, "Bad logcode in rwrite(): %d [%s]\n", (int)code, who_am_i()); + exit_cleanup(RERR_MESSAGEIO); + } - if (am_server) { + if (am_server && !msgs2stderr) { enum msgcode msg = (enum msgcode)code; if (protocol_version < 30) { if (msg == MSG_ERROR) @@ -321,33 +338,13 @@ void rwrite(enum logcode code, const char *buf, int len, int is_utf8) /* Pass the message to the non-server side. */ if (send_msg(msg, buf, len, !is_utf8)) return; - if (am_daemon) { + if (am_daemon > 0) { /* TODO: can we send the error to the user somehow? */ return; } f = stderr; } -output_msg: - switch (code) { - case FERROR_XFER: - got_xfer_error = 1; - /* FALL THROUGH */ - case FERROR: - case FERROR_UTF8: - case FERROR_SOCKET: - case FWARNING: - f = stderr; - break; - case FLOG: - case FINFO: - case FCLIENT: - break; - default: - fprintf(stderr, "Unknown logcode in rwrite(): %d [%s]\n", (int)code, who_am_i()); - exit_cleanup(RERR_MESSAGEIO); - } - if (output_needs_newline) { fputc('\n', f); output_needs_newline = 0; @@ -375,21 +372,28 @@ output_msg: iconvbufs(ic, &inbuf, &outbuf, inbuf.pos ? 0 : ICB_INIT); ierrno = errno; if (outbuf.len) { - filtered_fwrite(f, convbuf, outbuf.len, 0); + filtered_fwrite(f, convbuf, outbuf.len, 0, 0); outbuf.len = 0; } - if (!ierrno || ierrno == E2BIG) - continue; - fprintf(f, "\\#%03o", CVAL(inbuf.buf, inbuf.pos++)); - inbuf.len--; + /* Log one byte of illegal/incomplete sequence and continue with + * the next character. Check that the buffer is non-empty for the + * sake of robustness. */ + if ((ierrno == EILSEQ || ierrno == EINVAL) && inbuf.len) { + fprintf(f, "\\#%03o", CVAL(inbuf.buf, inbuf.pos++)); + inbuf.len--; + } + } + + if (trailing_CR_or_NL) { + fputc(trailing_CR_or_NL, f); + fflush(f); } } else #endif - filtered_fwrite(f, buf, len, !allow_8bit_chars); - - if (trailing_CR_or_NL) { - fputc(trailing_CR_or_NL, f); - fflush(f); + { + filtered_fwrite(f, buf, len, !allow_8bit_chars, trailing_CR_or_NL); + if (trailing_CR_or_NL) + fflush(f); } } @@ -448,8 +452,7 @@ void rsyserr(enum logcode code, int errcode, const char *format, ...) char buf[BIGPATHBUFLEN]; size_t len; - strlcpy(buf, RSYNC_NAME ": ", sizeof buf); - len = (sizeof RSYNC_NAME ": ") - 1; + len = snprintf(buf, sizeof buf, RSYNC_NAME ": [%s] ", who_am_i()); va_start(ap, format); len += vsnprintf(buf + len, sizeof buf - len, format, ap); @@ -672,9 +675,9 @@ static void log_formatted(enum logcode code, const char *format, const char *op, case 'C': n = NULL; if (S_ISREG(file->mode)) { - if (always_checksum && canonical_checksum(checksum_type)) + if (always_checksum) n = sum_as_hex(checksum_type, F_SUM(file), 1); - else if (iflags & ITEM_TRANSFER && canonical_checksum(xfersum_type)) + else if (iflags & ITEM_TRANSFER) n = sum_as_hex(xfersum_type, sender_file_sum, 0); } if (!n) { @@ -692,7 +695,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op, } n = c = buf2 + MAXPATHLEN - 32; c[0] = iflags & ITEM_LOCAL_CHANGE - ? iflags & ITEM_XNAME_FOLLOWS ? 'h' : 'c' + ? iflags & ITEM_XNAME_FOLLOWS ? 'h' : 'c' : !(iflags & ITEM_TRANSFER) ? '.' : !local_server && *op == 's' ? '<' : '>'; if (S_ISLNK(file->mode)) { @@ -713,7 +716,8 @@ static void log_formatted(enum logcode code, const char *format, const char *op, c[5] = !(iflags & ITEM_REPORT_PERMS) ? '.' : 'p'; c[6] = !(iflags & ITEM_REPORT_OWNER) ? '.' : 'o'; c[7] = !(iflags & ITEM_REPORT_GROUP) ? '.' : 'g'; - c[8] = !(iflags & ITEM_REPORT_ATIME) ? '.' : 'u'; + c[8] = !(iflags & ITEM_REPORT_ATIME) ? '.' + : S_ISLNK(file->mode) ? 'U' : 'u'; c[9] = !(iflags & ITEM_REPORT_ACL) ? '.' : 'a'; c[10] = !(iflags & ITEM_REPORT_XATTR) ? '.' : 'x'; c[11] = '\0'; @@ -810,8 +814,7 @@ void log_item(enum logcode code, struct file_struct *file, int iflags, const cha log_formatted(FLOG, logfile_format, s_or_r, file, NULL, iflags, hlink); } -void maybe_log_item(struct file_struct *file, int iflags, int itemizing, - const char *buf) +void maybe_log_item(struct file_struct *file, int iflags, int itemizing, const char *buf) { int significant_flags = iflags & SIGNIFICANT_ITEM_FLAGS; int see_item = itemizing && (significant_flags || *buf @@ -865,12 +868,15 @@ void log_delete(const char *fname, int mode) */ void log_exit(int code, const char *file, int line) { - if (code == 0) { + /* The receiving side's stats are split between 2 procs until the + * end of the run, so only the sender can output non-final info. */ + if (code == 0 || am_sender) { rprintf(FLOG,"sent %s bytes received %s bytes total size %s\n", big_num(stats.total_written), big_num(stats.total_read), big_num(stats.total_size)); - } else if (am_server != 2) { + } + if (code != 0 && am_server != 2) { const char *name; name = rerr_name(code); diff --git a/m4/header_major_fixed.m4 b/m4/header_major_fixed.m4 new file mode 100644 index 00000000..0f156aa0 --- /dev/null +++ b/m4/header_major_fixed.m4 @@ -0,0 +1,27 @@ +AC_DEFUN([AC_HEADER_MAJOR_FIXED], +[AC_CACHE_CHECK(whether sys/types.h defines makedev, + ac_cv_header_sys_types_h_makedev, +[AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include <sys/types.h>]], + [[return makedev(0, 0);]])], + [if grep sys/sysmacros.h conftest.err >/dev/null; then + ac_cv_header_sys_types_h_makedev=no + else + ac_cv_header_sys_types_h_makedev=yes + fi], + [ac_cv_header_sys_types_h_makedev=no]) +]) + +if test $ac_cv_header_sys_types_h_makedev = no; then +AC_CHECK_HEADER(sys/mkdev.h, + [AC_DEFINE(MAJOR_IN_MKDEV, 1, + [Define to 1 if `major', `minor', and `makedev' are + declared in <mkdev.h>.])]) + + if test $ac_cv_header_sys_mkdev_h = no; then + AC_CHECK_HEADER(sys/sysmacros.h, + [AC_DEFINE(MAJOR_IN_SYSMACROS, 1, + [Define to 1 if `major', `minor', and `makedev' + are declared in <sysmacros.h>.])]) + fi +fi +]) @@ -4,7 +4,7 @@ * Copyright (C) 1996-2001 Andrew Tridgell <tridge@samba.org> * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,6 +26,7 @@ #if defined CONFIG_LOCALE && defined HAVE_LOCALE_H #include <locale.h> #endif +#include <popt.h> extern int dry_run; extern int list_only; @@ -39,6 +40,7 @@ extern int blocking_io; extern int always_checksum; extern int remove_source_files; extern int output_needs_newline; +extern int called_from_signal_handler; extern int need_messages_from_generator; extern int kluge_around_eof; extern int got_xfer_error; @@ -75,20 +77,21 @@ extern pid_t cleanup_child_pid; extern size_t bwlimit_writemax; extern unsigned int module_dirlen; extern BOOL flist_receiving_enabled; +extern BOOL want_progress_now; extern BOOL shutting_down; extern int backup_dir_len; extern int basis_dir_cnt; +extern int default_af_hint; extern struct stats stats; extern char *stdout_format; extern char *logfile_format; extern char *filesfrom_host; extern char *partial_dir; -extern char *dest_option; extern char *rsync_path; extern char *shell_cmd; -extern char *batch_name; extern char *password_file; extern char *backup_dir; +extern char *copy_as; extern char curr_dir[MAXPATHLEN]; extern char backup_dir_buf[MAXPATHLEN]; extern char *basis_dir[MAX_BASIS_DIRS+1]; @@ -104,6 +107,8 @@ int daemon_over_rsh = 0; mode_t orig_umask = 0; int batch_gen_fd = -1; int sender_keeps_checksum = 0; +int raw_argc, cooked_argc; +char **raw_argv, **cooked_argv; /* There's probably never more than at most 2 outstanding child processes, * but set it higher, just in case. */ @@ -154,6 +159,27 @@ pid_t wait_process(pid_t pid, int *status_ptr, int flags) return waited_pid; } +int shell_exec(const char *cmd) +{ + char *shell = getenv("RSYNC_SHELL"); + int status; + pid_t pid; + + if (!shell) + return system(cmd); + + if ((pid = fork()) < 0) + return -1; + + if (pid == 0) { + execlp(shell, shell, "-c", cmd, NULL); + _exit(1); + } + + int ret = wait_process(pid, &status, 0); + return ret < 0 ? -1 : status; +} + /* Wait for a process to exit, calling io_flush while waiting. */ static void wait_process_with_flush(pid_t pid, int *exit_code_ptr) { @@ -210,6 +236,74 @@ void read_del_stats(int f) stats.deleted_files += stats.deleted_specials = read_varint(f); } +static void become_copy_as_user() +{ + char *gname; + uid_t uid; + gid_t gid; + + if (!copy_as) + return; + + if (DEBUG_GTE(CMD, 2)) + rprintf(FINFO, "[%s] copy_as=%s\n", who_am_i(), copy_as); + + if ((gname = strchr(copy_as, ':')) != NULL) + *gname++ = '\0'; + + if (!user_to_uid(copy_as, &uid, True)) { + rprintf(FERROR, "Invalid copy-as user: %s\n", copy_as); + exit_cleanup(RERR_SYNTAX); + } + + if (gname) { + if (!group_to_gid(gname, &gid, True)) { + rprintf(FERROR, "Invalid copy-as group: %s\n", gname); + exit_cleanup(RERR_SYNTAX); + } + } else { + struct passwd *pw; + if ((pw = getpwuid(uid)) == NULL) { + rsyserr(FERROR, errno, "getpwuid failed"); + exit_cleanup(RERR_SYNTAX); + } + gid = pw->pw_gid; + } + + if (setgid(gid) < 0) { + rsyserr(FERROR, errno, "setgid failed"); + exit_cleanup(RERR_SYNTAX); + } +#ifdef HAVE_SETGROUPS + if (setgroups(1, &gid)) { + rsyserr(FERROR, errno, "setgroups failed"); + exit_cleanup(RERR_SYNTAX); + } +#endif +#ifdef HAVE_INITGROUPS + if (!gname && initgroups(copy_as, gid) < 0) { + rsyserr(FERROR, errno, "initgroups failed"); + exit_cleanup(RERR_SYNTAX); + } +#endif + + if (setuid(uid) < 0 +#ifdef HAVE_SETEUID + || seteuid(uid) < 0 +#endif + ) { + rsyserr(FERROR, errno, "setuid failed"); + exit_cleanup(RERR_SYNTAX); + } + + our_uid = MY_UID(); + our_gid = MY_GID(); + am_root = (our_uid == 0); + + if (gname) + gname[-1] = ':'; +} + /* This function gets called from all 3 processes. We want the client side * to actually output the text, but the sender is the only process that has * all the stats we need. So, if we're a client sender, we do the report. @@ -432,8 +526,8 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in if (!*f) { if (in_quote) { rprintf(FERROR, - "Missing trailing-%c in remote-shell command.\n", - in_quote); + "Missing trailing-%c in remote-shell command.\n", + in_quote); exit_cleanup(RERR_SYNTAX); } f--; @@ -454,8 +548,13 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in *t++ = '\0'; } - /* check to see if we've already been given '-l user' in - * the remote-shell command */ + /* NOTE: must preserve t == start of command name until the end of the args handling! */ + if ((t = strrchr(cmd, '/')) != NULL) + t++; + else + t = cmd; + + /* Check to see if we've already been given '-l user' in the remote-shell command. */ for (i = 0; i < argc-1; i++) { if (!strcmp(args[i], "-l") && args[i+1][0] != '-') dash_l_set = 1; @@ -473,22 +572,23 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in args[argc++] = "-l"; args[argc++] = user; } +#ifdef AF_INET + if (default_af_hint == AF_INET && strcmp(t, "ssh") == 0) + args[argc++] = "-4"; /* we're using ssh so we can add a -4 option */ +#endif +#ifdef AF_INET6 + if (default_af_hint == AF_INET6 && strcmp(t, "ssh") == 0) + args[argc++] = "-6"; /* we're using ssh so we can add a -6 option */ +#endif args[argc++] = machine; #endif args[argc++] = rsync_path; - if (blocking_io < 0) { - char *cp; - if ((cp = strrchr(cmd, '/')) != NULL) - cp++; - else - cp = cmd; - if (strcmp(cp, "rsh") == 0 || strcmp(cp, "remsh") == 0) - blocking_io = 1; - } + if (blocking_io < 0 && (strcmp(t, "rsh") == 0 || strcmp(t, "remsh") == 0)) + blocking_io = 1; - server_options(args,&argc); + server_options(args, &argc); if (argc >= MAX_ARGS - 2) goto arg_overflow; @@ -592,7 +692,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path) /* Treat an empty string as a copy into the current directory. */ if (!*dest_path) - dest_path = "."; + dest_path = "."; if (daemon_filter_list.head) { char *slash = strrchr(dest_path, '/'); @@ -653,8 +753,7 @@ static char *get_local_name(struct file_list *flist, char *dest_path) *cp = '\0'; if (statret == 0) { - rprintf(FERROR, - "ERROR: destination path is not a directory\n"); + rprintf(FERROR, "ERROR: destination path is not a directory\n"); exit_cleanup(RERR_SYNTAX); } @@ -728,21 +827,21 @@ static void check_alt_basis_dirs(void) if (!new) out_of_memory("check_alt_basis_dirs"); if (slash && strncmp(bdir, "../", 3) == 0) { - /* We want to remove only one leading "../" prefix for - * the directory we couldn't create in dry-run mode: - * this ensures that any other ".." references get - * evaluated the same as they would for a live copy. */ - *slash = '\0'; - pathjoin(new, len, curr_dir, bdir + 3); - *slash = '/'; + /* We want to remove only one leading "../" prefix for + * the directory we couldn't create in dry-run mode: + * this ensures that any other ".." references get + * evaluated the same as they would for a live copy. */ + *slash = '\0'; + pathjoin(new, len, curr_dir, bdir + 3); + *slash = '/'; } else - pathjoin(new, len, curr_dir, bdir); + pathjoin(new, len, curr_dir, bdir); basis_dir[j] = bdir = new; } if (do_stat(bdir, &st) < 0) - rprintf(FWARNING, "%s arg does not exist: %s\n", dest_option, bdir); + rprintf(FWARNING, "%s arg does not exist: %s\n", alt_dest_opt(0), bdir); else if (!S_ISDIR(st.st_mode)) - rprintf(FWARNING, "%s arg is not a dir: %s\n", dest_option, bdir); + rprintf(FWARNING, "%s arg is not a dir: %s\n", alt_dest_opt(0), bdir); } } @@ -803,6 +902,8 @@ static void do_server_sender(int f_in, int f_out, int argc, char *argv[]) exit_cleanup(RERR_SYNTAX); } + become_copy_as_user(); + dir = argv[0]; if (!relative_paths) { if (!change_dir(dir, CD_NORMAL)) { @@ -1006,6 +1107,8 @@ static void do_server_recv(int f_in, int f_out, int argc, char *argv[]) return; } + become_copy_as_user(); + if (argc > 0) { char *dir = argv[0]; argc--; @@ -1072,8 +1175,7 @@ static void do_server_recv(int f_in, int f_out, int argc, char *argv[]) if (partial_dir && *partial_dir == '/' && check_filter(elp, FLOG, partial_dir + module_dirlen, 1) < 0) { options_rejected: - rprintf(FERROR, - "Your options have been rejected by the server.\n"); + rprintf(FERROR, "Your options have been rejected by the server.\n"); exit_cleanup(RERR_SYNTAX); } } @@ -1165,6 +1267,9 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) if (write_batch && !am_server) start_write_batch(f_out); + + become_copy_as_user(); + flist = send_file_list(f_out, argc, argv); if (DEBUG_GTE(FLIST, 3)) rprintf(FINFO,"file list sent\n"); @@ -1198,6 +1303,8 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) io_start_buffering_out(f_out); } + become_copy_as_user(); + send_filter_list(read_batch ? -1 : f_out); if (filesfrom_fd >= 0) { @@ -1258,7 +1365,7 @@ static int start_client(int argc, char *argv[]) { char *p, *shell_machine = NULL, *shell_user = NULL; char **remote_argv; - int remote_argc; + int remote_argc, env_port = rsync_port; int f_in, f_out; int ret; pid_t pid; @@ -1287,8 +1394,7 @@ static int start_client(int argc, char *argv[]) remote_argc--; /* don't count dest */ argc = 1; } - if (filesfrom_host && *filesfrom_host - && strcmp(filesfrom_host, shell_machine) != 0) { + if (filesfrom_host && *filesfrom_host && strcmp(filesfrom_host, shell_machine) != 0) { rprintf(FERROR, "--files-from hostname is not the same as the transfer hostname\n"); exit_cleanup(RERR_SYNTAX); @@ -1310,8 +1416,7 @@ static int start_client(int argc, char *argv[]) remote_argc = 1; path = check_for_hostspec(p, &shell_machine, &rsync_port); - if (path && filesfrom_host && *filesfrom_host - && strcmp(filesfrom_host, shell_machine) != 0) { + if (path && filesfrom_host && *filesfrom_host && strcmp(filesfrom_host, shell_machine) != 0) { rprintf(FERROR, "--files-from hostname is not the same as the transfer hostname\n"); exit_cleanup(RERR_SYNTAX); @@ -1324,6 +1429,7 @@ static int start_client(int argc, char *argv[]) exit_cleanup(RERR_SYNTAX); } shell_machine = NULL; + rsync_port = 0; } else { /* hostspec was found, so dest is remote */ argv[argc] = path; if (rsync_port) @@ -1338,6 +1444,7 @@ static int start_client(int argc, char *argv[]) } remote_argv = argv += argc - 1; remote_argc = argc = 1; + rsync_port = 0; } if (!rsync_port && remote_argc && !**remote_argv) /* Turn an empty arg into a dot dir. */ @@ -1384,6 +1491,11 @@ static int start_client(int argc, char *argv[]) } } + if (rsync_port < 0) + rsync_port = RSYNC_PORT; + else + env_port = rsync_port; + if (daemon_over_rsh < 0) return start_socket_client(shell_machine, remote_argc, remote_argv, argc, argv); @@ -1414,8 +1526,12 @@ static int start_client(int argc, char *argv[]) NS(remote_argv[0])); } - pid = do_cmd(shell_cmd, shell_machine, shell_user, remote_argv, remote_argc, - &f_in, &f_out); +#ifdef HAVE_PUTENV + if (daemon_over_rsh) + set_env_num("RSYNC_PORT", env_port); +#endif + + pid = do_cmd(shell_cmd, shell_machine, shell_user, remote_argv, remote_argc, &f_in, &f_out); /* if we're running an rsync server on the remote host over a * remote shell command, we need to do the RSYNCD protocol first */ @@ -1437,6 +1553,7 @@ static int start_client(int argc, char *argv[]) static void sigusr1_handler(UNUSED(int val)) { + called_from_signal_handler = 1; exit_cleanup(RERR_SIGNAL1); } @@ -1450,6 +1567,12 @@ static void sigusr2_handler(UNUSED(int val)) _exit(0); } +static void siginfo_handler(UNUSED(int val)) +{ + if (!am_server && !INFO_GTE(PROGRESS, 1)) + want_progress_now = True; +} + void remember_children(UNUSED(int val)) { #ifdef WNOHANG @@ -1497,9 +1620,7 @@ const char *get_panic_action(void) if (cmd_fmt) return cmd_fmt; - else - return "xterm -display :0 -T Panic -n Panic " - "-e gdb /proc/%d/exe %d"; + return "xterm -display :0 -T Panic -n Panic -e gdb /proc/%d/exe %d"; } @@ -1520,7 +1641,7 @@ static void rsync_panic_handler(UNUSED(int whatsig)) /* Unless we failed to execute gdb, we allow the process to * continue. I'm not sure if that's right. */ - ret = system(cmd_buf); + ret = shell_exec(cmd_buf); if (ret) _exit(ret); } @@ -1530,8 +1651,10 @@ static void rsync_panic_handler(UNUSED(int whatsig)) int main(int argc,char *argv[]) { int ret; - int orig_argc = argc; - char **orig_argv = argv; + + raw_argc = argc; + raw_argv = argv; + #ifdef HAVE_SIGACTION # ifdef HAVE_SIGPROCMASK sigset_t sigmask; @@ -1549,6 +1672,12 @@ int main(int argc,char *argv[]) SIGACTMASK(SIGABRT, rsync_panic_handler); SIGACTMASK(SIGBUS, rsync_panic_handler); #endif +#ifdef SIGINFO + SIGACTMASK(SIGINFO, siginfo_handler); +#endif +#ifdef SIGVTALRM + SIGACTMASK(SIGVTALRM, siginfo_handler); +#endif starttime = time(NULL); our_uid = MY_UID(); @@ -1557,6 +1686,10 @@ int main(int argc,char *argv[]) memset(&stats, 0, sizeof(stats)); + /* Even a non-daemon runs needs the default config values to be set, e.g. + * lp_dont_compress() is queried when no --skip-compress option is set. */ + reset_daemon_vars(); + if (argc < 2) { usage(FERROR); exit_cleanup(RERR_SYNTAX); @@ -1572,11 +1705,12 @@ int main(int argc,char *argv[]) #endif if (!parse_arguments(&argc, (const char ***) &argv)) { - /* FIXME: We ought to call the same error-handling - * code here, rather than relying on getopt. */ option_error(); exit_cleanup(RERR_SYNTAX); } + if (write_batch + && poptDupArgv(argc, (const char **)argv, &cooked_argc, (const char ***)&cooked_argv) != 0) + out_of_memory("main"); SIGACTMASK(SIGINT, sig_int); SIGACTMASK(SIGHUP, sig_int); @@ -1599,21 +1733,7 @@ int main(int argc,char *argv[]) change_dir(NULL, CD_NORMAL); if ((write_batch || read_batch) && !am_server) { - if (write_batch) - write_batch_shell_file(orig_argc, orig_argv, argc); - - if (read_batch && strcmp(batch_name, "-") == 0) - batch_fd = STDIN_FILENO; - else { - batch_fd = do_open(batch_name, - write_batch ? O_WRONLY | O_CREAT | O_TRUNC - : O_RDONLY, S_IRUSR | S_IWUSR); - } - if (batch_fd < 0) { - rsyserr(FERROR, errno, "Batch file %s open error", - full_fname(batch_name)); - exit_cleanup(RERR_FILEIO); - } + open_batch_files(); /* sets batch_fd */ if (read_batch) read_stream_flags(batch_fd); else @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -102,8 +102,7 @@ static OFF_T last_match; * If i >= 0, the number of a matched token. If < 0, indicates we have * only literal data. A -1 will send a 0-token-int too, and a -2 sends * only literal data, w/o any token-int. */ -static void matched(int f, struct sum_struct *s, struct map_struct *buf, - OFF_T offset, int32 i) +static void matched(int f, struct sum_struct *s, struct map_struct *buf, OFF_T offset, int32 i) { int32 n = (int32)(offset - last_match); /* max value: block_size (int32) */ int32 j; @@ -207,7 +206,7 @@ static void hash_search(int f,struct sum_struct *s, * either >= our offset or identical data at that offset. * Remove any bypassed entries that we can never use. */ if (updating_basis_file && s->sums[i].offset < offset - && !(s->sums[i].flags & SUMFLG_SAME_OFFSET)) { + && !(s->sums[i].flags & SUMFLG_SAME_OFFSET)) { *prev = s->sums[i].chain; continue; } @@ -288,10 +287,10 @@ static void hash_search(int f,struct sum_struct *s, /* we've found a match, but now check to see * if want_i can hint at a better match. */ if (i != want_i && want_i < s->count - && (!updating_basis_file || s->sums[want_i].offset >= offset - || s->sums[want_i].flags & SUMFLG_SAME_OFFSET) - && sum == s->sums[want_i].sum1 - && memcmp(sum2, s->sums[want_i].sum2, s->s2length) == 0) { + && (!updating_basis_file || s->sums[want_i].offset >= offset + || s->sums[want_i].flags & SUMFLG_SAME_OFFSET) + && sum == s->sums[want_i].sum1 + && memcmp(sum2, s->sums[want_i].sum2, s->s2length) == 0) { /* we've found an adjacent match - the RLL coder * will be happy */ i = want_i; @@ -317,8 +316,7 @@ static void hash_search(int f,struct sum_struct *s, /* Trim off the first byte from the checksum */ more = offset + k < len; - map = (schar *)map_ptr(buf, offset - backup, k + more + backup) - + backup; + map = (schar *)map_ptr(buf, offset - backup, k + more + backup) + backup; s1 -= map[0] + CHAR_OFFSET; s2 -= k * (map[0]+CHAR_OFFSET); diff --git a/maybe-make-man b/maybe-make-man new file mode 100755 index 00000000..b7f0a9f1 --- /dev/null +++ b/maybe-make-man @@ -0,0 +1,40 @@ +#!/bin/sh + +if [ x"$2" = x ]; then + echo "Usage: $0 SRC_DIR NAME.NUM.md" 1>&2 + exit 1 +fi + +srcdir="$1" +inname="$2" +flagfile="$srcdir/.md2man-works" + +if [ ! -d "$srcdir" ]; then + echo "The specified SRC_DIR is not a directory: $srcdir" 1>&2 + exit 1 +fi + +if [ ! -f "$flagfile" ]; then + # We test our smallest manpage just to see if the python setup works. + if "$srcdir/md2man" --test "$srcdir/rsync-ssl.1.md" >/dev/null 2>&1; then + touch $flagfile + else + outname=`echo "$inname" | sed 's/\.md$//'` + if [ -f "$outname" ]; then + exit 0 + elif [ -f "$srcdir/$outname" ]; then + echo "Copying $srcdir/$outname" + cp -p "$srcdir/$outname" . + exit 0 + else + echo "ERROR: $outname cannot be created." + if [ -f "$HOME/build_farm/build_test.fns" ]; then + exit 0 # No exit errorno to avoid a build failure in the samba build farm + else + exit 1 + fi + fi + fi +fi + +"$srcdir/md2man" "$srcdir/$inname" @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 + +# This script takes a manpage written in markdown and turns it into an html web +# page and a nroff man page. The input file must have the name of the program +# and the section in this format: NAME.NUM.md. The output files are written +# into the current directory named NAME.NUM.html and NAME.NUM. The input +# format has one extra extension: if a numbered list starts at 0, it is turned +# into a description list. The dl's dt tag is taken from the contents of the +# first tag inside the li, which is usually a p, code, or strong tag. The +# cmarkgfm or commonmark lib is used to transforms the input file into html. +# The html.parser is used as a state machine that both tweaks the html and +# outputs the nroff data based on the html tags. +# +# Copyright (C) 2020 Wayne Davison +# +# This program is freely redistributable. + +import sys, os, re, argparse, subprocess, time +from html.parser import HTMLParser + +CONSUMES_TXT = set('h1 h2 p li pre'.split()) + +HTML_START = """\ +<html><head> +<title>%s</title> +<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet"> +<style> +body { + max-width: 50em; + margin: auto; +} +body, b, strong, u { + font-family: 'Roboto', sans-serif; +} +code { + font-family: 'Roboto Mono', monospace; + font-weight: bold; + white-space: pre; +} +pre code { + display: block; + font-weight: normal; +} +blockquote pre code { + background: #f1f1f1; +} +dd p:first-of-type { + margin-block-start: 0em; +} +</style> +</head><body> +""" + +HTML_END = """\ +<div style="float: right"><p><i>%s</i></p></div> +</body></html> +""" + +MAN_START = r""" +.TH "%s" "%s" "%s" "%s" "User Commands" +""".lstrip() + +MAN_END = """\ +""" + +NORM_FONT = ('\1', r"\fP") +BOLD_FONT = ('\2', r"\fB") +UNDR_FONT = ('\3', r"\fI") +NBR_DASH = ('\4', r"\-") +NBR_SPACE = ('\xa0', r"\ ") + +md_parser = None + +def main(): + fi = re.match(r'^(?P<fn>(?P<srcdir>.+/)?(?P<name>(?P<prog>[^/]+)\.(?P<sect>\d+))\.md)$', args.mdfile) + if not fi: + die('Failed to parse NAME.NUM.md out of input file:', args.mdfile) + fi = argparse.Namespace(**fi.groupdict()) + + if not fi.srcdir: + fi.srcdir = './' + + fi.title = fi.prog + '(' + fi.sect + ') man page' + fi.mtime = 0 + + if os.path.lexists(fi.srcdir + '.git'): + fi.mtime = int(subprocess.check_output('git log -1 --format=%at'.split())) + + env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) } + + if args.test: + env_subs['VERSION'] = '1.0.0' + env_subs['libdir'] = '/usr' + else: + for fn in 'NEWS.md Makefile'.split(): + try: + st = os.lstat(fi.srcdir + fn) + except: + die('Failed to find', fi.srcdir + fn) + if not fi.mtime: + fi.mtime = st.st_mtime + + with open(fi.srcdir + 'Makefile', 'r', encoding='utf-8') as fh: + for line in fh: + m = re.match(r'^(\w+)=(.+)', line) + if not m: + continue + var, val = (m.group(1), m.group(2)) + if var == 'prefix' and env_subs[var] is not None: + continue + while re.search(r'\$\{', val): + val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val) + env_subs[var] = val + if var == 'VERSION': + break + + with open(fi.fn, 'r', encoding='utf-8') as fh: + txt = fh.read() + + txt = re.sub(r'@VERSION@', env_subs['VERSION'], txt) + txt = re.sub(r'@LIBDIR@', env_subs['libdir'], txt) + + fi.html_in = md_parser(txt) + txt = None + + fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime)) + fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog + ' ' + env_subs['VERSION']) + + HtmlToManPage(fi) + + if args.test: + print("The test was successful.") + return + + for fn, txt in ((fi.name + '.html', fi.html_out), (fi.name, fi.man_out)): + print("Wrote:", fn) + with open(fn, 'w', encoding='utf-8') as fh: + fh.write(txt) + + +def html_via_cmarkgfm(txt): + return cmarkgfm.markdown_to_html(txt) + + +def html_via_commonmark(txt): + return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt)) + + +class HtmlToManPage(HTMLParser): + def __init__(self, fi): + HTMLParser.__init__(self, convert_charrefs=True) + + st = self.state = argparse.Namespace( + list_state = [ ], + p_macro = ".P\n", + at_first_tag_in_li = False, + at_first_tag_in_dd = False, + dt_from = None, + in_pre = False, + in_code = False, + html_out = [ HTML_START % fi.title ], + man_out = [ MAN_START % fi.man_headings ], + txt = '', + ) + + self.feed(fi.html_in) + fi.html_in = None + + st.html_out.append(HTML_END % fi.date) + st.man_out.append(MAN_END) + + fi.html_out = ''.join(st.html_out) + st.html_out = None + + fi.man_out = ''.join(st.man_out) + st.man_out = None + + + def handle_starttag(self, tag, attrs_list): + st = self.state + if args.debug: + self.output_debug('START', (tag, attrs_list)) + if st.at_first_tag_in_li: + if st.list_state[-1] == 'dl': + st.dt_from = tag + if tag == 'p': + tag = 'dt' + else: + st.html_out.append('<dt>') + elif tag == 'p': + st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li. + st.at_first_tag_in_li = False + if tag == 'p': + if not st.at_first_tag_in_dd: + st.man_out.append(st.p_macro) + elif tag == 'li': + st.at_first_tag_in_li = True + lstate = st.list_state[-1] + if lstate == 'dl': + return + if lstate == 'o': + st.man_out.append(".IP o\n") + else: + st.man_out.append(".IP " + str(lstate) + ".\n") + st.list_state[-1] += 1 + elif tag == 'blockquote': + st.man_out.append(".RS 4\n") + elif tag == 'pre': + st.in_pre = True + st.man_out.append(st.p_macro + ".nf\n") + elif tag == 'code' and not st.in_pre: + st.in_code = True + st.txt += BOLD_FONT[0] + elif tag == 'strong' or tag == 'b': + st.txt += BOLD_FONT[0] + elif tag == 'em' or tag == 'i': + tag = 'u' # Change it into underline to be more like the man page + st.txt += UNDR_FONT[0] + elif tag == 'ol': + start = 1 + for var, val in attrs_list: + if var == 'start': + start = int(val) # We only support integers. + break + if st.list_state: + st.man_out.append(".RS\n") + if start == 0: + tag = 'dl' + attrs_list = [ ] + st.list_state.append('dl') + else: + st.list_state.append(start) + st.man_out.append(st.p_macro) + st.p_macro = ".IP\n" + elif tag == 'ul': + st.man_out.append(st.p_macro) + if st.list_state: + st.man_out.append(".RS\n") + st.p_macro = ".IP\n" + st.list_state.append('o') + st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>') + st.at_first_tag_in_dd = False + + + def handle_endtag(self, tag): + st = self.state + if args.debug: + self.output_debug('END', (tag,)) + if tag in CONSUMES_TXT or st.dt_from == tag: + txt = st.txt.strip() + st.txt = '' + else: + txt = None + add_to_txt = None + if tag == 'h1': + st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n') + elif tag == 'h2': + st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n') + elif tag == 'p': + if st.dt_from == 'p': + tag = 'dt' + st.man_out.append('.IP "' + manify(txt) + '"\n') + st.dt_from = None + elif txt != '': + st.man_out.append(manify(txt) + "\n") + elif tag == 'li': + if st.list_state[-1] == 'dl': + if st.at_first_tag_in_li: + die("Invalid 0. -> td translation") + tag = 'dd' + if txt != '': + st.man_out.append(manify(txt) + "\n") + st.at_first_tag_in_li = False + elif tag == 'blockquote': + st.man_out.append(".RE\n") + elif tag == 'pre': + st.in_pre = False + st.man_out.append(manify(txt) + "\n.fi\n") + elif (tag == 'code' and not st.in_pre): + st.in_code = False + add_to_txt = NORM_FONT[0] + elif tag == 'strong' or tag == 'b': + add_to_txt = NORM_FONT[0] + elif tag == 'em' or tag == 'i': + tag = 'u' # Change it into underline to be more like the man page + add_to_txt = NORM_FONT[0] + elif tag == 'ol' or tag == 'ul': + if st.list_state.pop() == 'dl': + tag = 'dl' + if st.list_state: + st.man_out.append(".RE\n") + else: + st.p_macro = ".P\n" + st.at_first_tag_in_dd = False + st.html_out.append('</' + tag + '>') + if add_to_txt: + if txt is None: + st.txt += add_to_txt + else: + txt += add_to_txt + if st.dt_from == tag: + st.man_out.append('.IP "' + manify(txt) + '"\n') + st.html_out.append('</dt><dd>') + st.at_first_tag_in_dd = True + st.dt_from = None + elif tag == 'dt': + st.html_out.append('<dd>') + st.at_first_tag_in_dd = True + + + def handle_data(self, txt): + st = self.state + if args.debug: + self.output_debug('DATA', (txt,)) + if st.in_pre: + html = htmlify(txt) + else: + txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2) + txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt) + html = htmlify(txt) + if st.in_code: + txt = re.sub(r'\s', NBR_SPACE[0], txt) + html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # <code> is non-breaking in CSS + st.html_out.append(html.replace(NBR_SPACE[0], ' ').replace(NBR_DASH[0], '-⁠')) + st.txt += txt + + + def output_debug(self, event, extra): + import pprint + st = self.state + if args.debug < 2: + st = argparse.Namespace(**vars(st)) + if len(st.html_out) > 2: + st.html_out = ['...'] + st.html_out[-2:] + if len(st.man_out) > 2: + st.man_out = ['...'] + st.man_out[-2:] + print(event, extra) + pprint.PrettyPrinter(indent=2).pprint(vars(st)) + + +def manify(txt): + return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\') + .replace(NBR_SPACE[0], NBR_SPACE[1]) + .replace(NBR_DASH[0], NBR_DASH[1]) + .replace(NORM_FONT[0], NORM_FONT[1]) + .replace(BOLD_FONT[0], BOLD_FONT[1]) + .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M) + + +def htmlify(txt): + return txt.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') + + +def warn(*msg): + print(*msg, file=sys.stderr) + + +def die(*msg): + warn(*msg) + sys.exit(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Transform a NAME.NUM.md markdown file into a NAME.NUM.html web page & a NAME.NUM man page.', add_help=False) + parser.add_argument('--test', action='store_true', help='Test if we can parse the input w/o updating any files.') + parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.') + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + parser.add_argument('mdfile', help="The NAME.NUM.md file to parse.") + args = parser.parse_args() + + try: + import cmarkgfm + md_parser = html_via_cmarkgfm + except: + try: + import commonmark + md_parser = html_via_commonmark + except: + die("Failed to find cmarkgfm or commonmark for python3.") + + main() diff --git a/mkproto.awk b/mkproto.awk new file mode 100644 index 00000000..ab97d54f --- /dev/null +++ b/mkproto.awk @@ -0,0 +1,39 @@ +#!/usr/bin/awk -f + +BEGIN { + while ((getline i < "proto.h") > 0) old_protos = old_protos ? old_protos "\n" i : i + protos = "/* This file is automatically generated with \"make proto\". DO NOT EDIT */\n" +} + +inheader { + protos = protos "\n" ((inheader = /\)[ \t]*$/ ? 0 : 1) ? $0 : $0 ";") + next +} + +/^FN_(LOCAL|GLOBAL)_[^(]+\([^,()]+/ { + local = /^FN_LOCAL/ + gsub(/^FN_(LOC|GLOB)AL_|,.*$/, "") + sub(/^BOOL\(/, "BOOL ") + sub(/^CHAR\(/, "char ") + sub(/^INTEGER\(/, "int ") + sub(/^STRING\(/, "char *") + protos = protos "\n" $0 (local ? "(int module_id);" : "(void);") + next +} + +/^static|^extern|;/||!/^[A-Za-z][A-Za-z0-9_]* / { next } + +/\(.*\)[ \t]*$/ { + protos = protos "\n" $0 ";" + next +} + +/\(/ { + inheader = 1 + protos = protos "\n" $0 +} + +END { + if (old_protos != protos) print protos > "proto.h" + printf "" > "proto.h-tstamp" +} diff --git a/mkproto.pl b/mkproto.pl deleted file mode 100644 index cdeb2ea3..00000000 --- a/mkproto.pl +++ /dev/null @@ -1,48 +0,0 @@ -# generate prototypes for rsync - -$old_protos = ''; -if (open(IN, 'proto.h')) { - $old_protos = join('', <IN>); - close IN; -} - -%FN_MAP = ( - BOOL => 'BOOL ', - CHAR => 'char ', - INTEGER => 'int ', - STRING => 'char *', -); - -$inheader = 0; -$protos = qq|/* This file is automatically generated with "make proto". DO NOT EDIT */\n\n|; - -while (<>) { - if ($inheader) { - if (/[)][ \t]*$/) { - $inheader = 0; - s/$/;/; - } - $protos .= $_; - } elsif (/^FN_(LOCAL|GLOBAL)_([^(]+)\(([^,()]+)/) { - $ret = $FN_MAP{$2}; - $func = $3; - $arg = $1 eq 'LOCAL' ? 'int module_id' : 'void'; - $protos .= "$ret$func($arg);\n"; - } elsif (/^static|^extern/ || /[;]/ || !/^[A-Za-z][A-Za-z0-9_]* /) { - ; - } elsif (/[(].*[)][ \t]*$/) { - s/$/;/; - $protos .= $_; - } elsif (/[(]/) { - $inheader = 1; - $protos .= $_; - } -} - -if ($old_protos ne $protos) { - open(OUT, '>proto.h') or die $!; - print OUT $protos; - close OUT; -} - -open(OUT, '>proto.h-tstamp') and close OUT; @@ -3,7 +3,7 @@ * * Copyright (C) 1998-2001 Andrew Tridgell <tridge@samba.org> * Copyright (C) 2000, 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2002-2018 Wayne Davison + * Copyright (C) 2002-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,19 +21,19 @@ #include "rsync.h" #include "itypes.h" +#include "latest-year.h" #include <popt.h> -#include <zlib.h> extern int module_id; extern int local_server; extern int sanitize_paths; extern int daemon_over_rsh; extern unsigned int module_dirlen; +extern struct name_num_obj valid_checksums; +extern struct name_num_obj valid_compressions; extern filter_rule_list filter_list; extern filter_rule_list daemon_filter_list; -#define NOT_SPECIFIED (-42) - int make_backups = 0; /** @@ -50,6 +50,7 @@ int append_mode = 0; int keep_dirlinks = 0; int copy_dirlinks = 0; int copy_links = 0; +int write_devices = 0; int preserve_links = 0; int preserve_hard_links = 0; int preserve_acls = 0; @@ -61,7 +62,9 @@ int preserve_specials = 0; int preserve_uid = 0; int preserve_gid = 0; int preserve_times = 0; +int preserve_atimes = 0; int update_only = 0; +int open_noatime = 0; int cvs_exclude = 0; int dry_run = 0; int do_xfers = 1; @@ -77,7 +80,7 @@ int protocol_version = PROTOCOL_VERSION; int sparse_files = 0; int preallocate_files = 0; int do_compression = 0; -int def_compress_level = NOT_SPECIFIED; +int do_compression_level = CLVL_NOT_SPECIFIED; int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */ int am_server = 0; int am_sender = 0; @@ -126,6 +129,7 @@ int inplace = 0; int delay_updates = 0; long block_size = 0; /* "long" because popt can't set an int32. */ char *skip_compress = NULL; +char *copy_as = NULL; item_list dparam_list = EMPTY_ITEM_LIST; /** Network address family. **/ @@ -173,16 +177,14 @@ char *sockopts = NULL; char *usermap = NULL; char *groupmap = NULL; int rsync_port = 0; -int compare_dest = 0; -int copy_dest = 0; -int link_dest = 0; +int alt_dest_type = 0; int basis_dir_cnt = 0; -char *dest_option = NULL; static int remote_option_alloc = 0; int remote_option_cnt = 0; const char **remote_options = NULL; const char *checksum_choice = NULL; +const char *compress_choice = NULL; int quiet = 0; int output_motd = 1; @@ -198,8 +200,11 @@ int list_only = 0; char *batch_name = NULL; int need_unsorted_flist = 0; +char *iconv_opt = #ifdef ICONV_OPTION -char *iconv_opt = ICONV_OPTION; + ICONV_OPTION; +#else + NULL; #endif struct chmod_mode_struct *chmod_modes = NULL; @@ -283,6 +288,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = { DEBUG_WORD(HLINK, W_SND|W_REC, "Debug hard-link actions (levels 1-3)"), DEBUG_WORD(ICONV, W_CLI|W_SRV, "Debug iconv character conversions (levels 1-2)"), DEBUG_WORD(IO, W_CLI|W_SRV, "Debug I/O routines (levels 1-4)"), + DEBUG_WORD(NSTR, W_CLI|W_SRV, "Debug negotiation strings"), DEBUG_WORD(OWN, W_REC, "Debug ownership changes in users & groups (levels 1-2)"), DEBUG_WORD(PROTO, W_CLI|W_SRV, "Debug protocol information"), DEBUG_WORD(RECV, W_REC, "Debug receiver functions"), @@ -305,9 +311,7 @@ static int refused_partial, refused_progress, refused_delete_before; static int refused_delete_during; static int refused_inplace, refused_no_iconv; static BOOL usermap_via_chown, groupmap_via_chown; -#ifdef HAVE_SETVBUF static char *outbuf_mode; -#endif static char *bwlimit_arg, *max_size_arg, *min_size_arg; static char tmp_partialdir[] = ".~tmp~"; @@ -406,8 +410,7 @@ static char *make_output_option(struct output_struct *words, short *levels, ucha return buf; } -static void parse_output_words(struct output_struct *words, short *levels, - const char *str, uchar priority) +static void parse_output_words(struct output_struct *words, short *levels, const char *str, uchar priority) { const char *s; int j, len, lev; @@ -445,7 +448,7 @@ static void parse_output_words(struct output_struct *words, short *levels, break; } } - if (len && !words[j].name) { + if (len && !words[j].name && !am_server) { rprintf(FERROR, "Unknown %s item: \"%.*s\"\n", words[j].help, len, str); exit_cleanup(RERR_SYNTAX); @@ -560,70 +563,127 @@ void negate_output_levels(void) debug_levels[j] *= -1; } -static void print_rsync_version(enum logcode f) +static char *istring(const char *fmt, int val) +{ + char *str; + if (asprintf(&str, fmt, val) < 0) + out_of_memory("istring"); + return str; +} + +static void print_capabilities(enum logcode f) { - char *subprotocol = ""; - char const *got_socketpair = "no "; - char const *have_inplace = "no "; - char const *hardlinks = "no "; - char const *prealloc = "no "; - char const *symtimes = "no "; - char const *acls = "no "; - char const *xattrs = "no "; - char const *links = "no "; - char const *iconv = "no "; - char const *ipv6 = "no "; STRUCT_STAT *dumstat; + char line_buf[75]; + int line_len, j; + char *capabilities[] = { + istring("%d-bit files", (int)(sizeof (OFF_T) * 8)), + istring("%d-bit inums", (int)(sizeof dumstat->st_ino * 8)), /* Don't check ino_t! */ + istring("%d-bit timestamps", (int)(sizeof (time_t) * 8)), + istring("%d-bit long ints", (int)(sizeof (int64) * 8)), + +#ifndef HAVE_SOCKETPAIR + "no " +#endif + "socketpairs", -#if SUBPROTOCOL_VERSION != 0 - if (asprintf(&subprotocol, ".PR%d", SUBPROTOCOL_VERSION) < 0) - out_of_memory("print_rsync_version"); +#ifndef SUPPORT_HARD_LINKS + "no " #endif -#ifdef HAVE_SOCKETPAIR - got_socketpair = ""; + "hardlinks", + +#ifndef SUPPORT_LINKS + "no " #endif -#ifdef HAVE_FTRUNCATE - have_inplace = ""; + "symlinks", + +#ifndef INET6 + "no " #endif -#ifdef SUPPORT_HARD_LINKS - hardlinks = ""; + "IPv6", + + "batchfiles", + +#ifndef HAVE_FTRUNCATE + "no " #endif -#ifdef SUPPORT_PREALLOCATION - prealloc = ""; + "inplace", + +#ifndef HAVE_FTRUNCATE + "no " #endif -#ifdef SUPPORT_ACLS - acls = ""; + "append", + +#ifndef SUPPORT_ACLS + "no " #endif -#ifdef SUPPORT_XATTRS - xattrs = ""; + "ACLs", + +#ifndef SUPPORT_XATTRS + "no " #endif -#ifdef SUPPORT_LINKS - links = ""; + "xattrs", + +#ifndef ICONV_OPTION + "no " #endif -#ifdef INET6 - ipv6 = ""; + "iconv", + +#ifndef CAN_SET_SYMLINK_TIMES + "no " #endif -#ifdef ICONV_OPTION - iconv = ""; + "symtimes", + +#ifndef SUPPORT_PREALLOCATION + "no " #endif -#ifdef CAN_SET_SYMLINK_TIMES - symtimes = ""; + "prealloc", + +#ifndef HAVE_SIMD + "no " #endif + "SIMD", + + NULL + }; + + for (line_len = 0, j = 0; ; j++) { + char *cap = capabilities[j]; + int cap_len = cap ? strlen(cap) : 1000; + int need_comma = cap && capabilities[j+1] != NULL ? 1 : 0; + if (line_len + 1 + cap_len + need_comma >= (int)sizeof line_buf) { + rprintf(f, " %s\n", line_buf); + line_len = 0; + } + if (!cap) + break; + line_len += snprintf(line_buf+line_len, sizeof line_buf - line_len, " %s%s", cap, need_comma ? "," : ""); + } +} +static void print_rsync_version(enum logcode f) +{ + char tmpbuf[256], *subprotocol = ""; + +#if SUBPROTOCOL_VERSION != 0 + subprotocol = istring(".PR%d", SUBPROTOCOL_VERSION); +#endif rprintf(f, "%s version %s protocol version %d%s\n", RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); - rprintf(f, "Copyright (C) 1996-2018 by Andrew Tridgell, Wayne Davison, and others.\n"); + + rprintf(f, "Copyright (C) 1996-" LATEST_YEAR " by Andrew Tridgell, Wayne Davison, and others.\n"); rprintf(f, "Web site: http://rsync.samba.org/\n"); + rprintf(f, "Capabilities:\n"); - rprintf(f, " %d-bit files, %d-bit inums, %d-bit timestamps, %d-bit long ints,\n", - (int)(sizeof (OFF_T) * 8), - (int)(sizeof dumstat->st_ino * 8), /* Don't check ino_t! */ - (int)(sizeof (time_t) * 8), - (int)(sizeof (int64) * 8)); - rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", - got_socketpair, hardlinks, links, ipv6, have_inplace); - rprintf(f, " %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc\n", - have_inplace, acls, xattrs, iconv, symtimes, prealloc); + print_capabilities(f); + + rprintf(f, "Checksum list:\n"); + get_default_nno_list(&valid_checksums, tmpbuf, sizeof tmpbuf, '('); + rprintf(f, " %s\n", tmpbuf); + + rprintf(f, "Compress list:\n"); + get_default_nno_list(&valid_compressions, tmpbuf, sizeof tmpbuf, '('); + rprintf(f, " %s\n", tmpbuf); #ifdef MAINTAINER_MODE rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); @@ -665,174 +725,33 @@ void usage(enum logcode F) rprintf(F,"to an rsync daemon, and require SRC or DEST to start with a module name.\n"); rprintf(F,"\n"); rprintf(F,"Options\n"); - rprintf(F," -v, --verbose increase verbosity\n"); - rprintf(F," --info=FLAGS fine-grained informational verbosity\n"); - rprintf(F," --debug=FLAGS fine-grained debug verbosity\n"); - rprintf(F," --msgs2stderr special output handling for debugging\n"); - rprintf(F," -q, --quiet suppress non-error messages\n"); - rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); - rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); - rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); - rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); - rprintf(F," -r, --recursive recurse into directories\n"); - rprintf(F," -R, --relative use relative path names\n"); - rprintf(F," --no-implied-dirs don't send implied dirs with --relative\n"); - rprintf(F," -b, --backup make backups (see --suffix & --backup-dir)\n"); - rprintf(F," --backup-dir=DIR make backups into hierarchy based in DIR\n"); - rprintf(F," --suffix=SUFFIX set backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX); - rprintf(F," -u, --update skip files that are newer on the receiver\n"); - rprintf(F," --inplace update destination files in-place (SEE MAN PAGE)\n"); - rprintf(F," --append append data onto shorter files\n"); - rprintf(F," --append-verify like --append, but with old data in file checksum\n"); - rprintf(F," -d, --dirs transfer directories without recursing\n"); - rprintf(F," -l, --links copy symlinks as symlinks\n"); - rprintf(F," -L, --copy-links transform symlink into referent file/dir\n"); - rprintf(F," --copy-unsafe-links only \"unsafe\" symlinks are transformed\n"); - rprintf(F," --safe-links ignore symlinks that point outside the source tree\n"); - rprintf(F," --munge-links munge symlinks to make them safer (but unusable)\n"); - rprintf(F," -k, --copy-dirlinks transform symlink to a dir into referent dir\n"); - rprintf(F," -K, --keep-dirlinks treat symlinked dir on receiver as dir\n"); - rprintf(F," -H, --hard-links preserve hard links\n"); - rprintf(F," -p, --perms preserve permissions\n"); - rprintf(F," -E, --executability preserve the file's executability\n"); - rprintf(F," --chmod=CHMOD affect file and/or directory permissions\n"); -#ifdef SUPPORT_ACLS - rprintf(F," -A, --acls preserve ACLs (implies --perms)\n"); -#endif -#ifdef SUPPORT_XATTRS - rprintf(F," -X, --xattrs preserve extended attributes\n"); -#endif - rprintf(F," -o, --owner preserve owner (super-user only)\n"); - rprintf(F," -g, --group preserve group\n"); - rprintf(F," --devices preserve device files (super-user only)\n"); - rprintf(F," --specials preserve special files\n"); - rprintf(F," -D same as --devices --specials\n"); - rprintf(F," -t, --times preserve modification times\n"); - rprintf(F," -O, --omit-dir-times omit directories from --times\n"); - rprintf(F," -J, --omit-link-times omit symlinks from --times\n"); - rprintf(F," --super receiver attempts super-user activities\n"); -#ifdef SUPPORT_XATTRS - rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); -#endif - rprintf(F," -S, --sparse turn sequences of nulls into sparse blocks\n"); -#ifdef SUPPORT_PREALLOCATION - rprintf(F," --preallocate allocate dest files before writing them\n"); -#else - rprintf(F," --preallocate pre-allocate dest files on remote receiver\n"); -#endif - rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); - rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); - rprintf(F," --checksum-choice=STR choose the checksum algorithms\n"); - rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n"); - rprintf(F," -B, --block-size=SIZE force a fixed checksum block-size\n"); - rprintf(F," -e, --rsh=COMMAND specify the remote shell to use\n"); - rprintf(F," --rsync-path=PROGRAM specify the rsync to run on the remote machine\n"); - rprintf(F," --existing skip creating new files on receiver\n"); - rprintf(F," --ignore-existing skip updating files that already exist on receiver\n"); - rprintf(F," --remove-source-files sender removes synchronized files (non-dirs)\n"); - rprintf(F," --del an alias for --delete-during\n"); - rprintf(F," --delete delete extraneous files from destination dirs\n"); - rprintf(F," --delete-before receiver deletes before transfer, not during\n"); - rprintf(F," --delete-during receiver deletes during the transfer\n"); - rprintf(F," --delete-delay find deletions during, delete after\n"); - rprintf(F," --delete-after receiver deletes after transfer, not during\n"); - rprintf(F," --delete-excluded also delete excluded files from destination dirs\n"); - rprintf(F," --ignore-missing-args ignore missing source args without error\n"); - rprintf(F," --delete-missing-args delete missing source args from destination\n"); - rprintf(F," --ignore-errors delete even if there are I/O errors\n"); - rprintf(F," --force force deletion of directories even if not empty\n"); - rprintf(F," --max-delete=NUM don't delete more than NUM files\n"); - rprintf(F," --max-size=SIZE don't transfer any file larger than SIZE\n"); - rprintf(F," --min-size=SIZE don't transfer any file smaller than SIZE\n"); - rprintf(F," --partial keep partially transferred files\n"); - rprintf(F," --partial-dir=DIR put a partially transferred file into DIR\n"); - rprintf(F," --delay-updates put all updated files into place at transfer's end\n"); - rprintf(F," -m, --prune-empty-dirs prune empty directory chains from the file-list\n"); - rprintf(F," --numeric-ids don't map uid/gid values by user/group name\n"); - rprintf(F," --usermap=STRING custom username mapping\n"); - rprintf(F," --groupmap=STRING custom groupname mapping\n"); - rprintf(F," --chown=USER:GROUP simple username/groupname mapping\n"); - rprintf(F," --timeout=SECONDS set I/O timeout in seconds\n"); - rprintf(F," --contimeout=SECONDS set daemon connection timeout in seconds\n"); - rprintf(F," -I, --ignore-times don't skip files that match in size and mod-time\n"); - rprintf(F," -M, --remote-option=OPTION send OPTION to the remote side only\n"); - rprintf(F," --size-only skip files that match in size\n"); - rprintf(F," -@, --modify-window=NUM set the accuracy for mod-time comparisons\n"); - rprintf(F," -T, --temp-dir=DIR create temporary files in directory DIR\n"); - rprintf(F," -y, --fuzzy find similar file for basis if no dest file\n"); - rprintf(F," --compare-dest=DIR also compare destination files relative to DIR\n"); - rprintf(F," --copy-dest=DIR ... and include copies of unchanged files\n"); - rprintf(F," --link-dest=DIR hardlink to files in DIR when unchanged\n"); - rprintf(F," -z, --compress compress file data during the transfer\n"); - rprintf(F," --compress-level=NUM explicitly set compression level\n"); - rprintf(F," --skip-compress=LIST skip compressing files with a suffix in LIST\n"); - rprintf(F," -C, --cvs-exclude auto-ignore files the same way CVS does\n"); - rprintf(F," -f, --filter=RULE add a file-filtering RULE\n"); - rprintf(F," -F same as --filter='dir-merge /.rsync-filter'\n"); - rprintf(F," repeated: --filter='- .rsync-filter'\n"); - rprintf(F," --exclude=PATTERN exclude files matching PATTERN\n"); - rprintf(F," --exclude-from=FILE read exclude patterns from FILE\n"); - rprintf(F," --include=PATTERN don't exclude files matching PATTERN\n"); - rprintf(F," --include-from=FILE read include patterns from FILE\n"); - rprintf(F," --files-from=FILE read list of source-file names from FILE\n"); - rprintf(F," -0, --from0 all *-from/filter files are delimited by 0s\n"); - rprintf(F," -s, --protect-args no space-splitting; only wildcard special-chars\n"); - rprintf(F," --address=ADDRESS bind address for outgoing socket to daemon\n"); - rprintf(F," --port=PORT specify double-colon alternate port number\n"); - rprintf(F," --sockopts=OPTIONS specify custom TCP options\n"); - rprintf(F," --blocking-io use blocking I/O for the remote shell\n"); - rprintf(F," --stats give some file-transfer stats\n"); - rprintf(F," -8, --8-bit-output leave high-bit chars unescaped in output\n"); - rprintf(F," -h, --human-readable output numbers in a human-readable format\n"); - rprintf(F," --progress show progress during transfer\n"); - rprintf(F," -P same as --partial --progress\n"); - rprintf(F," -i, --itemize-changes output a change-summary for all updates\n"); - rprintf(F," --out-format=FORMAT output updates using the specified FORMAT\n"); - rprintf(F," --log-file=FILE log what we're doing to the specified FILE\n"); - rprintf(F," --log-file-format=FMT log updates using the specified FMT\n"); - rprintf(F," --password-file=FILE read daemon-access password from FILE\n"); - rprintf(F," --list-only list the files instead of copying them\n"); - rprintf(F," --bwlimit=RATE limit socket I/O bandwidth\n"); -#ifdef HAVE_SETVBUF - rprintf(F," --outbuf=N|L|B set output buffering to None, Line, or Block\n"); -#endif - rprintf(F," --write-batch=FILE write a batched update to FILE\n"); - rprintf(F," --only-write-batch=FILE like --write-batch but w/o updating destination\n"); - rprintf(F," --read-batch=FILE read a batched update from FILE\n"); - rprintf(F," --protocol=NUM force an older protocol version to be used\n"); -#ifdef ICONV_OPTION - rprintf(F," --iconv=CONVERT_SPEC request charset conversion of filenames\n"); -#endif - rprintf(F," --checksum-seed=NUM set block/file checksum seed (advanced)\n"); - rprintf(F," -4, --ipv4 prefer IPv4\n"); - rprintf(F," -6, --ipv6 prefer IPv6\n"); - rprintf(F," --version print version number\n"); - rprintf(F,"(-h) --help show this help (-h is --help only if used alone)\n"); - +#include "help-rsync.h" rprintf(F,"\n"); rprintf(F,"Use \"rsync --daemon --help\" to see the daemon-mode command-line options.\n"); rprintf(F,"Please see the rsync(1) and rsyncd.conf(5) man pages for full documentation.\n"); rprintf(F,"See http://rsync.samba.org/ for updates, bug reports, and answers\n"); } -enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, +enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST, OPT_HELP, OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD, OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE, OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, - OPT_SERVER, OPT_REFUSED_BASE = 9000}; + OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, + OPT_REFUSED_BASE = 9000}; static struct poptOption long_options[] = { /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */ {"help", 0, POPT_ARG_NONE, 0, OPT_HELP, 0, 0 }, - {"version", 0, POPT_ARG_NONE, 0, OPT_VERSION, 0, 0}, + {"version", 'V', POPT_ARG_NONE, 0, 'V', 0, 0}, {"verbose", 'v', POPT_ARG_NONE, 0, 'v', 0, 0 }, {"no-verbose", 0, POPT_ARG_VAL, &verbose, 0, 0, 0 }, {"no-v", 0, POPT_ARG_VAL, &verbose, 0, 0, 0 }, {"info", 0, POPT_ARG_STRING, 0, OPT_INFO, 0, 0 }, {"debug", 0, POPT_ARG_STRING, 0, OPT_DEBUG, 0, 0 }, {"msgs2stderr", 0, POPT_ARG_NONE, &msgs2stderr, 0, 0, 0 }, + {"no-msgs2stderr", 0, POPT_ARG_VAL, &msgs2stderr, 0, 0, 0 }, {"quiet", 'q', POPT_ARG_NONE, 0, 'q', 0, 0 }, {"motd", 0, POPT_ARG_VAL, &output_motd, 1, 0, 0 }, {"no-motd", 0, POPT_ARG_VAL, &output_motd, 0, 0, 0 }, @@ -867,6 +786,11 @@ static struct poptOption long_options[] = { {"times", 't', POPT_ARG_VAL, &preserve_times, 1, 0, 0 }, {"no-times", 0, POPT_ARG_VAL, &preserve_times, 0, 0, 0 }, {"no-t", 0, POPT_ARG_VAL, &preserve_times, 0, 0, 0 }, + {"atimes", 'U', POPT_ARG_NONE, 0, 'U', 0, 0 }, + {"no-atimes", 0, POPT_ARG_VAL, &preserve_atimes, 0, 0, 0 }, + {"no-U", 0, POPT_ARG_VAL, &preserve_atimes, 0, 0, 0 }, + {"open-noatime", 0, POPT_ARG_VAL, &open_noatime, 1, 0, 0 }, + {"no-open-noatime", 0, POPT_ARG_VAL, &open_noatime, 0, 0, 0 }, {"omit-dir-times", 'O', POPT_ARG_VAL, &omit_dir_times, 1, 0, 0 }, {"no-omit-dir-times",0, POPT_ARG_VAL, &omit_dir_times, 0, 0, 0 }, {"no-O", 0, POPT_ARG_VAL, &omit_dir_times, 0, 0, 0 }, @@ -887,6 +811,8 @@ static struct poptOption long_options[] = { {"no-D", 0, POPT_ARG_NONE, 0, OPT_NO_D, 0, 0 }, {"devices", 0, POPT_ARG_VAL, &preserve_devices, 1, 0, 0 }, {"no-devices", 0, POPT_ARG_VAL, &preserve_devices, 0, 0, 0 }, + {"write-devices", 0, POPT_ARG_VAL, &write_devices, 1, 0, 0 }, + {"no-write-devices", 0, POPT_ARG_VAL, &write_devices, 0, 0, 0 }, {"specials", 0, POPT_ARG_VAL, &preserve_specials, 1, 0, 0 }, {"no-specials", 0, POPT_ARG_VAL, &preserve_specials, 0, 0, 0 }, {"links", 'l', POPT_ARG_VAL, &preserve_links, 1, 0, 0 }, @@ -955,11 +881,12 @@ static struct poptOption long_options[] = { {"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude, 0, 0, 0 }, {"whole-file", 'W', POPT_ARG_VAL, &whole_file, 1, 0, 0 }, {"no-whole-file", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 }, - {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 }, {"no-W", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 }, {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, + {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 }, + {"cc", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 }, {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, @@ -968,12 +895,14 @@ static struct poptOption long_options[] = { {"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, {"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, {"compress", 'z', POPT_ARG_NONE, 0, 'z', 0, 0 }, - {"old-compress", 0, POPT_ARG_VAL, &do_compression, 1, 0, 0 }, - {"new-compress", 0, POPT_ARG_VAL, &do_compression, 2, 0, 0 }, - {"no-compress", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 }, - {"no-z", 0, POPT_ARG_VAL, &do_compression, 0, 0, 0 }, + {"old-compress", 0, POPT_ARG_NONE, 0, OPT_OLD_COMPRESS, 0, 0 }, + {"new-compress", 0, POPT_ARG_NONE, 0, OPT_NEW_COMPRESS, 0, 0 }, + {"no-compress", 0, POPT_ARG_NONE, 0, OPT_NO_COMPRESS, 0, 0 }, + {"no-z", 0, POPT_ARG_NONE, 0, OPT_NO_COMPRESS, 0, 0 }, + {"compress-choice", 0, POPT_ARG_STRING, &compress_choice, 0, 0, 0 }, + {"zc", 0, POPT_ARG_STRING, &compress_choice, 0, 0, 0 }, {"skip-compress", 0, POPT_ARG_STRING, &skip_compress, 0, 0, 0 }, - {"compress-level", 0, POPT_ARG_INT, &def_compress_level, 0, 0, 0 }, + {"compress-level", 0, POPT_ARG_INT, &do_compression_level, 0, 0, 0 }, {0, 'P', POPT_ARG_NONE, 0, 'P', 0, 0 }, {"progress", 0, POPT_ARG_VAL, &do_progress, 1, 0, 0 }, {"no-progress", 0, POPT_ARG_VAL, &do_progress, 0, 0, 0 }, @@ -1020,25 +949,22 @@ static struct poptOption long_options[] = { {"rsh", 'e', POPT_ARG_STRING, &shell_cmd, 0, 0, 0 }, {"rsync-path", 0, POPT_ARG_STRING, &rsync_path, 0, 0, 0 }, {"temp-dir", 'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 }, -#ifdef ICONV_OPTION {"iconv", 0, POPT_ARG_STRING, &iconv_opt, 0, 0, 0 }, {"no-iconv", 0, POPT_ARG_NONE, 0, OPT_NO_ICONV, 0, 0 }, -#endif {"ipv4", '4', POPT_ARG_VAL, &default_af_hint, AF_INET, 0, 0 }, {"ipv6", '6', POPT_ARG_VAL, &default_af_hint, AF_INET6, 0, 0 }, {"8-bit-output", '8', POPT_ARG_VAL, &allow_8bit_chars, 1, 0, 0 }, {"no-8-bit-output", 0, POPT_ARG_VAL, &allow_8bit_chars, 0, 0, 0 }, {"no-8", 0, POPT_ARG_VAL, &allow_8bit_chars, 0, 0, 0 }, {"qsort", 0, POPT_ARG_NONE, &use_qsort, 0, 0, 0 }, + {"copy-as", 0, POPT_ARG_STRING, ©_as, 0, 0, 0 }, {"address", 0, POPT_ARG_STRING, &bind_address, 0, 0, 0 }, {"port", 0, POPT_ARG_INT, &rsync_port, 0, 0, 0 }, {"sockopts", 0, POPT_ARG_STRING, &sockopts, 0, 0, 0 }, {"password-file", 0, POPT_ARG_STRING, &password_file, 0, 0, 0 }, {"blocking-io", 0, POPT_ARG_VAL, &blocking_io, 1, 0, 0 }, {"no-blocking-io", 0, POPT_ARG_VAL, &blocking_io, 0, 0, 0 }, -#ifdef HAVE_SETVBUF {"outbuf", 0, POPT_ARG_STRING, &outbuf_mode, 0, 0, 0 }, -#endif {"remote-option", 'M', POPT_ARG_STRING, 0, 'M', 0, 0 }, {"protocol", 0, POPT_ARG_INT, &protocol_version, 0, 0, 0 }, {"checksum-seed", 0, POPT_ARG_INT, &checksum_seed, 0, 0, 0 }, @@ -1059,20 +985,7 @@ static void daemon_usage(enum logcode F) rprintf(F,"\n"); rprintf(F,"Usage: rsync --daemon [OPTION]...\n"); - rprintf(F," --address=ADDRESS bind to the specified address\n"); - rprintf(F," --bwlimit=RATE limit socket I/O bandwidth\n"); - rprintf(F," --config=FILE specify alternate rsyncd.conf file\n"); - rprintf(F," -M, --dparam=OVERRIDE override global daemon config parameter\n"); - rprintf(F," --no-detach do not detach from the parent\n"); - rprintf(F," --port=PORT listen on alternate port number\n"); - rprintf(F," --log-file=FILE override the \"log file\" setting\n"); - rprintf(F," --log-file-format=FMT override the \"log format\" setting\n"); - rprintf(F," --sockopts=OPTIONS specify custom TCP options\n"); - rprintf(F," -v, --verbose increase verbosity\n"); - rprintf(F," -4, --ipv4 prefer IPv4\n"); - rprintf(F," -6, --ipv6 prefer IPv6\n"); - rprintf(F," --help show this help screen\n"); - +#include "help-rsyncd.h" rprintf(F,"\n"); rprintf(F,"If you were not trying to invoke rsync as a daemon, avoid using any of the\n"); rprintf(F,"daemon-specific rsync options. See also the rsyncd.conf(5) man page.\n"); @@ -1121,79 +1034,160 @@ void option_error(void) } rprintf(FERROR, RSYNC_NAME ": %s", err_buf); + io_flush(MSG_FLUSH); msleep(20); } +static void parse_one_refuse_match(int negated, const char *ref, const struct poptOption *list_end) +{ + struct poptOption *op; + char shortName[2]; + int is_wild = strpbrk(ref, "*?[") != NULL; + int found_match = 0; + + shortName[1] = '\0'; + + if (strcmp("a", ref) == 0 || strcmp("archive", ref) == 0) { + ref = "[ardlptgoD]"; + is_wild = 1; + } + + for (op = long_options; op != list_end; op++) { + *shortName = op->shortName; + if ((op->longName && wildmatch(ref, op->longName)) + || (*shortName && wildmatch(ref, shortName))) { + if (op->descrip[1] == '*') + op->descrip = negated ? "a*" : "r*"; + else if (!is_wild) + op->descrip = negated ? "a=" : "r="; + found_match = 1; + if (!is_wild) + break; + } + } + + if (!found_match) + rprintf(FLOG, "No match for refuse-options string \"%s\"\n", ref); +} + + /** * Tweak the option table to disable all options that the rsyncd.conf * file has told us to refuse. **/ -static void set_refuse_options(char *bp) +static void set_refuse_options(void) { - struct poptOption *op; - char *cp, shortname[2]; - int is_wild, found_match; + struct poptOption *op, *list_end = NULL; + char *cp, *ref = lp_refuse_options(module_id); + int negated; + + if (!ref) + ref = ""; + + if (!am_daemon) + ref = ""; + + /* We abuse the descrip field in poptOption to make it easy to flag which options + * are refused (since we don't use it otherwise). Start by marking all options + * as "a"ccepted with a few options also marked as non-wild. */ + for (op = long_options; ; op++) { + const char *longName = op->longName ? op->longName : ""; + if (!op->longName && !op->shortName) { + list_end = op; + break; + } + if (!am_daemon + || op->shortName == 'e' /* Required for compatibility flags */ + || op->shortName == '0' /* --from0 just modifies --files-from, so refuse that instead (or not) */ + || op->shortName == 's' /* --protect-args is always OK */ + || op->shortName == 'n' /* --dry-run is always OK */ + || strcmp("iconv", longName) == 0 + || strcmp("no-iconv", longName) == 0 + || strcmp("checksum-seed", longName) == 0 + || strcmp("write-devices", longName) == 0 /* disable wild-match (it gets refused below) */ + || strcmp("log-format", longName) == 0 /* aka out-format (NOT log-file-format) */ + || strcmp("sender", longName) == 0 + || strcmp("server", longName) == 0) + op->descrip = "a="; /* exact-match only */ + else + op->descrip = "a*"; /* wild-card-able */ + } + assert(list_end != NULL); - shortname[1] = '\0'; + if (am_daemon) { /* Refused by default, but can be accepted via a negated exact match. */ + parse_one_refuse_match(0, "write-devices", list_end); + } while (1) { - while (*bp == ' ') bp++; - if (!*bp) + while (*ref == ' ') ref++; + if (!*ref) break; - if ((cp = strchr(bp, ' ')) != NULL) - *cp= '\0'; - is_wild = strpbrk(bp, "*?[") != NULL; - found_match = 0; - for (op = long_options; ; op++) { - *shortname = op->shortName; - if (!op->longName && !*shortname) - break; - if ((op->longName && wildmatch(bp, op->longName)) - || (*shortname && wildmatch(bp, shortname))) { - if (op->argInfo == POPT_ARG_VAL) - op->argInfo = POPT_ARG_NONE; - op->val = (op - long_options) + OPT_REFUSED_BASE; - found_match = 1; - /* These flags are set to let us easily check - * an implied option later in the code. */ - switch (*shortname) { - case 'r': case 'd': case 'l': case 'p': - case 't': case 'g': case 'o': case 'D': - refused_archive_part = op->val; - break; - case 'z': - refused_compress = op->val; - break; - case '\0': - if (wildmatch("delete", op->longName)) - refused_delete = op->val; - else if (wildmatch("delete-before", op->longName)) - refused_delete_before = op->val; - else if (wildmatch("delete-during", op->longName)) - refused_delete_during = op->val; - else if (wildmatch("partial", op->longName)) - refused_partial = op->val; - else if (wildmatch("progress", op->longName)) - refused_progress = op->val; - else if (wildmatch("inplace", op->longName)) - refused_inplace = op->val; - else if (wildmatch("no-iconv", op->longName)) - refused_no_iconv = op->val; - break; - } - if (!is_wild) - break; - } - } - if (!found_match) { - rprintf(FLOG, "No match for refuse-options string \"%s\"\n", - bp); - } + if ((cp = strchr(ref, ' ')) != NULL) + *cp = '\0'; + negated = *ref == '!'; + if (negated && ref[1]) + ref++; + parse_one_refuse_match(negated, ref, list_end); if (!cp) break; *cp = ' '; - bp = cp + 1; + ref = cp + 1; + } + + if (am_daemon) { +#ifdef ICONV_OPTION + if (!*lp_charset(module_id)) + parse_one_refuse_match(0, "iconv", list_end); +#endif + parse_one_refuse_match(0, "log-file*", list_end); + } + +#ifndef SUPPORT_HARD_LINKS + parse_one_refuse_match(0, "link-dest", list_end); +#endif +#ifndef ICONV_OPTION + parse_one_refuse_match(0, "iconv", list_end); +#endif +#ifndef HAVE_SETVBUF + parse_one_refuse_match(0, "outbuf", list_end); +#endif + + /* Now we use the descrip values to actually mark the options for refusal. */ + for (op = long_options; op != list_end; op++) { + int refused = op->descrip[0] == 'r'; + op->descrip = NULL; + if (!refused) + continue; + if (op->argInfo == POPT_ARG_VAL) + op->argInfo = POPT_ARG_NONE; + op->val = (op - long_options) + OPT_REFUSED_BASE; + /* The following flags are set to let us easily check an implied option later in the code. */ + switch (op->shortName) { + case 'r': case 'd': case 'l': case 'p': + case 't': case 'g': case 'o': case 'D': + refused_archive_part = op->val; + break; + case 'z': + refused_compress = op->val; + break; + case '\0': + if (strcmp("delete", op->longName) == 0) + refused_delete = op->val; + else if (strcmp("delete-before", op->longName) == 0) + refused_delete_before = op->val; + else if (strcmp("delete-during", op->longName) == 0) + refused_delete_during = op->val; + else if (strcmp("partial", op->longName) == 0) + refused_partial = op->val; + else if (strcmp("progress", op->longName) == 0) + refused_progress = op->val; + else if (strcmp("inplace", op->longName) == 0) + refused_inplace = op->val; + else if (strcmp("no-iconv", op->longName) == 0) + refused_no_iconv = op->val; + break; + } } } @@ -1270,15 +1264,19 @@ static OFF_T parse_size_arg(char **size_arg, char def_suf) static void create_refuse_error(int which) { + const char *msg; + if (am_daemon) + msg = "The server is configured to refuse"; + else if (am_server) + msg = "The server does not support"; + else + msg = "This rsync does not support"; + /* The "which" value is the index + OPT_REFUSED_BASE. */ struct poptOption *op = &long_options[which - OPT_REFUSED_BASE]; - int n = snprintf(err_buf, sizeof err_buf, - "The server is configured to refuse --%s\n", - op->longName) - 1; - if (op->shortName) { - snprintf(err_buf + n, sizeof err_buf - n, - " (-%c)\n", op->shortName); - } + int n = snprintf(err_buf, sizeof err_buf, "%s --%s\n", msg, op->longName) - 1; + if (op->shortName) + snprintf(err_buf + n, sizeof err_buf - n, " (-%c)\n", op->shortName); } /* This is used to make sure that --daemon & --server cannot be aliased to @@ -1289,15 +1287,33 @@ static void popt_unalias(poptContext con, const char *opt) { struct poptAlias unalias; + memset(&unalias, 0, sizeof unalias); + unalias.longName = opt + 2; /* point past the leading "--" */ - unalias.shortName = '\0'; unalias.argc = 1; - unalias.argv = new_array(const char*, 1); + unalias.argv = new_array0(const char*, 2); unalias.argv[0] = strdup(opt); poptAddAlias(con, unalias, 0); } +char *alt_dest_opt(int type) +{ + if (!type) + type = alt_dest_type; + + switch (type) { + case COMPARE_DEST: + return "--compare-dest"; + case COPY_DEST: + return "--copy-dest"; + case LINK_DEST: + return "--link-dest"; + default: + assert(0); + } +} + /** * Process command line arguments. Called on both local and remote. * @@ -1309,22 +1325,18 @@ static void popt_unalias(poptContext con, const char *opt) int parse_arguments(int *argc_p, const char ***argv_p) { static poptContext pc; - char *ref = lp_refuse_options(module_id); const char *arg, **argv = *argv_p; int argc = *argc_p; - int opt; + int opt, want_dest_type; int orig_protect_args = protect_args; - if (ref && *ref) - set_refuse_options(ref); - if (am_daemon) { - set_refuse_options("log-file*"); -#ifdef ICONV_OPTION - if (!*lp_charset(module_id)) - set_refuse_options("iconv"); -#endif + if (argc == 0) { + strlcpy(err_buf, "argc is zero!\n", sizeof err_buf); + return 0; } + set_refuse_options(); + #ifdef ICONV_OPTION if (!am_daemon && protect_args <= 0 && (arg = getenv("RSYNC_ICONV")) != NULL && *arg) iconv_opt = strdup(arg); @@ -1348,7 +1360,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) * only special cases are returned and listed here. */ switch (opt) { - case OPT_VERSION: + case 'V': print_rsync_version(FINFO); exit_cleanup(0); @@ -1357,8 +1369,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) /* Disable popt aliases on the server side and * then start parsing the options again. */ poptFreeContext(pc); - pc = poptGetContext(RSYNC_NAME, argc, argv, - long_options, 0); + pc = poptGetContext(RSYNC_NAME, argc, argv, long_options, 0); am_server = 1; } #ifdef ICONV_OPTION @@ -1386,8 +1397,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) #endif protect_args = 0; poptFreeContext(pc); - pc = poptGetContext(RSYNC_NAME, argc, argv, - long_daemon_options, 0); + pc = poptGetContext(RSYNC_NAME, argc, argv, long_daemon_options, 0); while ((opt = poptGetNextOpt(pc)) != -1) { char **cpp; switch (opt) { @@ -1399,8 +1409,8 @@ int parse_arguments(int *argc_p, const char ***argv_p) arg = poptGetOptArg(pc); if (!strchr(arg, '=')) { rprintf(FERROR, - "--dparam value is missing an '=': %s\n", - arg); + "--dparam value is missing an '=': %s\n", + arg); goto daemon_error; } cpp = EXPAND_ITEM_LIST(&dparam_list, char *, 4); @@ -1413,9 +1423,9 @@ int parse_arguments(int *argc_p, const char ***argv_p) default: rprintf(FERROR, - "rsync: %s: %s (in daemon mode)\n", - poptBadOption(pc, POPT_BADOPTION_NOALIAS), - poptStrerror(opt)); + "rsync: %s: %s (in daemon mode)\n", + poptBadOption(pc, POPT_BADOPTION_NOALIAS), + poptStrerror(opt)); goto daemon_error; } } @@ -1432,8 +1442,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (!daemon_opt) { rprintf(FERROR, "Daemon option(s) used without --daemon.\n"); daemon_error: - rprintf(FERROR, - "(Type \"rsync --daemon --help\" for assistance with daemon mode.)\n"); + rprintf(FERROR, "(Type \"rsync --daemon --help\" for assistance with daemon mode.)\n"); exit_cleanup(RERR_SYNTAX); } @@ -1530,6 +1539,11 @@ int parse_arguments(int *argc_p, const char ***argv_p) itemize_changes++; break; + case 'U': + if (++preserve_atimes > 1) + open_noatime = 1; + break; + case 'v': verbose++; break; @@ -1559,8 +1573,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) case 'P': if (refused_partial || refused_progress) { - create_refuse_error(refused_partial - ? refused_partial : refused_progress); + create_refuse_error(refused_partial ? refused_partial : refused_progress); return 0; } do_progress = 1; @@ -1571,6 +1584,19 @@ int parse_arguments(int *argc_p, const char ***argv_p) do_compression++; break; + case OPT_OLD_COMPRESS: + compress_choice = "zlib"; + break; + + case OPT_NEW_COMPRESS: + compress_choice = "zlibx"; + break; + + case OPT_NO_COMPRESS: + do_compression = 0; + compress_choice = NULL; + break; + case 'M': arg = poptGetOptArg(pc); if (*arg != '-') { @@ -1655,30 +1681,29 @@ int parse_arguments(int *argc_p, const char ***argv_p) break; case OPT_LINK_DEST: -#ifdef SUPPORT_HARD_LINKS - link_dest = 1; - dest_option = "--link-dest"; + want_dest_type = LINK_DEST; goto set_dest_dir; -#else - snprintf(err_buf, sizeof err_buf, - "hard links are not supported on this %s\n", - am_server ? "server" : "client"); - return 0; -#endif case OPT_COPY_DEST: - copy_dest = 1; - dest_option = "--copy-dest"; + want_dest_type = COPY_DEST; goto set_dest_dir; case OPT_COMPARE_DEST: - compare_dest = 1; - dest_option = "--compare-dest"; + want_dest_type = COMPARE_DEST; + set_dest_dir: + if (alt_dest_type && alt_dest_type != want_dest_type) { + snprintf(err_buf, sizeof err_buf, + "ERROR: the %s option conflicts with the %s option\n", + alt_dest_opt(want_dest_type), alt_dest_opt(0)); + return 0; + } + alt_dest_type = want_dest_type; + if (basis_dir_cnt >= MAX_BASIS_DIRS) { snprintf(err_buf, sizeof err_buf, "ERROR: at most %d %s args may be specified\n", - MAX_BASIS_DIRS, dest_option); + MAX_BASIS_DIRS, alt_dest_opt(0)); return 0; } /* We defer sanitizing this arg until we know what @@ -1690,8 +1715,8 @@ int parse_arguments(int *argc_p, const char ***argv_p) arg = poptGetOptArg(pc); if (!parse_chmod(arg, &chmod_modes)) { snprintf(err_buf, sizeof err_buf, - "Invalid argument passed to --chmod (%s)\n", - arg); + "Invalid argument passed to --chmod (%s)\n", + arg); return 0; } break; @@ -1710,11 +1735,11 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (usermap) { if (usermap_via_chown) { snprintf(err_buf, sizeof err_buf, - "--usermap conflicts with prior --chown.\n"); + "--usermap conflicts with prior --chown.\n"); return 0; } snprintf(err_buf, sizeof err_buf, - "You can only specify --usermap once.\n"); + "You can only specify --usermap once.\n"); return 0; } usermap = (char *)poptGetOptArg(pc); @@ -1725,11 +1750,11 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (groupmap) { if (groupmap_via_chown) { snprintf(err_buf, sizeof err_buf, - "--groupmap conflicts with prior --chown.\n"); + "--groupmap conflicts with prior --chown.\n"); return 0; } snprintf(err_buf, sizeof err_buf, - "You can only specify --groupmap once.\n"); + "You can only specify --groupmap once.\n"); return 0; } groupmap = (char *)poptGetOptArg(pc); @@ -1747,11 +1772,11 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (usermap) { if (!usermap_via_chown) { snprintf(err_buf, sizeof err_buf, - "--chown conflicts with prior --usermap.\n"); + "--chown conflicts with prior --usermap.\n"); return 0; } snprintf(err_buf, sizeof err_buf, - "You can only specify a user-affecting --chown once.\n"); + "You can only specify a user-affecting --chown once.\n"); return 0; } if (asprintf(&usermap, "*:%.*s", len, chown) < 0) @@ -1762,11 +1787,11 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (groupmap) { if (!groupmap_via_chown) { snprintf(err_buf, sizeof err_buf, - "--chown conflicts with prior --groupmap.\n"); + "--chown conflicts with prior --groupmap.\n"); return 0; } snprintf(err_buf, sizeof err_buf, - "You can only specify a group-affecting --chown once.\n"); + "You can only specify a group-affecting --chown once.\n"); return 0; } if (asprintf(&groupmap, "*:%s", arg) < 0) @@ -1792,7 +1817,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) * of any more restrictive ACLs, but this is safe * for now */ snprintf(err_buf,sizeof(err_buf), - "ACLs are not supported on this %s\n", + "ACLs are not supported on this %s\n", am_server ? "server" : "client"); return 0; #endif @@ -1837,12 +1862,11 @@ int parse_arguments(int *argc_p, const char ***argv_p) } } - if (checksum_choice && strcmp(checksum_choice, "auto") != 0 && strcmp(checksum_choice, "auto,auto") != 0) { + if (checksum_choice && strcasecmp(checksum_choice, "auto") != 0 && strcasecmp(checksum_choice, "auto,auto") != 0) { /* Call this early to verify the args and figure out if we need to force * --whole-file. Note that the parse function will get called again later, * just in case an "auto" choice needs to know the protocol_version. */ - if (parse_checksum_choice()) - whole_file = 1; + parse_checksum_choice(0); } else checksum_choice = NULL; @@ -1852,31 +1876,20 @@ int parse_arguments(int *argc_p, const char ***argv_p) exit_cleanup(0); } - if (do_compression || def_compress_level != NOT_SPECIFIED) { - if (def_compress_level == NOT_SPECIFIED) - def_compress_level = Z_DEFAULT_COMPRESSION; - else if (def_compress_level < Z_DEFAULT_COMPRESSION || def_compress_level > Z_BEST_COMPRESSION) { - snprintf(err_buf, sizeof err_buf, "--compress-level value is invalid: %d\n", - def_compress_level); - return 0; - } else if (def_compress_level == Z_NO_COMPRESSION) - do_compression = 0; - else if (!do_compression) - do_compression = 1; + if (!compress_choice && do_compression > 1) + compress_choice = "zlibx"; + if (compress_choice && strcasecmp(compress_choice, "auto") != 0) + parse_compress_choice(0); /* Twiddles do_compression and can possibly NULL-out compress_choice. */ + else + compress_choice = NULL; + + if (do_compression || do_compression_level != CLVL_NOT_SPECIFIED) { + if (!do_compression) + do_compression = CPRES_AUTO; if (do_compression && refused_compress) { create_refuse_error(refused_compress); return 0; } -#ifdef EXTERNAL_ZLIB - if (do_compression == 1) { - snprintf(err_buf, sizeof err_buf, - "This rsync lacks old-style --compress due to its external zlib. Try -zz.\n"); - if (am_server) - return 0; - fprintf(stderr, "%s" "Continuing without compression.\n\n", err_buf); - do_compression = 0; - } -#endif } #ifdef HAVE_SETVBUF @@ -1977,10 +1990,17 @@ int parse_arguments(int *argc_p, const char ***argv_p) } #endif - if (block_size > MAX_BLOCK_SIZE) { - snprintf(err_buf, sizeof err_buf, - "--block-size=%lu is too large (max: %u)\n", block_size, MAX_BLOCK_SIZE); - return 0; + if (block_size) { + /* We may not know the real protocol_version at this point if this is the client + * option parsing, but we still want to check it so that the client can specify + * a --protocol=29 option with a larger block size. */ + int32 max_blength = protocol_version < 30 ? OLD_MAX_BLOCK_SIZE : MAX_BLOCK_SIZE; + + if (block_size > max_blength) { + snprintf(err_buf, sizeof err_buf, + "--block-size=%lu is too large (max: %u)\n", block_size, max_blength); + return 0; + } } if (write_batch && read_batch) { @@ -2031,12 +2051,6 @@ int parse_arguments(int *argc_p, const char ***argv_p) max_delete = 0; } - if (compare_dest + copy_dest + link_dest > 1) { - snprintf(err_buf, sizeof err_buf, - "You may not mix --compare-dest, --copy-dest, and --link-dest.\n"); - return 0; - } - if (files_from) { if (recurse == 1) /* preserve recurse == 2 */ recurse = 0; @@ -2274,6 +2288,14 @@ int parse_arguments(int *argc_p, const char ***argv_p) inplace = 1; } + if (write_devices) { + if (refused_inplace) { + create_refuse_error(refused_inplace); + return 0; + } + inplace = 1; + } + if (delay_updates && !partial_dir) partial_dir = tmp_partialdir; @@ -2413,7 +2435,8 @@ void server_options(char **args, int *argc_p) for (i = 0; i < verbose; i++) argstr[x++] = 'v'; - /* the -q option is intentionally left out */ + if (quiet && msgs2stderr) + argstr[x++] = 'q'; if (make_backups) argstr[x++] = 'b'; if (update_only) @@ -2465,6 +2488,11 @@ void server_options(char **args, int *argc_p) argstr[x++] = 'D'; if (preserve_times) argstr[x++] = 't'; + if (preserve_atimes) { + argstr[x++] = 'U'; + if (preserve_atimes > 1) + argstr[x++] = 'U'; + } if (preserve_perms) argstr[x++] = 'p'; else if (preserve_executability && am_sender) @@ -2497,7 +2525,7 @@ void server_options(char **args, int *argc_p) } if (sparse_files) argstr[x++] = 'S'; - if (do_compression == 1) + if (do_compression == CPRES_ZLIB) argstr[x++] = 'z'; set_allow_inc_recurse(); @@ -2530,6 +2558,9 @@ void server_options(char **args, int *argc_p) eFlags[x++] = 'f'; /* flist I/O-error safety support */ eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */ eFlags[x++] = 'C'; /* support checksum seed order fix */ + eFlags[x++] = 'I'; /* support inplace_partial behavior */ + eFlags[x++] = 'v'; /* use varint for flist & compat flags; negotiate checksum */ + /* NOTE: Avoid using 'V' -- it was the high bit of a write_byte() that became write_varint(). */ #undef eFlags } @@ -2567,8 +2598,8 @@ void server_options(char **args, int *argc_p) if (xfer_dirs && !recurse && delete_mode && am_sender) args[ac++] = "--no-r"; - if (do_compression && def_compress_level != Z_DEFAULT_COMPRESSION) { - if (asprintf(&arg, "--compress-level=%d", def_compress_level) < 0) + if (do_compression && do_compression_level != CLVL_NOT_SPECIFIED) { + if (asprintf(&arg, "--compress-level=%d", do_compression_level) < 0) goto oom; args[ac++] = arg; } @@ -2631,6 +2662,16 @@ void server_options(char **args, int *argc_p) args[ac++] = arg; } + if (do_compression == CPRES_ZLIBX) + args[ac++] = "--new-compress"; + else if (compress_choice && do_compression == CPRES_ZLIB) + args[ac++] = "--old-compress"; + else if (compress_choice) { + if (asprintf(&arg, "--compress-choice=%s", compress_choice) < 0) + goto oom; + args[ac++] = arg; + } + if (am_sender) { if (max_delete > 0) { if (asprintf(&arg, "--max-delete=%d", max_delete) < 0) @@ -2752,7 +2793,7 @@ void server_options(char **args, int *argc_p) * option, so don't send it if client is the sender. */ for (i = 0; i < basis_dir_cnt; i++) { - args[ac++] = dest_option; + args[ac++] = alt_dest_opt(0); args[ac++] = basis_dir[i]; } } @@ -2764,10 +2805,6 @@ void server_options(char **args, int *argc_p) if (arg) args[ac++] = arg; - arg = make_output_option(debug_words, debug_levels, where); - if (arg) - args[ac++] = arg; - if (append_mode) { if (append_mode > 1) args[ac++] = "--append"; @@ -2792,6 +2829,9 @@ void server_options(char **args, int *argc_p) if (relative_paths && !implied_dirs && (!am_sender || protocol_version >= 30)) args[ac++] = "--no-implied-dirs"; + if (write_devices && am_sender) + args[ac++] = "--write-devices"; + if (remove_source_files == 1) args[ac++] = "--remove-source-files"; else if (remove_source_files) @@ -2800,14 +2840,14 @@ void server_options(char **args, int *argc_p) if (preallocate_files && am_sender) args[ac++] = "--preallocate"; + if (open_noatime && preserve_atimes <= 1) + args[ac++] = "--open-noatime"; + if (ac > MAX_SERVER_ARGS) { /* Not possible... */ rprintf(FERROR, "argc overflow in server_options().\n"); exit_cleanup(RERR_MALLOC); } - if (do_compression > 1) - args[ac++] = "--new-compress"; - if (remote_option_cnt) { int j; if (ac + remote_option_cnt > MAX_SERVER_ARGS) { @@ -2881,7 +2921,7 @@ static char *parse_hostspec(char *str, char **path_start_ptr, int *port_ptr) return ret; } -/* Look for a HOST specfication of the form "HOST:PATH", "HOST::PATH", or +/* Look for a HOST specification of the form "HOST:PATH", "HOST::PATH", or * "rsync://HOST:PORT/PATH". If found, *host_ptr will be set to some allocated * memory with the HOST. If a daemon-accessing spec was specified, the value * of *port_ptr will contain a non-0 port number, otherwise it will be set to @@ -2896,7 +2936,7 @@ char *check_for_hostspec(char *s, char **host_ptr, int *port_ptr) *host_ptr = parse_hostspec(s + strlen(URL_PREFIX), &path, port_ptr); if (*host_ptr) { if (!*port_ptr) - *port_ptr = RSYNC_PORT; + *port_ptr = -1; /* -1 indicates they want the default */ return path; } } @@ -2907,7 +2947,7 @@ char *check_for_hostspec(char *s, char **host_ptr, int *port_ptr) if (*path == ':') { if (port_ptr && !*port_ptr) - *port_ptr = RSYNC_PORT; + *port_ptr = -1; return path + 1; } if (port_ptr) diff --git a/packaging/bin/gpg b/packaging/bin/gpg deleted file mode 100755 index e5d8f476..00000000 --- a/packaging/bin/gpg +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -e -# This script gets git to run gpg with a --passphrase-file option. - -PATH=`echo $PATH | sed 's/^[^:]*://'` - -gpg --batch --passphrase-file=$GPG_PASSFILE "${@}" diff --git a/packaging/branch-from-patch b/packaging/branch-from-patch index 469f3d60..dfd3a29a 100755 --- a/packaging/branch-from-patch +++ b/packaging/branch-from-patch @@ -1,180 +1,174 @@ -#!/usr/bin/perl - -use strict; -use warnings; -use Getopt::Long; - -&Getopt::Long::Configure('bundling'); -&usage if !&GetOptions( - 'branch|b=s' => \( my $master_branch = 'master' ), - 'skip-check' => \( my $skip_branch_check ), - 'delete' => \( my $delete_local_branches ), - 'help|h' => \( my $help_opt ), -); -&usage if $help_opt; - -require 'packaging/git-status.pl'; -check_git_state($master_branch, !$skip_branch_check, 1); - -my %local_branch; -open PIPE, '-|', 'git branch -l' or die "Unable to fork: $!\n"; -while (<PIPE>) { - if (m# patch/\Q$master_branch\E/(.*)#o) { - $local_branch{$1} = 1; - } -} -close PIPE; - -if ($delete_local_branches) { - foreach my $name (sort keys %local_branch) { - my $branch = "patch/$master_branch/$name"; - system 'git', 'branch', '-D', $branch and exit 1; - } - %local_branch = ( ); -} - -my @patch_list; -foreach (@ARGV) { - if (!-f $_) { - die "File not found: $_\n"; - } - die "Filename is not a .diff file: $_\n" unless /\.diff$/; - push @patch_list, $_; -} - -exit unless @patch_list; - -my(%scanned, %created, %info); - -foreach my $patch (@patch_list) { - my($where, $name) = $patch =~ m{^(.*?)([^/]+)\.diff$}; - next if $scanned{$name}++; - - open IN, '<', $patch or die "Unable to open $patch: $!\n"; - - my $info = ''; - my $commit; - while (<IN>) { - if (m#^based-on: (\S+)#) { - $commit = $1; - last; - } - last if m#^index .*\.\..* \d#; - last if m#^diff --git #; - last if m#^--- (old|a)/#; - $info .= $_; - } - close IN; - - $info =~ s/\s+\Z/\n/; - - my $parent = $master_branch; - my @patches = $info =~ m#patch -p1 <patches/(\S+)\.diff#g; - if (@patches) { - if ($patches[-1] eq $name) { - pop @patches; - } else { - warn "No identity patch line in $patch\n"; - } - if (@patches) { - $parent = pop @patches; - if (!$scanned{$parent}) { - unless (-f "$where$parent.diff") { - die "Unknown parent of $patch: $parent\n"; - } - # Add parent to @patch_list so that we will look for the - # parent's parent. Any duplicates will just be ignored. - push @patch_list, "$where$parent.diff"; - } - } - } else { - warn "No patch lines found in $patch\n"; - } - - $info{$name} = [ $parent, $info, $commit ]; -} - -foreach my $patch (@patch_list) { - create_branch($patch); -} - -system 'git', 'checkout', $master_branch and exit 1; - -exit; - -sub create_branch -{ - my($patch) = @_; - my($where, $name) = $patch =~ m{^(.*?)([^/]+)\.diff$}; - - return if $created{$name}++; - - my $ref = $info{$name}; - my($parent, $info, $commit) = @$ref; - - my $parent_branch; - if ($parent eq $master_branch) { - $parent_branch = $master_branch; - $parent_branch = $commit if defined $commit; - } else { - create_branch("$where/$parent.diff"); - $parent_branch = "patch/$master_branch/$parent"; - } - - my $branch = "patch/$master_branch/$name"; - print "\n", '=' x 64, "\nProcessing $branch ($parent_branch)\n"; - - if ($local_branch{$name}) { - system 'git', 'branch', '-D', $branch and exit 1; - } - - system 'git', 'checkout', '-b', $branch, $parent_branch and exit 1; - - open OUT, '>', "PATCH.$name" or die $!; - print OUT $info; - close OUT; - system 'git', 'add', "PATCH.$name" and exit 1; - - open IN, '<', $patch or die "Unable to open $patch: $!\n"; - $_ = join('', <IN>); - close IN; - - open PIPE, '|-', 'patch -p1' or die $!; - print PIPE $_; - close PIPE; - - system 'rm -f *.orig */*.orig'; - - while (m#\nnew file mode (\d+)\s+--- /dev/null\s+\Q+++\E b/(.*)#g) { - chmod oct($1), $2; - system 'git', 'add', $2; - } - - while (1) { - system 'git status'; - print 'Press Enter to commit, Ctrl-C to abort, or type a wild-name to add a new file: '; - $_ = <STDIN>; - last if /^$/; - chomp; - system "git add $_"; - } - - while (system 'git', 'commit', '-a', '-m', "Creating branch from $name.diff.") { - exit 1 if system '/bin/zsh'; - } -} - -sub usage -{ - die <<EOT; -Usage branch-from-patch [OPTIONS] patches/DIFF... - -Options: --b, --branch=BRANCH Create branches relative to BRANCH if no "based-on" - header was found in the patch file. - --skip-check Skip the check that ensures starting with a clean branch. - --delete Delete all the local patch/BASE/* branches, not just the ones - that are being recreated. --h, --help Output this help message. -EOT -} +#!/usr/bin/env -S python3 -B + +# This script turns one or more diff files in the patches dir (which is +# expected to be a checkout of the rsync-patches git repo) into a branch +# in the main rsync git checkout. This allows the applied patch to be +# merged with the latest rsync changes and tested. To update the diff +# with the resulting changes, see the patch-update script. + +import os, sys, re, argparse, glob + +sys.path = ['packaging'] + sys.path + +from pkglib import * + +def main(): + global created, info, local_branch + + cur_branch, args.base_branch = check_git_state(args.base_branch, not args.skip_check, args.patches_dir) + + local_branch = get_patch_branches(args.base_branch) + + if args.delete_local_branches: + for name in sorted(local_branch): + branch = f"patch/{args.base_branch}/{name}" + cmd_chk(['git', 'branch', '-D', branch]) + local_branch = set() + + if args.add_missing: + for fn in sorted(glob.glob(f"{args.patches_dir}/*.diff")): + name = re.sub(r'\.diff$', '', re.sub(r'.+/', '', fn)) + if name not in local_branch and fn not in args.patch_files: + args.patch_files.append(fn) + + if not args.patch_files: + return + + for fn in args.patch_files: + if not fn.endswith('.diff'): + die(f"Filename is not a .diff file: {fn}") + if not os.path.isfile(fn): + die(f"File not found: {fn}") + + scanned = set() + info = { } + + patch_list = [ ] + for fn in args.patch_files: + m = re.match(r'^(?P<dir>.*?)(?P<name>[^/]+)\.diff$', fn) + patch = argparse.Namespace(**m.groupdict()) + if patch.name in scanned: + continue + patch.fn = fn + + lines = [ ] + commit_hash = None + with open(patch.fn, 'r', encoding='utf-8') as fh: + for line in fh: + m = re.match(r'^based-on: (\S+)', line) + if m: + commit_hash = m[1] + break + if (re.match(r'^index .*\.\..* \d', line) + or re.match(r'^diff --git ', line) + or re.match(r'^--- (old|a)/', line)): + break + lines.append(re.sub(r'\s*\Z', "\n", line, 1)) + info_txt = ''.join(lines).strip() + "\n" + lines = None + + parent = args.base_branch + patches = re.findall(r'patch -p1 <%s/(\S+)\.diff' % args.patches_dir, info_txt) + if patches: + last = patches.pop() + if last != patch.name: + warn(f"No identity patch line in {patch.fn}") + patches.append(last) + if patches: + parent = patches.pop() + if parent not in scanned: + diff_fn = patch.dir + parent + '.diff' + if not os.path.isfile(diff_fn): + die(f"Failed to find parent of {patch.fn}: {parent}") + # Add parent to args.patch_files so that we will look for the + # parent's parent. Any duplicates will be ignored. + args.patch_files.append(diff_fn) + else: + warn(f"No patch lines found in {patch.fn}") + + info[patch.name] = [ parent, info_txt, commit_hash ] + + patch_list.append(patch) + + created = set() + for patch in patch_list: + create_branch(patch) + + cmd_chk(['git', 'checkout', args.base_branch]) + + +def create_branch(patch): + if patch.name in created: + return + created.add(patch.name) + + parent, info_txt, commit_hash = info[patch.name] + parent = argparse.Namespace(dir=patch.dir, name=parent, fn=patch.dir + parent + '.diff') + + if parent.name == args.base_branch: + parent_branch = commit_hash if commit_hash else args.base_branch + else: + create_branch(parent) + parent_branch = '/'.join(['patch', args.base_branch, parent.name]) + + branch = '/'.join(['patch', args.base_branch, patch.name]) + print("\n" + '=' * 64) + print(f"Processing {branch} ({parent_branch})") + + if patch.name in local_branch: + cmd_chk(['git', 'branch', '-D', branch]) + + cmd_chk(['git', 'checkout', '-b', branch, parent_branch]) + + info_fn = 'PATCH.' + patch.name + with open(info_fn, 'w', encoding='utf-8') as fh: + fh.write(info_txt) + cmd_chk(['git', 'add', info_fn]) + + with open(patch.fn, 'r', encoding='utf-8') as fh: + patch_txt = fh.read() + + cmd_run('patch -p1'.split(), input=patch_txt) + + for fn in glob.glob('*.orig') + glob.glob('*/*.orig'): + os.unlink(fn) + + pos = 0 + new_file_re = re.compile(r'\nnew file mode (?P<mode>\d+)\s+--- /dev/null\s+\+\+\+ b/(?P<fn>.+)') + while True: + m = new_file_re.search(patch_txt, pos) + if not m: + break + os.chmod(m['fn'], int(m['mode'], 8)) + cmd_chk(['git', 'add', m['fn']]) + pos = m.end() + + while True: + cmd_chk('git status'.split()) + ans = input('Press Enter to commit, Ctrl-C to abort, or type a wild-name to add a new file: ') + if ans == '': + break + cmd_chk("git add " + ans, shell=True) + + while True: + s = cmd_run(['git', 'commit', '-a', '-m', f"Creating branch from {patch.name}.diff."]) + if not s.returncode: + break + s = cmd_run(['/bin/zsh']) + if s.returncode: + die('Aborting due to shell error code') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Create a git patch branch from an rsync patch file.", add_help=False) + parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.") + parser.add_argument('--add-missing', '-a', action='store_true', help="Add a branch for every patches/*.diff that doesn't have a branch.") + parser.add_argument('--skip-check', action='store_true', help="Skip the check that ensures starting with a clean branch.") + parser.add_argument('--delete', dest='delete_local_branches', action='store_true', help="Delete all the local patch/BASE/* branches, not just the ones that are being recreated.") + parser.add_argument('--patches-dir', '-p', metavar='DIR', default='patches', help="Override the location of the rsync-patches dir. Default: patches.") + parser.add_argument('patch_files', metavar='patches/DIFF_FILE', nargs='*', help="Specify what patch diff files to process. Default: all of them.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/packaging/cull_options b/packaging/cull_options index 388d33f0..7588002b 100755 --- a/packaging/cull_options +++ b/packaging/cull_options @@ -1,11 +1,11 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This script outputs some perl code that parses all possible options # that the code in options.c might send to the server. This perl code # is included in the rrsync script. use strict; our %short_no_arg; -our %short_with_num; +our %short_with_num = ( '@' => 1 ); our %long_opt = ( # These include some extra long-args that BackupPC uses: 'block-size' => 1, 'daemon' => -1, @@ -23,6 +23,7 @@ our %long_opt = ( # These include some extra long-args that BackupPC uses: 'perms' => 0, 'recursive' => 0, 'times' => 0, + 'write-devices' => -1, ); our $last_long_opt; @@ -39,13 +40,13 @@ while (<IN>) { $last_long_opt = $1; $long_opt{$1} = 0 unless exists $long_opt{$1}; } elsif (defined($last_long_opt) - && /\Qargs[ac++]\E = ([^["\s]+);/ && $1 ne 'dest_option') { + && /\Qargs[ac++]\E = ([^["\s]+);/) { $long_opt{$last_long_opt} = 2; undef $last_long_opt; - } elsif (/dest_option = "--([^"]+)"/) { + } elsif (/return "--([^"]+-dest)";/) { $long_opt{$1} = 2; undef $last_long_opt; - } elsif (/\Qasprintf(\E[^,]+, "--([^"=]+)=/ || /\Qargs[ac++]\E = "--([^"=]+)=/) { + } elsif (/\Qasprintf(\E[^,]+, "--([^"=]+)=/ || /\Qargs[ac++]\E = "--([^"=]+)=/ || /fmt = .*: "--([^"=]+)=/) { $long_opt{$1} = 1; undef $last_long_opt; } @@ -76,7 +77,8 @@ foreach my $opt (sort keys %long_opt) { my $val = $long_opt{$opt}; $val = 1 if $opt =~ /^(max-|min-)/; $val = 3 if $opt eq 'files-from'; - $val = '$ro ? -1 : ' . $val if $opt =~ /^remove-/; + $val = q"$only eq 'r' ? -1 : " . $val if $opt =~ /^(remove-|log-file)/; + $val = q"$only eq 'w' ? -1 : " . $val if $opt eq 'sender'; print " '$opt' => $val,\n"; } diff --git a/packaging/git-status.pl b/packaging/git-status.pl deleted file mode 100644 index 9d322e65..00000000 --- a/packaging/git-status.pl +++ /dev/null @@ -1,51 +0,0 @@ -# Do some git-status checking for the current dir and (optionally) -# the patches dir. - -sub check_git_state -{ - my($master_branch, $fatal_unless_clean, $check_patches_dir) = @_; - - my($cur_branch) = check_git_status($fatal_unless_clean); - (my $branch = $cur_branch) =~ s{^patch/([^/]+)/[^/]+$}{$1}; # change patch/BRANCH/PATCH_NAME into BRANCH - if ($branch ne $master_branch) { - print "The checkout is not on the $master_branch branch.\n"; - exit 1 if $master_branch ne 'master'; - print "Do you want me to continue with --branch=$branch? [n] "; - $_ = <STDIN>; - exit 1 unless /^y/i; - $_[0] = $master_branch = $branch; # Updates caller's $master_branch too. - } - - if ($check_patches_dir && -d 'patches/.git') { - ($branch) = check_git_status($fatal_unless_clean, 'patches'); - if ($branch ne $master_branch) { - print "The *patches* checkout is on branch $branch, not branch $master_branch.\n"; - print "Do you want to change it to branch $master_branch? [n] "; - $_ = <STDIN>; - exit 1 unless /^y/i; - system "cd patches && git checkout '$master_branch'"; - } - } - - return $cur_branch; -} - -sub check_git_status -{ - my($fatal_unless_clean, $subdir) = @_; - $subdir = '.' unless defined $subdir; - my $status = `cd '$subdir' && git status`; - my $is_clean = $status =~ /\nnothing to commit.+working directory clean/; - my($cur_branch) = $status =~ /^(?:# )?On branch (.+)\n/; - if ($fatal_unless_clean && !$is_clean) { - if ($subdir eq '.') { - $subdir = ''; - } else { - $subdir = " *$subdir*"; - } - die "The$subdir checkout is not clean:\n", $status; - } - ($cur_branch, $is_clean, $status); -} - -1; diff --git a/packaging/lsb/rsync.spec b/packaging/lsb/rsync.spec index 0e7794c4..9db4e3ad 100644 --- a/packaging/lsb/rsync.spec +++ b/packaging/lsb/rsync.spec @@ -1,6 +1,6 @@ Summary: A fast, versatile, remote (and local) file-copying tool Name: rsync -Version: 3.1.3 +Version: 3.2.0 %define fullversion %{version} Release: 1 %define srcdir src @@ -13,11 +13,6 @@ URL: http://rsync.samba.org/ Prefix: %{_prefix} BuildRoot: /var/tmp/%{name}-root -%package ssl-client -Summary: Provides rsync-ssl -Group: Applications/Internet -Requires: rsync, stunnel >= 4 - %package ssl-daemon Summary: An stunnel config file to support ssl rsync daemon connections. Group: Applications/Internet @@ -34,11 +29,6 @@ differences between the source files and the existing files in the destination. Rsync is widely used for backups and mirroring and as an improved copy command for everyday use. -%description ssl-client -Provides the rsync-ssl script that makes use of stunnel 4 to open an ssl -connection to an rsync daemon (on port 874). This setup does NOT require -any local stunnel daemon to be running to connect to the remote ssl rsyncd. - %description ssl-daemon Provides a config file for stunnel that will (if you start your stunnel service) cause stunnel to listen for ssl rsync-daemon connections and run @@ -66,7 +56,7 @@ make %install rm -rf $RPM_BUILD_ROOT -make install install-ssl-client install-ssl-daemon DESTDIR=$RPM_BUILD_ROOT +make install install-ssl-daemon DESTDIR=$RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT/etc/xinetd.d $RPM_BUILD_ROOT/etc/rsync-ssl/certs install -m 644 packaging/lsb/rsync.xinetd $RPM_BUILD_ROOT/etc/xinetd.d/rsync @@ -76,24 +66,22 @@ rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) -%doc COPYING NEWS OLDNEWS README support/ tech_report.tex +%doc COPYING NEWS.md OLDNEWS.md README.md support/ tech_report.tex %config(noreplace) /etc/xinetd.d/rsync %{_prefix}/bin/rsync +%{_prefix}/bin/rsync-ssl %{_mandir}/man1/rsync.1* +%{_mandir}/man1/rsync-ssl.1* %{_mandir}/man5/rsyncd.conf.5* -%files ssl-client -%{_prefix}/bin/rsync-ssl -%{_prefix}/bin/stunnel-rsync - %files ssl-daemon %config(noreplace) /etc/stunnel/rsyncd.conf %dir /etc/rsync-ssl/certs %changelog -* Sun Jan 28 2018 Wayne Davison <wayned@samba.org> -Released 3.1.3. +* Fri Jun 19 2020 Wayne Davison <wayne@opencoder.net> +Released 3.2.0. -* Fri Mar 21 2008 Wayne Davison <wayned@samba.org> +* Fri Mar 21 2008 Wayne Davison <wayne@opencoder.net> Added installation of /etc/xinetd.d/rsync file and some commented-out lines that demonstrate how to use the rsync-patches tar file. diff --git a/packaging/md2html b/packaging/md2html new file mode 100755 index 00000000..e4ee226a --- /dev/null +++ b/packaging/md2html @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020 Wayne Davison +# +# This program is freely redistributable. + +import os, re, argparse + +HTML_START = """\ +<html><head> +<title>%s</title> +<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet"> +<style> +body { + max-width: 50em; + margin: auto; +} +body, b, strong, u { + font-family: 'Roboto', sans-serif; +} +code { + font-family: 'Roboto Mono', monospace; + font-weight: bold; +} +pre code { + display: block; + font-weight: normal; +} +blockquote pre code { + background: #f1f1f1; +} +dd p:first-of-type { + margin-block-start: 0em; +} +</style> +</head><body> +""" + +HTML_END = """\ +</body></html> +""" + +md_parser = None + +def main(): + for mdfn in args.mdfiles: + if not mdfn.endswith('.md'): + print('Ignoring non-md input file:', mdfn) + continue + title = re.sub(r'.*/', '', mdfn).replace('.md', '') + htfn = mdfn.replace('.md', '.html') + + print("Parsing", mdfn, '->', htfn) + + with open(mdfn, 'r', encoding='utf-8') as fh: + txt = fh.read() + + txt = re.sub(r'\s--\s', '\xa0-- ', txt) + + html = md_parser(txt) + + html = re.sub(r'(<code>)([\s\S]*?)(</code>)', lambda m: m[1] + re.sub(r'\s', '\xa0', m[2]) + m[3], html) + html = html.replace('--', '‑‑').replace("\xa0-", ' ‑').replace("\xa0", ' ') + html = re.sub(r'(\W)-', r'\1‑', html) + + if os.path.lexists(htfn): + os.unlink(htfn) + + with open(htfn, 'w', encoding='utf-8') as fh: + fh.write(HTML_START % title) + fh.write(html) + fh.write(HTML_END) + + +def html_via_cmarkgfm(txt): + return cmarkgfm.markdown_to_html(txt) + + +def html_via_commonmark(txt): + return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Output html for md pages.', add_help=False) + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + parser.add_argument("mdfiles", nargs='+', help="The .md files to turn into .html files.") + args = parser.parse_args() + + try: + import cmarkgfm + md_parser = html_via_cmarkgfm + except: + try: + import commonmark + md_parser = html_via_commonmark + except: + die("Failed to find cmarkgfm or commonmark for python3.") + + main() diff --git a/packaging/nightly-rsync b/packaging/nightly-rsync index 23451ca7..020a6d56 100755 --- a/packaging/nightly-rsync +++ b/packaging/nightly-rsync @@ -1,174 +1,106 @@ -#!/usr/bin/perl -use strict; +#!/usr/bin/env -S python3 -B # This script expects the directory ~/samba-rsync-ftp to exist and to be a # copy of the /home/ftp/pub/rsync dir on samba.org. It also requires a # git checkout of rsync (feel free to use your normal rsync build dir as # long as it doesn't have any uncommitted changes). # -# If this is run with -ctu, it will make an updated "nightly" tar file in +# If this is run with -tu, it will make an updated "nightly" tar file in # the nightly dir. It will also remove any old tar files, regenerate the # HTML man pages in the nightly dir, and then rsync the changes to the # samba.org server. -use Getopt::Long; -use Date::Format; +import os, sys, re, argparse, glob +from datetime import datetime, timezone +from getpass import getpass + +sys.path = ['packaging'] + sys.path + +from pkglib import * # Where the local copy of /home/ftp/pub/rsync/dev/nightly should be updated. -our $dest = $ENV{HOME} . '/samba-rsync-ftp/dev/nightly'; -our $samba_host = $ENV{SAMBA_HOST} || 'samba.org'; -our $nightly_symlink = "$dest/rsync-HEAD.tar.gz"; - -our($make_tar, $upload, $help_opt); -&Getopt::Long::Configure('bundling'); -&usage if !&GetOptions( - 'make-tar|t' => \$make_tar, - 'upload|u' => \$upload, - 'help|h' => \$help_opt, -) || $help_opt; - -our $name = time2str('rsync-HEAD-%Y%m%d-%H%M%Z', time, 'GMT'); -our $ztoday = time2str('%d %b %Y', time); -our $today = $ztoday; -our $gen_target = $upload ? 'gensend' : 'gen'; - -die "$dest does not exist\n" unless -d $dest; -die "There is no .git dir in the current directory.\n" unless -d '.git'; -die "There is no rsync checkout in the current directory.\n" unless -f 'rsyncd.conf.yo'; - -if ($make_tar) { - open(IN, '-|', 'git status') or die $!; - my $status = join('', <IN>); - close IN; - die "The checkout is not clean:\n", $status unless $status =~ /\nnothing to commit.+working directory clean/; - die "The checkout is not on the master branch.\n" unless $status =~ /^(?:# )?On branch master\n/; - system "make $gen_target" and die "make $gen_target failed!\n"; - - my @extra_files; - open(IN, '<', 'Makefile.in') or die "Couldn't open Makefile.in: $!\n"; - while (<IN>) { - if (s/^GENFILES=//) { - while (s/\\$//) { - $_ .= <IN>; - } - @extra_files = split(' ', $_); - last; - } - } - close IN; - - my $confversion; - open(IN, '<', 'configure.ac') or die "Unable to open configure.ac: $!\n"; - while (<IN>) { - if (/^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]/) { - $confversion = $1; - last; - } - } - close IN; - die "Unable to find AC_INIT with version in configure.ac\n" unless defined $confversion; - - open(IN, '<', 'OLDNEWS') or die "Unable to open OLDNEWS: $!\n"; - $_ = <IN>; - my($lastversion) = /(\d+\.\d+\.\d+)/; - my $last_protocol_version; - while (<IN>) { - if (my($ver,$pdate,$pver) = /^\s+\S\S\s\S\S\S\s\d\d\d\d\s+(\d+\.\d+\.\d+)\s+(\d\d \w\w\w \d\d\d\d\s+)?(\d+)$/) { - $last_protocol_version = $pver if $ver eq $lastversion; - } - } - close IN; - die "Unable to determine protocol_version for $lastversion.\n" unless defined $last_protocol_version; - - my($protocol_version,$subprotocol_version); - open(IN, '<', 'rsync.h') or die "Unable to open rsync.h: $!\n"; - while (<IN>) { - if (/^#define\s+PROTOCOL_VERSION\s+(\d+)/) { - $protocol_version = $1; - } elsif (/^#define\s+SUBPROTOCOL_VERSION\s+(\d+)/) { - $subprotocol_version = $1; - } - } - close IN; - die "Unable to determine the current PROTOCOL_VERSION.\n" unless defined $protocol_version; - die "Unable to determine the current SUBPROTOCOL_VERSION.\n" unless defined $subprotocol_version; - - if ($confversion =~ /dev|pre/) { - if ($last_protocol_version ne $protocol_version) { - if ($subprotocol_version == 0) { - die "SUBPROTOCOL_VERSION must not be 0 for a non-final release with a changed PROTOCOL_VERSION.\n"; - } - } else { - if ($subprotocol_version != 0) { - die "SUBPROTOCOL_VERSION must be 0 when the PROTOCOL_VERSION hasn't changed from the last release.\n"; - } - } - } else { - if ($subprotocol_version != 0) { - die "SUBPROTOCOL_VERSION must be 0 for a final release.\n"; - } - } - - print "Creating $name.tar.gz\n"; - system "rsync -a @extra_files $name/"; - system "git archive --format=tar --prefix=$name/ HEAD | tar xf -"; - system "support/git-set-file-times --prefix=$name/"; - system "fakeroot tar czf $dest/$name.tar.gz $name; rm -rf $name"; - - unlink($nightly_symlink); - symlink("$name.tar.gz", $nightly_symlink); -} - -foreach my $fn (qw( rsync.yo rsyncd.conf.yo )) { - my $yo_tmp = "$dest/$fn"; - (my $html_fn = "$dest/$fn") =~ s/\.yo/.html/; - - open(IN, '<', $fn) or die $!; - undef $/; $_ = <IN>; $/ = "\n"; - close IN; - - s/^(manpage\([^)]+\)\(\d+\)\()[^)]+(\).*)/$1$today$2/m; - #s/^(This man ?page is current for version) \S+ (of rsync)/$1 $version $2/m; - - open(OUT, '>', $yo_tmp) or die $!; - print OUT $_; - close OUT; - - system 'yodl2html', '-o', $html_fn, $yo_tmp; - - unlink($yo_tmp); -} - -chdir($dest) or die $!; - -my $cnt = 0; -open(PIPE, '-|', 'ls -1t rsync-HEAD-*') or die $!; -while (<PIPE>) { - chomp; - next if $cnt++ < 10; - unlink($_); -} -close PIPE; - -system 'ls -ltr'; - -if ($upload) { - my $opt = ''; - if (defined $ENV{RSYNC_PARTIAL_DIR}) { - $opt = " -f 'R $ENV{RSYNC_PARTIAL_DIR}'"; - } - system "rsync$opt -aviHP --delete-after . $samba_host\:/home/ftp/pub/rsync/dev/nightly"; -} - -exit; - -sub usage -{ - die <<EOT; -Usage: nightly-rsync [OPTIONS] - - -t, --make-tar create a new tar file in $dest - -u, --upload upload the revised nightly dir to samba.org - -h, --help display this help -EOT -} +dest = os.environ['HOME'] + '/samba-rsync-ftp/dev/nightly' +samba_host = os.environ['SAMBA_HOST'] +nightly_symlink = f"{dest}/rsync-HEAD.tar.gz" + +def main(): + now = datetime.now(timezone.utc) + name = now.strftime('rsync-HEAD-%Y%m%d-%H%MGMT') + ztoday = now.strftime('%d %b %Y') + today = ztoday.lstrip('0') + gen_target = 'gensend' if args.upload else 'gen' + + if not os.path.isdir(dest): + die("$dest does not exist") + if not os.path.isdir('.git'): + die("There is no .git dir in the current directory.") + if not os.path.exists('rsyncd.conf.5.md'): + die("There is no rsync checkout in the current directory.") + + mandate_gensend_hook() + + if args.make_tar: + check_git_state('master') + cmd_chk(['touch', 'NEWS.md']) + cmd_chk(['make', gen_target]) + cmd_chk(['rsync', '-a', *glob.glob('*.[1-9].html'), dest]) + + gen_files = get_gen_files() + + confversion = get_configure_version() + + # All version values are strings! + last_version, last_protocol_version = get_OLDNEWS_version_info() + protocol_version, subprotocol_version = get_protocol_versions() + + if 'dev' in confversion or 'pre' in confversion: + if last_protocol_version != protocol_version: + if subprotocol_version == '0': + die("SUBPROTOCOL_VERSION must not be 0 for a non-final release with a changed PROTOCOL_VERSION.") + elif subprotocol_version != '0': + die("SUBPROTOCOL_VERSION must be 0 when the PROTOCOL_VERSION hasn't changed from the last release.") + elif subprotocol_version != '0': + die("SUBPROTOCOL_VERSION must be 0 for a final release.") + + name_slash = name + '/' + tar_name = f"{name}.tar.gz" + + print('Creating', tar_name) + + cmd_chk(['rsync', '-a', *gen_files, name_slash]) + cmd_chk(f"git archive --format=tar --prefix={name}/ HEAD | tar xf -") + cmd_chk(['support/git-set-file-times', '--quiet', '--prefix', name_slash]) + cmd_chk(['fakeroot', 'tar', 'czf', os.path.join(dest, tar_name), name]) + cmd_chk(['rm', '-rf', name]) + + if os.path.lexists(nightly_symlink): + os.unlink(nightly_symlink) + os.symlink(tar_name, nightly_symlink) + + os.chdir(dest) + + tar_files = list(reversed(sorted(glob.glob('rsync-HEAD-*')))) + if len(tar_files) > 10: + for fn in tar_files[10:]: + print('Removing', fn) + os.unlink(fn) + + cmd_run('ls -ltr'.split()) + + if args.upload: + cmd = 'rsync -aivHP --delete-after'.split() + partial_dir = os.environ.get('RSYNC_PARTIAL_DIR', None) + if partial_dir: + cmd.append('-fR ' + partial_dir) + cmd_chk([*cmd, '.', f"{samba_host}:/home/ftp/pub/rsync/dev/nightly"]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='A helper script for "nightly" tar files.', add_help=False) + parser.add_argument('--make-tar', '-t', action='store_true', help=f"Create a new tar file in {dest}.") + parser.add_argument('--upload', '-u', action='store_true', help="Upload the revised nightly dir to {samba_host}.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/packaging/patch-update b/packaging/patch-update index 4839716d..c6b51527 100755 --- a/packaging/patch-update +++ b/packaging/patch-update @@ -1,242 +1,210 @@ -#!/usr/bin/perl +#!/usr/bin/env -S python3 -B + # This script is used to turn one or more of the "patch/BASE/*" branches # into one or more diffs in the "patches" directory. Pass the option # --gen if you want generated files in the diffs. Pass the name of # one or more diffs if you want to just update a subset of all the # diffs. -use strict; -use warnings; -use Getopt::Long; - -my $patches_dir = 'patches'; -my $tmp_dir = "patches.$$"; -my $make_gen_cmd = 'make -f prepare-source.mak conf && ./config.status && make gen'; - -&Getopt::Long::Configure('bundling'); -&usage if !&GetOptions( - 'branch|b=s' => \( my $master_branch = 'master' ), - 'skip-check' => \( my $skip_branch_check ), - 'shell|s' => \( my $launch_shell ), - 'gen:s' => \( my $incl_generated_files ), - 'help|h' => \( my $help_opt ), -); -&usage if $help_opt; - -$ENV{GIT_MERGE_AUTOEDIT} = 'no'; - -if (defined $incl_generated_files) { - $patches_dir = $incl_generated_files if $incl_generated_files ne ''; - $incl_generated_files = 1; -} - -die "No '$patches_dir' directory was found.\n" unless -d $patches_dir; -die "No '.git' directory present in the current dir.\n" unless -d '.git'; - -require 'packaging/git-status.pl'; -my $starting_branch = check_git_state($master_branch, !$skip_branch_check, 1); - -my $master_commit; -open PIPE, '-|', "git log -1 --no-color $master_branch" or die $!; -while (<PIPE>) { - if (/^commit (\S+)/) { - $master_commit = $1; - last; - } -} -close PIPE; -die "Unable to determine commit hash for master branch: $master_branch\n" unless defined $master_commit; - -if ($incl_generated_files) { - my @extra_files = get_extra_files(); - die "'$tmp_dir' must not exist in the current directory.\n" if -e $tmp_dir; - mkdir($tmp_dir, 0700) or die "Unable to mkdir($tmp_dir): $!\n"; - system "$make_gen_cmd && rsync -a @extra_files $tmp_dir/master/" and exit 1; -} -our $last_touch = time; - -my %patches; - -# Start by finding all patches so that we can load all possible parents. -open(PIPE, '-|', 'git', 'branch', '-l') or die $!; -while (<PIPE>) { - if (m# patch/\Q$master_branch\E/(.*)#o) { - $patches{$1} = 1; - } -} -close PIPE; - -my @patches = sort keys %patches; - -my(%parent, %description); -foreach my $patch (@patches) { - my $branch = "patch/$master_branch/$patch"; - my $desc = ''; - open(PIPE, '-|', 'git', 'diff', '-U1000', "$master_branch...$branch", '--', "PATCH.$patch") or die $!; - while (<PIPE>) { - last if /^@@ /; - } - while (<PIPE>) { - next unless s/^[ +]//; - if (m#patch -p1 <patches/(\S+)\.diff# && $1 ne $patch) { - my $parent = $parent{$patch} = $1; - if (!$patches{$parent}) { - die "Parent of $patch is not a local branch: $parent\n"; - } - } - $desc .= $_; - } - close PIPE; - $description{$patch} = $desc; -} - -if (@ARGV) { - # Limit the list of patches to actually process based on @ARGV. - @patches = ( ); - foreach (@ARGV) { - s{^patch(es)?/} {}; - s{\.diff$} {}; - if (!$patches{$_}) { - die "Local branch not available for patch: $_\n"; - } - push(@patches, $_); - } -} - -my %completed; -foreach my $patch (@patches) { - next if $completed{$patch}++; - last unless update_patch($patch); -} - -if ($incl_generated_files) { - system "rm -rf $tmp_dir"; -} - -sleep 1 while $last_touch >= time; -system "git checkout $starting_branch" and exit 1; - -exit; - - -sub update_patch -{ - my($patch) = @_; - - my $parent = $parent{$patch}; - my $based_on; - if (defined $parent) { - unless ($completed{$parent}++) { - update_patch($parent); - } - $based_on = $parent = "patch/$master_branch/$parent"; - } else { - $parent = $master_branch; - $based_on = $master_commit; - } - - print "======== $patch ========\n"; - - sleep 1 while $incl_generated_files && $last_touch >= time; - system "git checkout patch/$master_branch/$patch" and return 0; - - my $ok = system("git merge $based_on") == 0; - if (!$ok || $launch_shell) { - my($parent_dir) = $parent =~ m{([^/]+)$}; - print qq|"git merge $based_on" incomplete -- please fix.\n| if !$ok; - $ENV{PS1} = "[$parent_dir] $patch: "; - while (1) { - if (system($ENV{SHELL}) != 0) { - print "Abort? [n/y] "; - $_ = <STDIN>; - next unless /^y/i; - return 0; - } - my($cur_branch, $is_clean, $status) = check_git_status(0); - last if $is_clean; - print $status; - } - } - - open(OUT, '>', "$patches_dir/$patch.diff") or die $!; - print OUT $description{$patch}, "\nbased-on: $based_on\n"; - - my @extra_files; - if ($incl_generated_files) { - @extra_files = get_extra_files(); - system "$make_gen_cmd && rsync -a @extra_files $tmp_dir/$patch/" and exit 1; - } - $last_touch = time; - - open(PIPE, '-|', 'git', 'diff', $based_on) or die $!; - DIFF: while (<PIPE>) { - while (m{^diff --git a/PATCH}) { - while (<PIPE>) { - last if m{^diff --git a/}; - } - last DIFF if !defined $_; - } - next if /^index /; - print OUT $_; - } - close PIPE; - - if ($incl_generated_files) { - my $parent_dir; - if ($parent eq $master_branch) { - $parent_dir = 'master'; - } else { - ($parent_dir) = $parent =~ m{([^/]+)$}; - } - open(PIPE, '-|', 'diff', '-Nurp', "$tmp_dir/$parent_dir", "$tmp_dir/$patch") or die $!; - while (<PIPE>) { - s#^(diff -Nurp) $tmp_dir/[^/]+/(.*?) $tmp_dir/[^/]+/(.*)#$1 a/$2 b/$3#o; - s#^\Q---\E $tmp_dir/[^/]+/([^\t]+)\t.*#--- a/$1#o; - s#^\Q+++\E $tmp_dir/[^/]+/([^\t]+)\t.*#+++ b/$1#o; - print OUT $_; - } - close PIPE; - unlink @extra_files; - } - - close OUT; - - 1; -} - -exit; - -sub get_extra_files -{ - my @extras; - - open(IN, '<', 'Makefile.in') or die "Couldn't open Makefile.in: $!\n"; - while (<IN>) { - if (s/^GENFILES=//) { - while (s/\\$//) { - $_ .= <IN>; - } - @extras = split(' ', $_); - last; - } - } - close IN; - - return @extras; -} - -sub usage -{ - die <<EOT; -Usage: patch-update [OPTIONS] [patches/DIFF...] - -Options: --b, --branch=BRANCH The master branch to merge into the patch/BASE/* branches. - --gen[=DIR] Include generated files. Optional destination DIR - arg overrides the default of using the "patches" dir. - --skip-check Skip the check that ensures starting with a clean branch. --s, --shell Launch a shell for every patch/BASE/* branch updated, not - just when a conflict occurs. --h, --help Output this help message. -EOT -} +import os, sys, re, argparse, time, shutil + +sys.path = ['packaging'] + sys.path + +from pkglib import * + +MAKE_GEN_CMDS = [ + 'make -f prepare-source.mak conf'.split(), + './config.status'.split(), + 'make gen'.split(), + ] +TMP_DIR = "patches.gen" + +os.environ['GIT_MERGE_AUTOEDIT'] = 'no' + +def main(): + global master_commit, parent_patch, description, completed, last_touch + + if not os.path.isdir(args.patches_dir): + die(f'No "{args.patches_dir}" directory was found.') + if not os.path.isdir('.git'): + die('No ".git" directory present in the current dir.') + + starting_branch, args.base_branch = check_git_state(args.base_branch, not args.skip_check, args.patches_dir) + + master_commit = latest_git_hash(args.base_branch) + + if args.gen: + if os.path.lexists(TMP_DIR): + die(f'"{TMP_DIR}" must not exist in the current directory.') + gen_files = get_gen_files() + os.mkdir(TMP_DIR, 0o700) + for cmd in MAKE_GEN_CMDS: + cmd_chk(cmd) + cmd_chk(['rsync', '-a', *gen_files, f'{TMP_DIR}/master/']) + + last_touch = time.time() + + # Start by finding all patches so that we can load all possible parents. + patches = sorted(list(get_patch_branches(args.base_branch))) + + parent_patch = { } + description = { } + + for patch in patches: + branch = f"patch/{args.base_branch}/{patch}" + desc = '' + proc = cmd_pipe(['git', 'diff', '-U1000', f"{args.base_branch}...{branch}", '--', f"PATCH.{patch}"]) + in_diff = False + for line in proc.stdout: + if in_diff: + if not re.match(r'^[ +]', line): + continue + line = line[1:] + m = re.search(r'patch -p1 <patches/(\S+)\.diff', line) + if m and m[1] != patch: + parpat = parent_patch[patch] = m[1] + if not parpat in patches: + die(f"Parent of {patch} is not a local branch: {parpat}") + desc += line + elif re.match(r'^@@ ', line): + in_diff = True + description[patch] = desc + proc.communicate() + + if args.patch_files: # Limit the list of patches to actually process + valid_patches = patches + patches = [ ] + for fn in args.patch_files: + name = re.sub(r'\.diff$', '', re.sub(r'.+/', '', fn)) + if name not in valid_patches: + die(f"Local branch not available for patch: {name}") + patches.append(name) + + completed = set() + + for patch in patches: + if patch in completed: + continue + if not update_patch(patch): + break + + if args.gen: + shutil.rmtree(TMP_DIR) + + while last_touch >= time.time(): + time.sleep(1) + cmd_chk(['git', 'checkout', starting_branch]) + + +def update_patch(patch): + global last_touch + + completed.add(patch) # Mark it as completed early to short-circuit any (bogus) dependency loops. + + parent = parent_patch.get(patch, None) + if parent: + if parent not in completed: + if not update_patch(parent): + return 0 + based_on = parent = f"patch/{args.base_branch}/{parent}" + else: + parent = args.base_branch + based_on = master_commit + + print(f"======== {patch} ========") + + while args.gen and last_touch >= time.time(): + time.sleep(1) + s = cmd_run(f"git checkout patch/{args.base_branch}/{patch}".split()) + if s.returncode != 0: + return 0 + + s = cmd_run(['git', 'merge', based_on]) + ok = s.returncode == 0 + if not ok or args.shell: + m = re.search(r'([^/]+)$', parent) + parent_dir = m[1] + if not ok: + print(f'"git merge {based_on}" incomplete -- please fix.') + os.environ['PS1'] = f"[{parent_dir}] {patch}: " + while True: + s = cmd_run([os.environ.get('SHELL', '/bin/sh')]) + if s.returncode != 0: + ans = input("Abort? [n/y] ") + if re.match(r'^y', ans, flags=re.I): + return 0 + continue + cur_branch, is_clean, status_txt = check_git_status(0) + if is_clean: + break + print(status_txt, end='') + + with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh: + fh.write(description[patch]) + fh.write(f"\nbased-on: {based_on}\n") + + if args.gen: + gen_files = get_gen_files() + for cmd in MAKE_GEN_CMDS: + cmd_chk(cmd) + cmd_chk(['rsync', '-a', *gen_files, f"{TMP_DIR}/{patch}/"]) + else: + gen_files = [ ] + last_touch = time.time() + + proc = cmd_pipe(['git', 'diff', based_on]) + skipping = False + for line in proc.stdout: + if skipping: + if not re.match(r'^diff --git a/', line): + continue + skipping = False + elif re.match(r'^diff --git a/PATCH', line): + skipping = True + continue + if not re.match(r'^index ', line): + fh.write(line) + proc.communicate() + + if args.gen: + e_tmp_dir = re.escape(TMP_DIR) + diff_re = re.compile(r'^(diff -Nurp) %s/[^/]+/(.*?) %s/[^/]+/(.*)' % (e_tmp_dir, e_tmp_dir)) + minus_re = re.compile(r'^\-\-\- %s/[^/]+/([^\t]+)\t.*' % e_tmp_dir) + plus_re = re.compile(r'^\+\+\+ %s/[^/]+/([^\t]+)\t.*' % e_tmp_dir) + + if parent == args.base_branch: + parent_dir = 'master' + else: + m = re.search(r'([^/]+)$', parent) + parent_dir = m[1] + + proc = cmd_pipe(['diff', '-Nurp', f"{TMP_DIR}/{parent_dir}", f"{TMP_DIR}/{patch}"]) + for line in proc.stdout: + line = diff_re.sub(r'\1 a/\2 b/\3', line) + line = minus_re.sub(r'--- a/\1', line) + line = plus_re.sub(r'+++ b/\1', line) + fh.write(line) + proc.communicate() + for fn in gen_files: + os.unlink(fn) + + return 1 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Turn a git branch back into a diff files in the patches dir.", add_help=False) + parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.") + parser.add_argument('--skip-check', action='store_true', help="Skip the check that ensures starting with a clean branch.") + parser.add_argument('--shell', '-s', action='store_true', help="Launch a shell for every patch/BASE/* branch updated, not just when a conflict occurs.") + parser.add_argument('--gen', metavar='DIR', nargs='?', const='', help='Include generated files. Optional DIR value overrides the default of using the "patches" dir.') + parser.add_argument('--patches-dir', '-p', metavar='DIR', default='patches', help="Override the location of the rsync-patches dir. Default: patches.") + parser.add_argument('patch_files', metavar='patches/DIFF_FILE', nargs='*', help="Specify what patch diff files to process. Default: all of them.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + if args.gen == '': + args.gen = args.patches_dir + elif args.gen is not None: + args.patches_dir = args.gen + main() + +# vim: sw=4 et diff --git a/packaging/pkglib.py b/packaging/pkglib.py new file mode 100644 index 00000000..b640d734 --- /dev/null +++ b/packaging/pkglib.py @@ -0,0 +1,261 @@ +import os, sys, re, subprocess + +# This python3 library provides a few helpful routines that are +# used by the latest packaging scripts. + +default_encoding = 'utf-8' + +# Output the msg args to stderr. Accepts all the args that print() accepts. +def warn(*msg): + print(*msg, file=sys.stderr) + + +# Output the msg args to stderr and die with a non-zero return-code. +# Accepts all the args that print() accepts. +def die(*msg): + warn(*msg) + sys.exit(1) + + +# Set this to an encoding name or set it to None to avoid the default encoding idiom. +def set_default_encoding(enc): + default_encoding = enc + + +# Set shell=True if the cmd is a string; sets a default encoding unless raw=True was specified. +def _tweak_opts(cmd, opts, **maybe_set): + # This sets any maybe_set value that isn't already set AND creates a copy of opts for us. + opts = {**maybe_set, **opts} + + if type(cmd) == str: + opts = {'shell': True, **opts} + + want_raw = opts.pop('raw', False) + if default_encoding and not want_raw: + opts = {'encoding': default_encoding, **opts} + + capture = opts.pop('capture', None) + if capture: + if capture == 'stdout': + opts = {'stdout': subprocess.PIPE, **opts} + elif capture == 'stderr': + opts = {'stderr': subprocess.PIPE, **opts} + elif capture == 'output': + opts = {'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE, **opts} + elif capture == 'combined': + opts = {'stdout': subprocess.PIPE, 'stderr': subprocess.STDOUT, **opts} + + discard = opts.pop('discard', None) + if discard: + # We DO want to override any already set stdout|stderr values (unlike above). + if discard == 'stdout' or discard == 'output': + opts['stdout'] = subprocess.DEVNULL + if discard == 'stderr' or discard == 'output': + opts['stderr'] = subprocess.DEVNULL + + return opts + + +# This does a normal subprocess.run() with some auto-args added to make life easier. +def cmd_run(cmd, **opts): + return subprocess.run(cmd, **_tweak_opts(cmd, opts)) + + +# Like cmd_run() with a default check=True specified. +def cmd_chk(cmd, **opts): + return subprocess.run(cmd, **_tweak_opts(cmd, opts, check=True)) + + +# Capture stdout in a string and return the (output, return_code) tuple. +# Use capture='combined' opt to get both stdout and stderr together. +def cmd_txt_status(cmd, **opts): + input = opts.pop('input', None) + if input is not None: + opts['stdin'] = subprocess.PIPE + proc = subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout')) + out = proc.communicate(input=input)[0] + return (out, proc.returncode) + + +# Like cmd_txt_status() but just return the output. +def cmd_txt(cmd, **opts): + return cmd_txt_status(cmd, **opts)[0] + + +# Capture stdout in a string and return the output if the command has a 0 return code. +# Otherwise it throws an exception that indicates the return code and the output. +def cmd_txt_chk(cmd, **opts): + out, rc = cmd_txt_status(cmd, **opts) + if rc != 0: + cmd_err = f'Command "{cmd}" returned non-zero exit status "{rc}" and output:\n{out}' + raise Exception(cmd_err) + return out + + +# Starts a piped-output command of stdout (by default) and leaves it up to you to read +# the output and call communicate() on the returned object. +def cmd_pipe(cmd, **opts): + return subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout')) + + +# Runs a "git status" command and dies if the checkout is not clean (the +# arg fatal_unless_clean can be used to make that non-fatal. Returns a +# tuple of the current branch, the is_clean flag, and the status text. +def check_git_status(fatal_unless_clean=True, subdir='.'): + status_txt = cmd_txt_chk(f"cd '{subdir}' && git status") + is_clean = re.search(r'\nnothing to commit.+working (directory|tree) clean', status_txt) != None + + if not is_clean and fatal_unless_clean: + if subdir == '.': + subdir = '' + else: + subdir = f" *{subdir}*" + die(f"The{subdir} checkout is not clean:\n" + status_txt) + + m = re.match(r'^(?:# )?On branch (.+)\n', status_txt) + cur_branch = m[1] if m else None + + return (cur_branch, is_clean, status_txt) + + +# Calls check_git_status() on the current git checkout and (optionally) a subdir path's +# checkout. Use fatal_unless_clean to indicate if an unclean checkout is fatal or not. +# The master_branch arg indicates what branch we want both checkouts to be using, and +# if the branch is wrong the user is given the option of either switching to the right +# branch or aborting. +def check_git_state(master_branch, fatal_unless_clean=True, check_extra_dir=None): + cur_branch = check_git_status(fatal_unless_clean)[0] + branch = re.sub(r'^patch/([^/]+)/[^/]+$', r'\1', cur_branch) # change patch/BRANCH/PATCH_NAME into BRANCH + if branch != master_branch: + print(f"The checkout is not on the {master_branch} branch.") + if master_branch != 'master': + sys.exit(1) + ans = input(f"Do you want me to continue with --branch={branch}? [n] ") + if not ans or not re.match(r'^y', ans, flags=re.I): + sys.exit(1) + master_branch = branch + + if check_extra_dir and os.path.isdir(os.path.join(check_extra_dir, '.git')): + branch = check_git_status(fatal_unless_clean, check_extra_dir)[0] + if branch != master_branch: + print(f"The *{check_extra_dir}* checkout is on branch {branch}, not branch {master_branch}.") + ans = input(f"Do you want to change it to branch {master_branch}? [n] ") + if not ans or not re.match(r'^y', ans, flags=re.I): + sys.exit(1) + subdir.check_call(f"cd {check_extra_dir} && git checkout '{master_branch}'", shell=True) + + return (cur_branch, master_branch) + + +# Return the git hash of the most recent commit. +def latest_git_hash(branch): + out = cmd_txt_chk(['git', 'log', '-1', '--no-color', branch]) + m = re.search(r'^commit (\S+)', out, flags=re.M) + if not m: + die(f"Unable to determine commit hash for master branch: {branch}") + return m[1] + + +# Return a set of all branch names that have the format "patch/BASE_BRANCH/NAME" +# for the given base_branch string. Just the NAME portion is put into the set. +def get_patch_branches(base_branch): + branches = set() + proc = cmd_pipe('git branch -l'.split()) + for line in proc.stdout: + m = re.search(r' patch/([^/]+)/(.+)', line) + if m and m[1] == base_branch: + branches.add(m[2]) + proc.communicate() + return branches + + +def mandate_gensend_hook(): + hook = '.git/hooks/pre-push' + if not os.path.exists(hook): + print('Creating hook file:', hook) + cmd_chk(['./rsync', '-a', 'packaging/pre-push', hook]) + else: + out, rc = cmd_txt_status(['fgrep', 'make gensend', hook], discard='output') + if rc: + die('Please add a "make gensend" into your', hook, 'script.') + + +# Snag the GENFILES values out of the Makefile.in file and return them as a list. +def get_gen_files(): + cont_re = re.compile(r'\\\n') + + extras = [ ] + + with open('Makefile.in', 'r', encoding='utf-8') as fh: + for line in fh: + if not extras: + chk = re.sub(r'^GENFILES=', '', line) + if line == chk: + continue + line = chk + m = re.search(r'\\$', line) + line = re.sub(r'^\s+|\s*\\\n?$|\s+$', '', line) + extras += line.split() + if not m: + break + + return extras + + +def get_configure_version(): + with open('configure.ac', 'r', encoding='utf-8') as fh: + for line in fh: + m = re.match(r'^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]', line) + if m: + return m[1] + die("Unable to find AC_INIT with version in configure.ac") + + +def get_OLDNEWS_version_info(): + rel_re = re.compile(r'^\| \d{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4}\s+)?\|\s+(?P<pver>\d+)\s+\|') + last_version = last_protocol_version = None + pdate = { } + + with open('OLDNEWS.md', 'r', encoding='utf-8') as fh: + for line in fh: + if not last_version: + m = re.search(r'(\d+\.\d+\.\d+)', line) + if m: + last_version = m[1] + m = rel_re.match(line) + if m: + if m['pdate']: + pdate[m['ver']] = m['pdate'] + if m['ver'] == last_version: + last_protocol_version = m['pver'] + break + + if not last_protocol_version: + die(f"Unable to determine protocol_version for {last_version}.") + + return last_version, last_protocol_version + + +def get_protocol_versions(): + protocol_version = subprotocol_version = None + + with open('rsync.h', 'r', encoding='utf-8') as fh: + for line in fh: + m = re.match(r'^#define\s+PROTOCOL_VERSION\s+(\d+)', line) + if m: + protocol_version = m[1] + continue + m = re.match(r'^#define\s+SUBPROTOCOL_VERSION\s+(\d+)', line) + if m: + subprotocol_version = m[1] + break + + if not protocol_version: + die("Unable to determine the current PROTOCOL_VERSION.") + + if not subprotocol_version: + die("Unable to determine the current SUBPROTOCOL_VERSION.") + + return protocol_version, subprotocol_version + +# vim: sw=4 et diff --git a/packaging/pre-push b/packaging/pre-push new file mode 100755 index 00000000..04b0d20b --- /dev/null +++ b/packaging/pre-push @@ -0,0 +1,3 @@ +#!/bin/sh +cat >/dev/null # Just discard stdin data +make gensend diff --git a/packaging/release-rsync b/packaging/release-rsync index 94818831..1fd3bbe3 100755 --- a/packaging/release-rsync +++ b/packaging/release-rsync @@ -1,421 +1,394 @@ -#!/usr/bin/perl +#!/usr/bin/env -S python3 -B + # This script expects the directory ~/samba-rsync-ftp to exist and to be a # copy of the /home/ftp/pub/rsync dir on samba.org. When the script is done, # the git repository in the current directory will be updated, and the local # ~/samba-rsync-ftp dir will be ready to be rsynced to samba.org. -use strict; -use warnings; -use Cwd; -use Getopt::Long; -use Term::ReadKey; -use Date::Format; - -my $dest = $ENV{HOME} . '/samba-rsync-ftp'; -my $passfile = $ENV{HOME} . '/.rsyncpass'; -my $path = $ENV{PATH}; -my $make_gen_cmd = 'make -f prepare-source.mak conf && ./config.status && make gen'; - -&Getopt::Long::Configure('bundling'); -&usage if !&GetOptions( - 'branch|b=s' => \( my $master_branch = 'master' ), - 'help|h' => \( my $help_opt ), -); -&usage if $help_opt; - -my $now = time; -my $cl_today = time2str('* %a %b %d %Y', $now); -my $year = time2str('%Y', $now); -my $ztoday = time2str('%d %b %Y', $now); -(my $today = $ztoday) =~ s/^0//; - -my $curdir = Cwd::cwd; - -END { - unlink($passfile); -} - -my @extra_files; -open(IN, '<', 'Makefile.in') or die "Couldn't open Makefile.in: $!\n"; -while (<IN>) { - if (s/^GENFILES=//) { - while (s/\\$//) { - $_ .= <IN>; - } - @extra_files = split(' ', $_); - last; - } -} -close IN; - -my $break = <<EOT; -========================================================================== -EOT - -print $break, <<EOT, $break, "\n"; +import os, sys, re, argparse, glob, shutil, signal +from datetime import datetime +from getpass import getpass + +sys.path = ['packaging'] + sys.path + +from pkglib import * + +dest = os.environ['HOME'] + '/samba-rsync-ftp' +ORIGINAL_PATH = os.environ['PATH'] + +def main(): + now = datetime.now() + cl_today = now.strftime('* %a %b %d %Y') + year = now.strftime('%Y') + ztoday = now.strftime('%d %b %Y') + today = ztoday.lstrip('0') + + mandate_gensend_hook() + + curdir = os.getcwd() + + signal.signal(signal.SIGINT, signal_handler) + + gen_files = get_gen_files() + + dash_line = '=' * 74 + + print(f"""\ +{dash_line} == This will release a new version of rsync onto an unsuspecting world. == -EOT - -die "$dest does not exist\n" unless -d $dest; -die "There is no .git dir in the current directory.\n" unless -d '.git'; -die "'a' must not exist in the current directory.\n" if -e 'a'; -die "'b' must not exist in the current directory.\n" if -e 'b'; - -require 'packaging/git-status.pl'; -check_git_state($master_branch, 1, 1); - -my $confversion; -open(IN, '<', 'configure.ac') or die $!; -while (<IN>) { - if (/^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]/) { - $confversion = $1; - last; - } -} -close IN; -die "Unable to find AC_INIT with version in configure.ac\n" unless defined $confversion; - -open(IN, '<', 'OLDNEWS') or die $!; -$_ = <IN>; -my($lastversion) = /(\d+\.\d+\.\d+)/; -my($last_protocol_version, %pdate); -while (<IN>) { - if (my($ver,$pdate,$pver) = /^\s+\S\S\s\S\S\S\s\d\d\d\d\s+(\d+\.\d+\.\d+)\s+(\d\d \w\w\w \d\d\d\d\s+)?(\d+)$/) { - $pdate{$ver} = $pdate if defined $pdate; - $last_protocol_version = $pver if $ver eq $lastversion; - } -} -close IN; -die "Unable to determine protocol_version for $lastversion.\n" unless defined $last_protocol_version; - -my $protocol_version; -open(IN, '<', 'rsync.h') or die $!; -while (<IN>) { - if (/^#define\s+PROTOCOL_VERSION\s+(\d+)/) { - $protocol_version = $1; - last; - } -} -close IN; -die "Unable to determine the current PROTOCOL_VERSION.\n" unless defined $protocol_version; - -my $version = $confversion; -$version =~ s/dev/pre1/ || $version =~ s/pre(\d+)/ 'pre' . ($1 + 1) /e; - -print "Please enter the version number of this release: [$version] "; -chomp($_ = <STDIN>); -if ($_ eq '.') { - $version =~ s/pre\d+//; -} elsif ($_ ne '') { - $version = $_; -} -die "Invalid version: `$version'\n" unless $version =~ /^[\d.]+(pre\d+)?$/; - -if (`git tag -l v$version` ne '') { - print "Tag v$version already exists.\n\nDelete tag or quit? [q/del] "; - $_ = <STDIN>; - exit 1 unless /^del/i; - system "git tag -d v$version"; -} - -if ($version =~ s/[-.]*pre[-.]*/pre/ && $confversion !~ /dev$/) { - $lastversion = $confversion; -} - -print "Enter the previous version to produce a patch against: [$lastversion] "; -chomp($_ = <STDIN>); -$lastversion = $_ if $_ ne ''; -$lastversion =~ s/[-.]*pre[-.]*/pre/; - -my $pre = $version =~ /(pre\d+)/ ? $1 : ''; - -my $release = $pre ? '0.1' : '1'; -print "Please enter the RPM release number of this release: [$release] "; -chomp($_ = <STDIN>); -$release = $_ if $_ ne ''; -$release .= ".$pre" if $pre; - -(my $finalversion = $version) =~ s/pre\d+//; -my($proto_changed,$proto_change_date); -if ($protocol_version eq $last_protocol_version) { - $proto_changed = 'unchanged'; - $proto_change_date = "\t\t"; -} else { - $proto_changed = 'changed'; - if (!defined($proto_change_date = $pdate{$finalversion})) { - while (1) { - print "On what date did the protocol change to $protocol_version get checked in? (dd Mmm yyyy) "; - chomp($_ = <STDIN>); - last if /^\d\d \w\w\w \d\d\d\d$/; - } - $proto_change_date = "$_\t"; - } -} - -my($srcdir,$srcdiffdir,$lastsrcdir,$skipping); -if ($lastversion =~ /pre/) { - if (!$pre) { - die "You should not diff a release version against a pre-release version.\n"; - } - $srcdir = $srcdiffdir = $lastsrcdir = 'src-previews'; - $skipping = ' ** SKIPPING **'; -} elsif ($pre) { - $srcdir = $srcdiffdir = 'src-previews'; - $lastsrcdir = 'src'; - $skipping = ' ** SKIPPING **'; -} else { - $srcdir = $lastsrcdir = 'src'; - $srcdiffdir = 'src-diffs'; - $skipping = ''; -} - -print "\n", $break, <<EOT; -\$version is "$version" -\$lastversion is "$lastversion" -\$dest is "$dest" -\$curdir is "$curdir" -\$srcdir is "$srcdir" -\$srcdiffdir is "$srcdiffdir" -\$lastsrcdir is "$lastsrcdir" -\$release is "$release" +{dash_line} +""") + + if not os.path.isdir(dest): + die(dest, "dest does not exist") + if not os.path.isdir('.git'): + die("There is no .git dir in the current directory.") + if os.path.lexists('a'): + die('"a" must not exist in the current directory.') + if os.path.lexists('b'): + die('"b" must not exist in the current directory.') + if os.path.lexists('patches.gen'): + die('"patches.gen" must not exist in the current directory.') + + check_git_state(args.master_branch, True, 'patches') + + confversion = get_configure_version() + + # All version values are strings! + lastversion, last_protocol_version = get_OLDNEWS_version_info() + protocol_version, subprotocol_version = get_protocol_versions() + + version = confversion + m = re.search(r'pre(\d+)', version) + if m: + version = re.sub(r'pre\d+', 'pre' + str(int(m[1]) + 1), version) + else: + version = version.replace('dev', 'pre1') + + ans = input(f"Please enter the version number of this release: [{version}] ") + if ans == '.': + version = re.sub(r'pre\d+', '', version) + elif ans != '': + version = ans + if not re.match(r'^[\d.]+(pre\d+)?$', version): + die(f'Invalid version: "{version}"') + + v_ver = 'v' + version + rsync_ver = 'rsync-' + version + + if os.path.lexists(rsync_ver): + die(f'"{rsync_ver}" must not exist in the current directory.') + + out = cmd_txt_chk(['git', 'tag', '-l', v_ver]) + if out != '': + print(f"Tag {v_ver} already exists.") + ans = input("\nDelete tag or quit? [Q/del] ") + if not re.match(r'^del', ans, flags=re.I): + die("Aborted") + cmd_chk(['git', 'tag', '-d', v_ver]) + + version = re.sub(r'[-.]*pre[-.]*', 'pre', version) + if 'pre' in version and not confversion.endswith('dev'): + lastversion = confversion + + ans = input(f"Enter the previous version to produce a patch against: [{lastversion}] ") + if ans != '': + lastversion = ans + lastversion = re.sub(r'[-.]*pre[-.]*', 'pre', lastversion) + + rsync_lastver = 'rsync-' + lastversion + if os.path.lexists(rsync_lastver): + die(f'"{rsync_lastver}" must not exist in the current directory.') + + m = re.search(r'(pre\d+)', version) + pre = m[1] if m else '' + + release = '0.1' if pre else '1' + ans = input(f"Please enter the RPM release number of this release: [{release}] ") + if ans != '': + release = ans + if pre: + release += '.' + pre + + finalversion = re.sub(r'pre\d+', '', version) + if protocol_version == last_protocol_version: + proto_changed = 'unchanged' + proto_change_date = ' ' * 11 + else: + proto_changed = 'changed' + if finalversion in pdate: + proto_change_date = pdate[finalversion] + else: + while True: + ans = input("On what date did the protocol change to {protocol_version} get checked in? (dd Mmm yyyy) ") + if re.match(r'^\d\d \w\w\w \d\d\d\d$', ans): + break + proto_change_date = ans + + if 'pre' in lastversion: + if not pre: + die("You should not diff a release version against a pre-release version.") + srcdir = srcdiffdir = lastsrcdir = 'src-previews' + skipping = ' ** SKIPPING **' + elif pre: + srcdir = srcdiffdir = 'src-previews' + lastsrcdir = 'src' + skipping = ' ** SKIPPING **' + else: + srcdir = lastsrcdir = 'src' + srcdiffdir = 'src-diffs' + skipping = '' + + print(f""" +{dash_line} +version is "{version}" +lastversion is "{lastversion}" +dest is "{dest}" +curdir is "{curdir}" +srcdir is "{srcdir}" +srcdiffdir is "{srcdiffdir}" +lastsrcdir is "{lastsrcdir}" +release is "{release}" About to: - tweak SUBPROTOCOL_VERSION in rsync.h, if needed - tweak the version in configure.ac and the spec files - - tweak NEWS and OLDNEWS to ensure header values are correct - - tweak the date in the *.yo files and generate the manpages + - tweak NEWS.md and OLDNEWS.md to ensure header values are correct - generate configure.sh, config.h.in, and proto.h - page through the differences - -EOT -print "<Press Enter to continue> "; -$_ = <STDIN>; - -my %specvars = ( 'Version:' => $finalversion, 'Release:' => $release, - '%define fullversion' => "\%{version}$pre", 'Released' => "$version.", - '%define srcdir' => $srcdir ); -my @tweak_files = ( glob('packaging/*.spec'), glob('packaging/*/*.spec'), glob('*.yo'), - qw( configure.ac rsync.h NEWS OLDNEWS options.c ) ); - -foreach my $fn (@tweak_files) { - open(IN, '<', $fn) or die $!; - undef $/; $_ = <IN>; $/ = "\n"; - close IN; - if ($fn =~ /configure/) { - s/^(AC_INIT\(\[rsync\],\s*\[)\d.+?(\])/$1$version$2/m - or die "Unable to update AC_INIT with version in $fn\n"; - } elsif ($fn =~ /\.spec/) { - while (my($str, $val) = each %specvars) { - s/^\Q$str\E .*/$str $val/m - or die "Unable to update $str in $fn\n"; - } - s/^\* \w\w\w \w\w\w \d\d \d\d\d\d (.*)/$cl_today $1/m - or die "Unable to update ChangeLog header in $fn\n"; - } elsif ($fn =~ /\.yo/) { - s/^(manpage\([^)]+\)\(\d+\)\()[^)]+(\).*)/$1$today$2/m - or die "Unable to update date in manpage() header in $fn\n"; - s/^(This man ?page is current for version) \S+ (of rsync)/$1 $version $2/m - or die "Unable to update current version info in $fn\n"; - } elsif ($fn eq 'rsync.h') { - s{(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)} - { $1 . ' ' . get_subprotocol_version($2) }e - or die "Unable to find SUBPROTOCOL_VERSION define in $fn\n"; - } elsif ($fn eq 'NEWS') { - s{^(NEWS for rsync \Q$finalversion\E )(\(UNRELEASED\))\s*(\nProtocol: )(\d+) (\([^)]+\))\n} - { $1 . ($pre ? $2 : "($today)") . "$3$protocol_version ($proto_changed)\n" }ei - or die "The first 2 lines of $fn are not in the right format. They must be:\n" - . "NEWS for rsync $finalversion (UNRELEASED)\n" - . "Protocol: $protocol_version ($proto_changed)\n"; - } elsif ($fn eq 'OLDNEWS') { - s{^(\t\S\S\s\S\S\S\s\d\d\d\d)(\t\Q$finalversion\E\t).*} - { ($pre ? $1 : "\t$ztoday") . $2 . $proto_change_date . $protocol_version }em - or die "Unable to find \"?? ??? $year\t$finalversion\" line in $fn\n"; - } elsif ($fn eq 'options.c') { - if (s/(Copyright \(C\) 2002-)(\d+)( Wayne Davison)/$1$year$3/ - && $2 ne $year) { - die "Copyright comments need to be updated to $year in all files!\n"; - } - # Adjust the year in the --version output. - s/(rprintf\(f, "Copyright \(C\) 1996-)(\d+)/$1$year/ - or die "Unable to find Copyright string in --version output of $fn\n"; - next if $2 eq $year; - } else { - die "Unrecognized file in \@tweak_files: $fn\n"; - } - open(OUT, '>', $fn) or die $!; - print OUT $_; - close OUT; -} - -print $break; -system "git diff --color | less -p '^diff .*'"; - -my $srctar_name = "rsync-$version.tar.gz"; -my $pattar_name = "rsync-patches-$version.tar.gz"; -my $diff_name = "rsync-$lastversion-$version.diffs.gz"; -my $srctar_file = "$dest/$srcdir/$srctar_name"; -my $pattar_file = "$dest/$srcdir/$pattar_name"; -my $diff_file = "$dest/$srcdiffdir/$diff_name"; -my $news_file = "$dest/$srcdir/rsync-$version-NEWS"; -my $lasttar_file = "$dest/$lastsrcdir/rsync-$lastversion.tar.gz"; - -print $break, <<EOT; +""") + ans = input("<Press Enter to continue> ") + + specvars = { + 'Version:': finalversion, + 'Release:': release, + '%define fullversion': f'%{{version}}{pre}', + 'Released': version + '.', + '%define srcdir': srcdir, + } + + tweak_files = 'configure.ac rsync.h NEWS.md OLDNEWS.md'.split() + tweak_files += glob.glob('packaging/*.spec') + tweak_files += glob.glob('packaging/*/*.spec') + + for fn in tweak_files: + with open(fn, 'r', encoding='utf-8') as fh: + old_txt = txt = fh.read() + if 'configure' in fn: + x_re = re.compile(r'^(AC_INIT\(\[rsync\],\s*\[)\d.+?(\])', re.M) + txt = replace_or_die(x_re, r'\g<1>%s\2' % version, txt, f"Unable to update AC_INIT with version in {fn}") + elif '.spec' in fn: + for var, val in specvars.items(): + x_re = re.compile(r'^%s .*' % re.escape(var), re.M) + txt = replace_or_die(x_re, var + ' ' + val, txt, f"Unable to update {var} in {fn}") + x_re = re.compile(r'^\* \w\w\w \w\w\w \d\d \d\d\d\d (.*)', re.M) + txt = replace_or_die(x_re, r'%s \1' % cl_today, txt, f"Unable to update ChangeLog header in {fn}") + elif fn == 'rsync.h': + x_re = re.compile('(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)') + repl = lambda m: m[1] + ' ' + '0' if not pre or proto_changed != 'changed' else 1 if m[2] == '0' else m[2] + txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}") + elif fn == 'NEWS.md': + x_re = re.compile( + r'^(# NEWS for rsync %s )(\(UNRELEASED\))\s*(\n\nProtocol: )(\d+) (\([^)]+\))\n' % re.escape(finalversion), + re.I) + repl = lambda m: m[1] + (m[2] if pre else f"({today})") + m[3] + f"{protocol_version} ({proto_changed})\n" + msg = (f"The first 3 lines of {fn} are not in the right format. They must be:\n" + + f"# NEWS for rsync {finalversion} (UNRELEASED)\n\n" + + f"Protocol: {protocol_version} ({proto_changed})") + txt = replace_or_die(x_re, repl, txt, msg) + elif fn == 'OLDNEWS.md': + efv = re.escape(finalversion) + x_re = re.compile(r'^(\| )(\S{2} \S{3} \d{4})(\s+\|\s+%s\s+\| ).{11}(\s+\| )\S{2}(\s+\|+)$' % efv, re.M) + repl = lambda m: m[1] + (m[2] if pre else ztoday) + m[3] + proto_change_date + m[4] + protocol_version + m[5] + txt = replace_or_die(x_re, repl, txt, f'Unable to find "| ?? ??? {year} | {finalversion} | ... |" line in {fn}') + else: + die(f"Unrecognized file in tweak_files: {fn}") + + if txt != old_txt: + print(f"Updating {fn}") + with open(fn, 'w', encoding='utf-8') as fh: + fh.write(txt) + + cmd_chk(['packaging/year-tweak']) + + print(dash_line) + cmd_run("git diff --color | less -p '^diff .*'") + + srctar_name = f"{rsync_ver}.tar.gz" + pattar_name = f"rsync-patches-{version}.tar.gz" + diff_name = f"{rsync_lastver}-{version}.diffs.gz" + srctar_file = f"{dest}/{srcdir}/{srctar_name}" + pattar_file = f"{dest}/{srcdir}/{pattar_name}" + diff_file = f"{dest}/{srcdiffdir}/{diff_name}" + news_file = f"{dest}/{srcdir}/{rsync_ver}-NEWS.md" + lasttar_file = f"{dest}/{lastsrcdir}/{rsync_lastver}.tar.gz" + + print(f"""\ +{dash_line} About to: - - commit all version changes - - merge the $master_branch branch into the patch/$master_branch/* branches + - git commit all changes + - generate the manpages + - merge the {args.master_branch} branch into the patch/{args.master_branch}/* branches - update the files in the "patches" dir and OPTIONALLY (if you type 'y') to launch a shell for each patch +""") + ans = input("<Press Enter OR 'y' to continue> ") + + s = cmd_run(['git', 'commit', '-a', '-m', f'Preparing for release of {version}']) + if s.returncode: + die('Aborting') + + cmd_chk('make reconfigure ; make gen') + cmd_chk(['rsync', '-a', *gen_files, 'SaVeDiR/']) -EOT -print "<Press Enter OR 'y' to continue> "; -my $ans = <STDIN>; + print(f'Creating any missing patch branches.') + s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing') + if s.returncode: + die('Aborting') -system "git commit -a -m 'Preparing for release of $version'" and exit 1; + print('Updating files in "patches" dir ...') + s = cmd_run(f'packaging/patch-update --branch={args.master_branch}') + if s.returncode: + die('Aborting') -print "Updating files in \"patches\" dir ...\n"; -system "packaging/patch-update --branch=$master_branch"; + if re.match(r'^y', ans, re.I): + print(f'\nVisiting all "patch/{args.master_branch}/*" branches ...') + cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --shell") -if ($ans =~ /^y/i) { - print "\nVisiting all \"patch/$master_branch/*\" branches ...\n"; - system "packaging/patch-update --branch=$master_branch --skip-check --shell"; -} + cmd_run("rm -f *.[o15] *.html") + cmd_chk('rsync -a SaVeDiR/ .'.split()) -if (-d 'patches/.git') { - system "cd patches && git commit -a -m 'The patches for $version.'" and exit 1; -} + if os.path.isdir('patches/.git'): + s = cmd_run(f"cd patches && git commit -a -m 'The patches for {version}.'") + if s.returncode: + die('Aborting') -print $break, <<EOT; + print(f"""\ +{dash_line} About to: - - create signed tag for this release: v$version - - create release diffs, "$diff_name" - - create release tar, "$srctar_name" - - generate rsync-$version/patches/* files - - create patches tar, "$pattar_name" - - update top-level README, *NEWS, TODO, and ChangeLog + - create signed tag for this release: {v_ver} + - create release diffs, "{diff_name}" + - create release tar, "{srctar_name}" + - generate {rsync_ver}/patches/* files + - create patches tar, "{pattar_name}" + - update top-level README.md, *NEWS.md, TODO, and ChangeLog - update top-level rsync*.html manpages - gpg-sign the release files - - update hard-linked top-level release files$skipping - -EOT -print "<Press Enter to continue> "; -$_ = <STDIN>; - -# We want to use our passphrase-providing "gpg" script, so modify the PATH. -$ENV{PATH} = "$curdir/packaging/bin:$path"; - -my $passphrase; -while (1) { - ReadMode('noecho'); - print "\nEnter your GPG pass-phrase: "; - chomp($passphrase = <STDIN>); - ReadMode(0); - print "\n"; - - # Briefly create a temp file with the passphrase for git's tagging use. - my $oldmask = umask 077; - unlink($passfile); - open(OUT, '>', $passfile) or die $!; - print OUT $passphrase, "\n"; - close OUT; - umask $oldmask; - $ENV{'GPG_PASSFILE'} = $passfile; - - $_ = `git tag -s -m 'Version $version.' v$version 2>&1`; - print $_; - next if /bad passphrase/; - exit 1 if /failed/; - - if (-d 'patches/.git') { - $_ = `cd patches && git tag -s -m 'Version $version.' v$version 2>&1`; - print $_; - exit 1 if /bad passphrase|failed/; - } - - unlink($passfile); - last; -} - -$ENV{PATH} = $path; - -# Extract the generated files from the old tar. -@_ = @extra_files; -map { s#^#rsync-$lastversion/# } @_; -system "tar xzf $lasttar_file @_"; -rename("rsync-$lastversion", 'a'); - -print "Creating $diff_file ...\n"; -system "$make_gen_cmd && rsync -a @extra_files b/" and exit 1; -my $sed_script = 's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:'; -system "(git diff v$lastversion v$version; diff -upN a b | sed -r '$sed_script') | gzip -9 >$diff_file"; -system "rm -rf a"; -rename('b', "rsync-$version"); - -print "Creating $srctar_file ...\n"; -system "git archive --format=tar --prefix=rsync-$version/ v$version | tar xf -"; -system "support/git-set-file-times --prefix=rsync-$version/"; -system "fakeroot tar czf $srctar_file rsync-$version; rm -rf rsync-$version"; - -print "Updating files in \"rsync-$version/patches\" dir ...\n"; -mkdir("rsync-$version", 0755); -mkdir("rsync-$version/patches", 0755); -system "packaging/patch-update --skip-check --branch=$master_branch --gen=rsync-$version/patches"; - -print "Creating $pattar_file ...\n"; -system "fakeroot tar chzf $pattar_file rsync-$version/patches; rm -rf rsync-$version"; - -print "Updating the other files in $dest ...\n"; -system "rsync -a README NEWS OLDNEWS TODO $dest"; -unlink($news_file); -link("$dest/NEWS", $news_file); -system "git log --name-status | gzip -9 >$dest/ChangeLog.gz"; - -system "yodl2html -o $dest/rsync.html rsync.yo"; -system "yodl2html -o $dest/rsyncd.conf.html rsyncd.conf.yo"; - -foreach my $fn ($srctar_file, $pattar_file, $diff_file) { - unlink("$fn.asc"); - open(GPG, '|-', "gpg --batch --passphrase-fd=0 -ba $fn") or die $!; - print GPG $passphrase, "\n"; - close GPG; -} - -if (!$pre) { - system "rm $dest/rsync-*.gz $dest/rsync-*.asc $dest/rsync-*-NEWS $dest/src-previews/rsync-*diffs.gz*"; - - foreach my $fn ($srctar_file, "$srctar_file.asc", - $pattar_file, "$pattar_file.asc", - $diff_file, "$diff_file.asc", $news_file) { - (my $top_fn = $fn) =~ s#/src(-\w+)?/#/#; - link($fn, $top_fn); - } -} - -print $break, <<'EOT'; + - update hard-linked top-level release files{skipping} +""") + ans = input("<Press Enter to continue> ") + + # TODO: is there a better way to ensure that our passphrase is in the agent? + cmd_run("touch TeMp; gpg --sign TeMp; rm TeMp*") + + out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined') + print(out, end='') + if 'bad passphrase' in out or 'failed' in out: + die('Aborting') + + if os.path.isdir('patches/.git'): + out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined') + print(out, end='') + if 'bad passphrase' in out or 'failed' in out: + die('Aborting') + + os.environ['PATH'] = ORIGINAL_PATH + + # Extract the generated files from the old tar. + tweaked_gen_files = [ f"{rsync_lastver}/{x}" for x in gen_files ] + cmd_run(['tar', 'xzf', lasttar_file, *tweaked_gen_files]) + os.rename(rsync_lastver, 'a') + + print(f"Creating {diff_file} ...") + cmd_chk(['rsync', '-a', *gen_files, 'b/']) + + sed_script = r's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:' # CAUTION: must not contain any single quotes! + cmd_chk(f"(git diff v{lastversion} {v_ver} -- ':!.github'; diff -upN a b | sed -r '{sed_script}') | gzip -9 >{diff_file}") + shutil.rmtree('a') + os.rename('b', rsync_ver) + + print(f"Creating {srctar_file} ...") + cmd_chk(f"git archive --format=tar --prefix={rsync_ver}/ {v_ver} | tar xf -") + cmd_chk(f"support/git-set-file-times --quiet --prefix={rsync_ver}/") + cmd_chk(['fakeroot', 'tar', 'czf', srctar_file, '--exclude=.github', rsync_ver]) + shutil.rmtree(rsync_ver) + + print(f'Updating files in "{rsync_ver}/patches" dir ...') + os.mkdir(rsync_ver, 0o755) + os.mkdir(f"{rsync_ver}/patches", 0o755) + cmd_chk(f"packaging/patch-update --skip-check --branch={args.master_branch} --gen={rsync_ver}/patches".split()) + + cmd_run("rm -f *.[o15] *.html") + cmd_chk('rsync -a SaVeDiR/ .'.split()) + shutil.rmtree('SaVeDiR') + cmd_chk('make gen'.split()) + + print(f"Creating {pattar_file} ...") + cmd_chk(['fakeroot', 'tar', 'chzf', pattar_file, rsync_ver + '/patches']) + shutil.rmtree(rsync_ver) + + print(f"Updating the other files in {dest} ...") + md_files = 'README.md NEWS.md OLDNEWS.md'.split() + html_files = [ fn for fn in gen_files if fn.endswith('.html') ] + cmd_chk(['rsync', '-a', *md_files, *html_files, dest]) + cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ]) + + for topfn, verfn in (('NEWS.md', news_file), ('NEWS.html', news_file.replace('.md', '.html'))): + topfn = dest + '/' + topfn + if os.path.lexists(verfn): + os.unlink(verfn) + os.link(topfn, verfn) + + cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz") + + for fn in (srctar_file, pattar_file, diff_file): + asc_fn = fn + '.asc' + if os.path.lexists(asc_fn): + os.unlink(asc_fn) + res = cmd_run(['gpg', '--batch', '-ba', fn]) + if res.returncode != 0 and res.returncode != 2: + die("gpg signing failed") + + if not pre: + for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/rsync-*-NEWS.md {dest}/src-previews/rsync-*diffs.gz*'.split(): + for fn in glob.glob(find): + os.unlink(fn) + top_link = [ + srctar_file, f"{srctar_file}.asc", + pattar_file, f"{pattar_file}.asc", + diff_file, f"{diff_file}.asc", + news_file, + ] + for fn in top_link: + os.link(fn, re.sub(r'/src(-\w+)?/', '/', fn)) + + print(f"""\ +{dash_line} Local changes are done. When you're satisfied, push the git repository and rsync the release files. Remember to announce the release on *BOTH* rsync-announce@lists.samba.org and rsync@lists.samba.org (and the web)! -EOT - -exit; - -sub get_subprotocol_version -{ - my($subver) = @_; - if ($pre && $proto_changed eq 'changed') { - return $subver == 0 ? 1 : $subver; - } - 0; -} - -sub usage -{ - die <<EOT; -Usage: release-rsync [OPTIONS] - --b, --branch=BRANCH The branch to release (default: master) --h, --help Display this help message -EOT -} +""") + + +def replace_or_die(regex, repl, txt, die_msg): + m = regex.search(txt) + if not m: + die(die_msg) + return regex.sub(repl, txt, 1) + + +def signal_handler(sig, frame): + die("\nAborting due to SIGINT.") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Prepare a new release of rsync in the git repo & ftp dir.", add_help=False) + parser.add_argument('--branch', '-b', dest='master_branch', default='master', help="The branch to release. Default: master.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/packaging/solaris/build_pkg.sh b/packaging/solaris/build_pkg.sh index 3d982f27..29c035a0 100644 --- a/packaging/solaris/build_pkg.sh +++ b/packaging/solaris/build_pkg.sh @@ -41,7 +41,7 @@ mkdir -p $FAKE_ROOT/man/man5 cp ../../../rsync $FAKE_ROOT/bin/rsync cp ../../../rsync.1 $FAKE_ROOT/man/man1/rsync.1 cp ../../../rsyncd.conf.5 $FAKE_ROOT/man/man5/rsyncd.conf.5 -cp ../../../README $FAKE_ROOT/doc/rsync/README +cp ../../../README.md $FAKE_ROOT/doc/rsync/README.md cp ../../../COPYING $FAKE_ROOT/doc/rsync/COPYING cp ../../../tech_report.pdf $FAKE_ROOT/doc/rsync/tech_report.pdf cp ../../../COPYING $FAKE_ROOT/COPYING @@ -68,7 +68,7 @@ d none bin 0755 bin bin f none bin/rsync 0755 bin bin d none doc 0755 bin bin d none doc/$NAME 0755 bin bin -f none doc/$NAME/README 0644 bin bin +f none doc/$NAME/README.md 0644 bin bin f none doc/$NAME/COPYING 0644 bin bin f none doc/$NAME/tech_report.pdf 0644 bin bin d none man 0755 bin bin diff --git a/packaging/systemd/rsync.service b/packaging/systemd/rsync.service index f64d2b6b..6d1031fd 100644 --- a/packaging/systemd/rsync.service +++ b/packaging/systemd/rsync.service @@ -1,9 +1,30 @@ [Unit] Description=fast remote file copy program daemon ConditionPathExists=/etc/rsyncd.conf +After=network.target [Service] ExecStart=/usr/bin/rsync --daemon --no-detach +RestartSec=1 + +# Citing README.md: +# +# [...] Using ssh is recommended for its security features. +# +# Alternatively, rsync can run in `daemon' mode, listening on a socket. +# This is generally used for public file distribution, [...] +# +# So let's assume some extra security is more than welcome here. We do full +# system protection (which makes it read-only) and hide users' homes and +# devices. To override these defaults, it's best to do so in the drop-in +# directory, often done via `systemctl edit rsync.service`. The file needs +# just the bare minimum of the right [heading] and override values. +# See systemd.unit(5) and search for "drop-in" for full details. + +ProtectSystem=full +ProtectHome=on +PrivateDevices=on +NoNewPrivileges=on [Install] WantedBy=multi-user.target diff --git a/packaging/systemd/rsync.socket b/packaging/systemd/rsync.socket new file mode 100644 index 00000000..5bceefe9 --- /dev/null +++ b/packaging/systemd/rsync.socket @@ -0,0 +1,10 @@ +[Unit] +Description=socket for fast remote file copy program daemon +Conflicts=rsync.service + +[Socket] +ListenStream=873 +Accept=true + +[Install] +WantedBy=sockets.target diff --git a/packaging/systemd/rsync@.service b/packaging/systemd/rsync@.service new file mode 100644 index 00000000..e1f48da8 --- /dev/null +++ b/packaging/systemd/rsync@.service @@ -0,0 +1,28 @@ +[Unit] +Description=fast remote file copy program daemon +After=network.target + +[Service] +ExecStart=-/usr/bin/rsync --daemon +StandardInput=socket +StandardOutput=inherit +StandardError=journal + +# Citing README.md: +# +# [...] Using ssh is recommended for its security features. +# +# Alternatively, rsync can run in `daemon' mode, listening on a socket. +# This is generally used for public file distribution, [...] +# +# So let's assume some extra security is more than welcome here. We do full +# system protection (which makes it read-only) and hide users' homes and +# devices. To override these defaults, it's best to do so in the drop-in +# directory, often done via `systemctl edit rsync@.service`. The file needs +# just the bare minimum of the right [heading] and override values. +# See systemd.unit(5) and search for "drop-in" for full details. + +ProtectSystem=full +ProtectHome=on +PrivateDevices=on +NoNewPrivileges=on diff --git a/packaging/var-checker b/packaging/var-checker index 7c017252..e021127c 100755 --- a/packaging/var-checker +++ b/packaging/var-checker @@ -1,83 +1,87 @@ -#!/usr/bin/perl +#!/usr/bin/env -S python3 -B + # This script checks the *.c files for extraneous "extern" variables, # for vars that are defined but not used, and for inconsistent array # sizes. Run it from inside the main rsync directory. -use strict; -use warnings; - -my %add_syscall_c = map { $_ => 1 } qw( t_stub.c t_unsafe.c tls.c trimslash.c ); -my %add_compat_c = map { $_ => 1 } qw( t_stub.c tls.c trimslash.c wildtest.c ); -my %add_util_c = map { $_ => 1 } qw( t_stub.c t_unsafe.c ); -my %sizes; - -open(IN, '<', 'syscall.c') or die $!; -undef $/; my $syscall_c = <IN>; $/ = "\n"; -close IN; -$syscall_c =~ s/^extern\s.*//mg; - -open(IN, '<', 'lib/compat.c') or die $!; -undef $/; my $compat_c = <IN>; $/ = "\n"; -close IN; -$compat_c =~ s/^extern\s.*//mg; - -open(IN, '<', 'util.c') or die $!; -undef $/; my $util_c = <IN>; $/ = "\n"; -close IN; -$util_c =~ s/^extern\s.*//mg; - -my @files = glob('*.c'); - -foreach my $fn (@files) { - open(IN, '<', $fn) or die $!; - undef $/; $_ = <IN>; $/ = "\n"; - close IN; - - my @vars = /^(?!(?:extern|enum)\s)([a-zA-Z]\S*\s+.*);/mg; - my @externs = /^extern\s+(.*);/mg; - - $_ .= $syscall_c if $add_syscall_c{$fn}; - $_ .= $compat_c if $add_compat_c{$fn}; - $_ .= $util_c if $add_util_c{$fn}; - s/INFO_GTE/info_levels/g; - s/DEBUG_GTE/debug_levels/g; - - check_vars($fn, 'var', @vars); - check_vars($fn, 'extern', @externs); -} - -exit; - -# The file's contents are in $_. -sub check_vars -{ - my $fn = shift; - my $type = shift; - - foreach my $line (@_) { - $line =~ s/\s*\{.*\}//; - $line =~ s/\s*\(.*\)//; - foreach my $item (split(/\s*,\s*/, $line)) { - $item =~ s/\s*=.*//; - my $sz = $item =~ s/(\[.*?\])// ? $1 : ''; - my($var) = $item =~ /([^*\s]+)$/; - if (!defined $var) { - print "Bogus match? ($item)\n"; - next; - } - if ($sz) { - if (defined $sizes{$var}) { - if ($sizes{$var} ne $sz) { - print $fn, ' has inconsistent size for "', $var, - "\": $sizes{$var} vs $sz\n"; - } - } else { - $sizes{$var} = $sz; - } - } - my @matches = /(?<!\sstruct )\b(\Q$var\E)(?!\w)/g; - push(@matches, /(\QSIGACTION(\E)/g) if $var eq 'sigact'; - print $fn, " has extraneous $type: \"", $var, "\"\n" if @matches == 1; - } - } -} +import re, argparse, glob + +VARS_RE = re.compile(r'^(?!(?:extern|enum)\s)([a-zA-Z]\S*\s+.*);', re.M) +EXTERNS_RE = re.compile(r'^extern\s+(.*);', re.M) + +sizes = { } + +def main(): + add_syscall_c = set('t_stub.c t_unsafe.c tls.c trimslash.c'.split()) + add_util_c = set('t_stub.c t_unsafe.c'.split()) + + syscall_c = slurp_file('syscall.c', True) + util_c = slurp_file('util.c', True) + + for fn in sorted(glob.glob('*.c')): + txt = slurp_file(fn) + + var_list = parse_vars(fn, VARS_RE.findall(txt)) + extern_list = parse_vars(fn, EXTERNS_RE.findall(txt)) + if not var_list and not extern_list: + continue + + if fn in add_syscall_c: + txt += syscall_c + if fn in add_util_c: + txt += util_c + + txt = re.sub(r'INFO_GTE', 'info_levels ', txt) + txt = re.sub(r'DEBUG_GTE', 'debug_levels ', txt) + txt = re.sub(r'SIGACTION\(', 'sigact (', txt) + + find = '|'.join([ re.escape(x) for x in var_list + extern_list ]) + var_re = re.compile(r'(?<!\sstruct )\b(%s)(?!\w)' % find) + + found = { x: 0 for x in var_list + extern_list } + for var in var_re.findall(txt): + found[var] += 1 + + for var in sorted(var_list + extern_list): + if found[var] == 1: + vtype = 'var' if var in var_list else 'extern' + print(fn, f'has extraneous {vtype}: "{var}"') + + +def slurp_file(fn, drop_externs=False): + with open(fn, 'r', encoding='utf-8') as fh: + txt = fh.read() + if drop_externs: + txt = EXTERNS_RE.sub('', txt) + return txt + + +def parse_vars(fn, lines): + ret = [ ] + for line in lines: + line = re.sub(r'\s*\{.*\}', '', line) + line = re.sub(r'\s*\(.*\)', '', line) + for item in re.split(r'\s*,\s*', line): + item = re.sub(r'\s*=.*', '', item) + m = re.search(r'(?P<var>\w+)(?P<sz>\[.*?\])?$', item) + if not m: + print(f"Bogus match? ({item})") + continue + if m['sz']: + if m['var'] in sizes: + if sizes[m['var']] != m['sz']: + var = m['var'] + print(fn, f'has inconsistent size for "{var}":', m['sz'], 'vs', sizes[var]) + else: + sizes[m['var']] = m['sz'] + ret.append(m['var']) + return ret + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Check the *.c files for extraneous extern vars.', add_help=False) + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/packaging/year-tweak b/packaging/year-tweak new file mode 100755 index 00000000..69d2f2ff --- /dev/null +++ b/packaging/year-tweak @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +# This uses the output from "support/git-set-file-times --list" to discern +# the last-modified year of each *.c & *.h file and updates the copyright +# year if it isn't set right. + +import sys, os, re, argparse, subprocess +from datetime import datetime + +MAINTAINER_NAME = 'Wayne Davison' +MAINTAINER_SUF = ' ' + MAINTAINER_NAME + "\n" + +def main(): + latest_year = '2000' + + proc = subprocess.Popen('support/git-set-file-times --list'.split(), stdout=subprocess.PIPE, encoding='utf-8') + for line in proc.stdout: + m = re.match(r'^\S\s+(?P<year>\d\d\d\d)\S+\s+\S+\s+(?P<fn>.+)', line) + if not m: + print("Failed to parse line from git-set-file-times:", line) + sys.exit(1) + m = argparse.Namespace(**m.groupdict()) + if m.year > latest_year: + latest_year = m.year + if m.fn.startswith('zlib/') or m.fn.startswith('popt/'): + continue + if re.search(r'\.(c|h|sh|test)$', m.fn): + maybe_edit_copyright_year(m.fn, m.year) + proc.communicate() + + fn = 'latest-year.h' + with open(fn, 'r', encoding='utf-8') as fh: + old_txt = fh.read() + + txt = f'#define LATEST_YEAR "{latest_year}"\n' + if txt != old_txt: + print(f"Updating {fn} with year {latest_year}") + with open(fn, 'w', encoding='utf-8') as fh: + fh.write(txt) + + +def maybe_edit_copyright_year(fn, year): + opening_lines = [ ] + copyright_line = None + + with open(fn, 'r', encoding='utf-8') as fh: + for lineno, line in enumerate(fh): + opening_lines.append(line) + if lineno > 3 and not re.search(r'\S', line): + break + m = re.match(r'^(?P<pre>.*Copyright\s+\S+\s+)(?P<year>\d\d\d\d(?:-\d\d\d\d)?(,\s+\d\d\d\d)*)(?P<suf>.+)', line) + if not m: + continue + copyright_line = argparse.Namespace(**m.groupdict()) + copyright_line.lineno = len(opening_lines) + copyright_line.is_maintainer_line = MAINTAINER_NAME in copyright_line.suf + copyright_line.txt = line + if copyright_line.is_maintainer_line: + break + + if not copyright_line: + return + + if copyright_line.is_maintainer_line: + cyears = copyright_line.year.split('-') + if year == cyears[0]: + cyears = [ year ] + else: + cyears = [ cyears[0], year ] + txt = copyright_line.pre + '-'.join(cyears) + MAINTAINER_SUF + if txt == copyright_line.txt: + return + opening_lines[copyright_line.lineno - 1] = txt + else: + if fn.startswith('lib/') or fn.startswith('testsuite/'): + return + txt = copyright_line.pre + year + MAINTAINER_SUF + opening_lines[copyright_line.lineno - 1] += txt + + remaining_txt = fh.read() + + print(f"Updating {fn} with year {year}") + + with open(fn, 'w', encoding='utf-8') as fh: + fh.write(''.join(opening_lines)) + fh.write(remaining_txt) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Grab the year of last mod for our c & h files and make sure the Copyright comment is up-to-date.") + args = parser.parse_args() + main() + +# vim: sw=4 et @@ -1,5 +1,5 @@ /* This modules is based on the params.c module from Samba, written by Karl Auer - and much modifed by Christopher Hertel. */ + and much modified by Christopher Hertel. */ /* * This program is free software; you can redistribute it and/or modify @@ -59,7 +59,7 @@ * beginning with either a semicolon (';') or a pound sign ('#'). * * All whitespace in section names and parameter names is compressed - * to single spaces. Leading and trailing whitespace is stipped from + * to single spaces. Leading and trailing whitespace is stripped from * both names and values. * * Only the first equals sign in a parameter line is significant. @@ -153,7 +153,7 @@ static int EatComment( FILE *InFile ) static int Continuation( char *line, int pos ) /* ------------------------------------------------------------------------ ** - * Scan backards within a string to discover if the last non-whitespace + * Scan backwards within a string to discover if the last non-whitespace * character is a line-continuation character ('\\'). * * Input: line - A pointer to a buffer containing the string to be @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -66,10 +66,10 @@ pid_t piped_child(char **command, int *f_in, int *f_out) } if (pid == 0) { - if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 || - close(to_child_pipe[1]) < 0 || - close(from_child_pipe[0]) < 0 || - dup2(from_child_pipe[1], STDOUT_FILENO) < 0) { + if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 + || close(to_child_pipe[1]) < 0 + || close(from_child_pipe[0]) < 0 + || dup2(from_child_pipe[1], STDOUT_FILENO) < 0) { rsyserr(FERROR, errno, "Failed to dup/close"); exit_cleanup(RERR_IPC); } @@ -116,8 +116,7 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, /* The parent process is always the sender for a local rsync. */ assert(am_sender); - if (fd_pair(to_child_pipe) < 0 || - fd_pair(from_child_pipe) < 0) { + if (fd_pair(to_child_pipe) < 0 || fd_pair(from_child_pipe) < 0) { rsyserr(FERROR, errno, "pipe"); exit_cleanup(RERR_IPC); } @@ -150,10 +149,10 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, } } - if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 || - close(to_child_pipe[1]) < 0 || - close(from_child_pipe[0]) < 0 || - dup2(from_child_pipe[1], STDOUT_FILENO) < 0) { + if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 + || close(to_child_pipe[1]) < 0 + || close(from_child_pipe[0]) < 0 + || dup2(from_child_pipe[1], STDOUT_FILENO) < 0) { rsyserr(FERROR, errno, "Failed to dup/close"); exit_cleanup(RERR_IPC); } @@ -167,8 +166,7 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, child_main(argc, argv); } - if (close(from_child_pipe[1]) < 0 || - close(to_child_pipe[0]) < 0) { + if (close(from_child_pipe[1]) < 0 || close(to_child_pipe[0]) < 0) { rsyserr(FERROR, errno, "Failed to close"); exit_cleanup(RERR_IPC); } diff --git a/popt/popt.c b/popt/popt.c index ec6f3bd8..5d927f78 100644 --- a/popt/popt.c +++ b/popt/popt.c @@ -626,7 +626,7 @@ expandNextArg(/*@special@*/ poptContext con, const char * s) pos = te - t; t = realloc(t, tn); te = t + pos; - strncpy(te, a, alen); te += alen; + memcpy(te, a, alen+1); te += alen; continue; /*@notreached@*/ /*@switchbreak@*/ break; default: diff --git a/prepare-source b/prepare-source index b1e76282..3e4f9edf 100755 --- a/prepare-source +++ b/prepare-source @@ -12,8 +12,8 @@ # The script stops after the first successful action. dir=`dirname $0` -if test x"$dir" != x -a x"$dir" != x.; then - cd "$dir" +if test x"$dir" = x; then + dir=. fi if test $# = 0; then @@ -23,21 +23,20 @@ fi for action in "${@}"; do case "$action" in build|make) - make -f prepare-source.mak + (cd $dir && make -f prepare-source.mak) ;; fetch) - if perl --version >/dev/null 2>/dev/null; then - files='[ca]*' - else - files='[cap]*' + $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir + if ! perl --version >/dev/null 2>/dev/null; then + $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'p*' . fi - rsync -pvz rsync://rsync.samba.org/rsyncftp/generated-files/"$files" . ;; fetchgen) - rsync -pvz rsync://rsync.samba.org/rsyncftp/generated-files/'*' . + $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir + $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[^ca]*' . ;; fetchSRC) - rsync -pvrz --exclude=/.git/ rsync://rsync.samba.org/ftp/pub/unpacked/rsync/ . + ./rsync-ssl -iipr --no-motd --exclude=/.git/ rsync://download.samba.org/ftp/pub/unpacked/rsync/ . ;; *) echo "Unknown action: $action" diff --git a/prepare-source.mak b/prepare-source.mak index ad291feb..bc1938de 100644 --- a/prepare-source.mak +++ b/prepare-source.mak @@ -1,6 +1,6 @@ conf: configure.sh config.h.in -aclocal.m4: +aclocal.m4: m4/*.m4 aclocal -I m4 configure.sh: configure.ac aclocal.m4 @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -28,9 +28,12 @@ extern int flist_eof; extern int quiet; extern int need_unsorted_flist; extern int output_needs_newline; +extern int stdout_format_has_i; extern struct stats stats; extern struct file_list *cur_flist; +BOOL want_progress_now = False; + #define PROGRESS_HISTORY_SECS 5 #ifdef GETPGRP_VOID @@ -63,8 +66,7 @@ static unsigned long msdiff(struct timeval *t1, struct timeval *t2) * printed for this file, so we should output a newline. (Not * necessarily the same as all bytes being received.) **/ -static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, - int is_last) +static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, int is_last) { char rembuf[64], eol[128]; const char *units; @@ -89,8 +91,7 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, is_last = 0; } /* Compute stats based on the starting info. */ - if (!ph_start.time.tv_sec - || !(diff = msdiff(&ph_start.time, now))) + if (!ph_start.time.tv_sec || !(diff = msdiff(&ph_start.time, now))) diff = 1; rate = (double) (ofs - ph_start.ofs) * 1000.0 / diff / 1024.0; /* Switch to total time taken for our last update. */ @@ -100,8 +101,7 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, /* Compute stats based on recent progress. */ if (!(diff = msdiff(&ph_list[oldest_hpos].time, now))) diff = 1; - rate = (double) (ofs - ph_list[oldest_hpos].ofs) * 1000.0 - / diff / 1024.0; + rate = (double) (ofs - ph_list[oldest_hpos].ofs) * 1000.0 / diff / 1024.0; remain = rate ? (double) (size - ofs) / rate / 1000.0 : 0.0; } @@ -134,6 +134,16 @@ static void rprint_progress(OFF_T ofs, OFF_T size, struct timeval *now, } } +void progress_init(void) +{ + if (!am_server && !INFO_GTE(PROGRESS, 1)) { + struct timeval now; + gettimeofday(&now, NULL); + ph_start.time.tv_sec = now.tv_sec; + ph_start.time.tv_usec = now.tv_usec; + } +} + void set_current_file_index(struct file_struct *file, int ndx) { if (!file) @@ -145,12 +155,21 @@ void set_current_file_index(struct file_struct *file, int ndx) current_file_index -= cur_flist->flist_num; } +void instant_progress(const char *fname) +{ + /* We only get here if want_progress_now is True */ + if (!stdout_format_has_i && !INFO_GTE(NAME, 1)) + rprintf(FINFO, "%s\n", fname); + end_progress(0); + want_progress_now = False; +} + void end_progress(OFF_T size) { if (!am_server) { struct timeval now; gettimeofday(&now, NULL); - if (INFO_GTE(PROGRESS, 2)) { + if (INFO_GTE(PROGRESS, 2) || want_progress_now) { rprint_progress(stats.total_transferred_size, stats.total_size, &now, True); } else { @@ -19,9 +19,10 @@ int make_backup(const char *fname, BOOL prefer_rename); void write_stream_flags(int fd); void read_stream_flags(int fd); void check_batch_flags(void); -void write_batch_shell_file(int argc, char *argv[], int file_arg_cnt); -int parse_checksum_choice(void); +void open_batch_files(void); +void write_batch_shell_file(void); int parse_csum_name(const char *name, int len); +void parse_checksum_choice(int final_call); int csum_len_for_type(int cst, BOOL flist_csum); int canonical_checksum(int csum_type); uint32 get_checksum1(char *buf1, int32 len); @@ -41,25 +42,19 @@ void cleanup_set(const char *fnametmp, const char *fname, struct file_struct *fi int fd_r, int fd_w); void cleanup_set_pid(pid_t pid); char *client_addr(int fd); -char *client_name(int fd); -void client_sockaddr(int fd, - struct sockaddr_storage *ss, - socklen_t *ss_len); -int lookup_name(int fd, const struct sockaddr_storage *ss, - socklen_t ss_len, - char *name_buf, size_t name_buf_size, - char *port_buf, size_t port_buf_size); -int compare_addrinfo_sockaddr(const struct addrinfo *ai, - const struct sockaddr_storage *ss); -int check_name(int fd, - const struct sockaddr_storage *ss, - char *name_buf, size_t name_buf_size); +char *client_name(const char *ipaddr); +int read_proxy_protocol_header(int fd); int start_socket_client(char *host, int remote_argc, char *remote_argv[], int argc, char *argv[]); int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char *argv[]); +void set_env_num(const char *var, long num); int start_daemon(int f_in, int f_out); int daemon_main(void); void set_allow_inc_recurse(void); +void parse_compress_choice(int final_call); +struct name_num_item *get_nni_by_name(struct name_num_obj *nno, const char *name, int len); +struct name_num_item *get_nni_by_num(struct name_num_obj *nno, int num); +int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup); void setup_protocol(int f_out,int f_in); int claim_connection(char *fname, int max_connections); enum delret delete_item(char *fbuf, uint16 mode, uint16 flags); @@ -92,6 +87,7 @@ int link_stat(const char *path, STRUCT_STAT *stp, int follow_dirlinks); int change_pathname(struct file_struct *file, const char *dir, int dirlen); struct file_struct *make_file(const char *fname, struct file_list *flist, STRUCT_STAT *stp, int flags, int filter_level); +OFF_T get_device_size(int fd, const char *fname); void unmake_file(struct file_struct *file); void send_extra_file_list(int f, int at_least); struct file_list *send_file_list(int f, int argc, char *argv[]); @@ -119,7 +115,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo); void generate_files(int f_out, const char *local_name); struct hashtable *hashtable_create(int size, int key64); void hashtable_destroy(struct hashtable *tbl); -void *hashtable_find(struct hashtable *tbl, int64 key, int allocate_if_missing); +void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new); uint32_t hashlittle(const void *key, size_t length); void init_hard_links(void); struct ht_int64_node *idev_find(int64 dev, int64 ino); @@ -168,7 +164,7 @@ uchar read_byte(int f); int read_vstring(int f, char *buf, int bufsize); void read_sum_head(int f, struct sum_struct *sum); void write_sum_head(int f, struct sum_struct *sum); -void io_flush(int flush_it_all); +void io_flush(int flush_type); void write_shortint(int f, unsigned short x); void write_int(int f, int32 x); void write_varint(int f, int32 x); @@ -189,6 +185,7 @@ int io_end_multiplex_in(int mode); int io_end_multiplex_out(int mode); void start_write_batch(int fd); void stop_write_batch(void); +void reset_daemon_vars(void); char *lp_bind_address(void); char *lp_daemon_chroot(void); char *lp_daemon_gid(void); @@ -198,10 +195,12 @@ char *lp_pid_file(void); char *lp_socket_options(void); int lp_listen_backlog(void); int lp_rsync_port(void); +BOOL lp_proxy_protocol(void); char *lp_auth_users(int module_id); char *lp_charset(int module_id); char *lp_comment(int module_id); char *lp_dont_compress(int module_id); +char *lp_early_exec(int module_id); char *lp_exclude(int module_id); char *lp_exclude_from(int module_id); char *lp_filter(int module_id); @@ -255,11 +254,11 @@ void rflush(enum logcode code); void remember_initial_stats(void); int log_format_has(const char *format, char esc); void log_item(enum logcode code, struct file_struct *file, int iflags, const char *hlink); -void maybe_log_item(struct file_struct *file, int iflags, int itemizing, - const char *buf); +void maybe_log_item(struct file_struct *file, int iflags, int itemizing, const char *buf); void log_delete(const char *fname, int mode); void log_exit(int code, const char *file, int line); pid_t wait_process(pid_t pid, int *status_ptr, int flags); +int shell_exec(const char *cmd); void write_del_stats(int f); void read_del_stats(int f); int child_main(int argc, char *argv[]); @@ -275,6 +274,7 @@ void reset_output_levels(void); void negate_output_levels(void); void usage(enum logcode F); void option_error(void); +char *alt_dest_opt(int type); int parse_arguments(int *argc_p, const char ***argv_p); void server_options(char **args, int *argc_p); char *check_for_hostspec(char *s, char **host_ptr, int *port_ptr); @@ -284,7 +284,9 @@ int pm_process( char *FileName, pid_t piped_child(char **command, int *f_in, int *f_out); pid_t local_child(int argc, char **argv, int *f_in, int *f_out, int (*child_main)(int, char*[])); +void progress_init(void); void set_current_file_index(struct file_struct *file, int ndx); +void instant_progress(const char *fname); void end_progress(OFF_T size); void show_progress(OFF_T ofs, OFF_T size); int get_tmpname(char *fnametmp, const char *fname, BOOL make_unique); @@ -293,8 +295,7 @@ int recv_files(int f_in, int f_out, char *local_name); void setup_iconv(void); int iconvbufs(iconv_t ic, xbuf *in, xbuf *out, int flags); void send_protected_args(int fd, char *args[]); -int read_ndx_and_attrs(int f_in, int f_out, int *iflag_ptr, uchar *type_ptr, - char *buf, int *len_ptr); +int read_ndx_and_attrs(int f_in, int f_out, int *iflag_ptr, uchar *type_ptr, char *buf, int *len_ptr); void free_sums(struct sum_struct *s); mode_t dest_mode(mode_t flist_mode, mode_t stat_mode, int dflt_perms, int exists); @@ -311,23 +312,21 @@ void successful_send(int ndx); void send_files(int f_in, int f_out); int try_bind_local(int s, int ai_family, int ai_socktype, const char *bind_addr); -int open_socket_out(char *host, int port, const char *bind_addr, - int af_hint); -int open_socket_out_wrapped(char *host, int port, const char *bind_addr, - int af_hint); +int open_socket_out(char *host, int port, const char *bind_addr, int af_hint); +int open_socket_out_wrapped(char *host, int port, const char *bind_addr, int af_hint); int is_a_socket(int fd); void start_accept_loop(int port, int (*fn)(int, int)); void set_socket_options(int fd, char *options); int do_unlink(const char *fname); int do_symlink(const char *lnk, const char *fname); ssize_t do_readlink(const char *path, char *buf, size_t bufsiz); -int do_link(const char *fname1, const char *fname2); +int do_link(const char *old_path, const char *new_path); int do_lchown(const char *path, uid_t owner, gid_t group); int do_mknod(const char *pathname, mode_t mode, dev_t dev); int do_rmdir(const char *pathname); int do_open(const char *pathname, int flags, mode_t mode); int do_chmod(const char *path, mode_t mode); -int do_rename(const char *fname1, const char *fname2); +int do_rename(const char *old_path, const char *new_path); int do_ftruncate(int fd, OFF_T size); void trim_trailing_slashes(char *name); int do_mkdir(char *fname, mode_t mode); @@ -336,14 +335,15 @@ int do_stat(const char *fname, STRUCT_STAT *st); int do_lstat(const char *fname, STRUCT_STAT *st); int do_fstat(int fd, STRUCT_STAT *st); OFF_T do_lseek(int fd, OFF_T offset, int whence); -int do_setattrlist_times(const char *fname, time_t modtime, uint32 mod_nsec); -int do_utimensat(const char *fname, time_t modtime, uint32 mod_nsec); -int do_lutimes(const char *fname, time_t modtime, uint32 mod_nsec); -int do_utimes(const char *fname, time_t modtime, uint32 mod_nsec); -int do_utime(const char *fname, time_t modtime, UNUSED(uint32 mod_nsec)); +int do_setattrlist_times(const char *fname, STRUCT_STAT *stp); +int do_utimensat(const char *fname, STRUCT_STAT *stp); +int do_lutimes(const char *fname, STRUCT_STAT *stp); +int do_utimes(const char *fname, STRUCT_STAT *stp); +int do_utime(const char *fname, STRUCT_STAT *stp); OFF_T do_fallocate(int fd, OFF_T offset, OFF_T length); -int do_punch_hole(int fd, UNUSED(OFF_T pos), int len); +int do_punch_hole(int fd, UNUSED(OFF_T pos), OFF_T len); int do_open_nofollow(const char *pathname, int flags); +void init_compression_level(void); void set_compression(const char *fname); void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 n, int32 toklen); @@ -367,7 +367,7 @@ void set_nonblocking(int fd); void set_blocking(int fd); int fd_pair(int fd[2]); void print_child_argv(const char *prefix, char **cmd); -int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode); +int set_times(const char *fname, STRUCT_STAT *stp); int make_path(char *fname, int flags); int full_write(int desc, const char *ptr, size_t len); int copy_file(const char *source, const char *dest, int ofd, mode_t mode); @@ -385,8 +385,7 @@ size_t pathjoin(char *dest, size_t destsize, const char *p1, const char *p2); size_t stringjoin(char *dest, size_t destsize, ...); int count_dir_elements(const char *p); int clean_fname(char *name, int flags); -char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, - int flags); +char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, int flags); int change_dir(const char *dir, int set_path_only); char *normalize_path(char *path, BOOL force_newbuf, unsigned int *len_ptr); char *full_fname(const char *fn); @@ -394,7 +393,7 @@ char *partial_dir_fname(const char *fname); int handle_partial_dir(const char *fname, int create); int unsafe_symlink(const char *dest, const char *src); char *timestring(time_t t); -int cmp_time(time_t f1_sec, unsigned long f1_nsec, time_t f2_sec, unsigned long f2_nsec); +int same_time(time_t f1_sec, unsigned long f1_nsec, time_t f2_sec, unsigned long f2_nsec); int _Insure_trap_error(int a1, int a2, int a3, int a4, int a5, int a6); const char *find_filename_suffix(const char *fn, int fn_len, int *len_ptr); uint32 fuzzy_distance(const char *s1, unsigned len1, const char *s2, unsigned len2); @@ -405,8 +404,8 @@ int bitbag_check_bit(struct bitbag *bb, int ndx); int bitbag_next_bit(struct bitbag *bb, int after); void flist_ndx_push(flist_ndx_list *lp, int ndx); int flist_ndx_pop(flist_ndx_list *lp); -void *expand_item_list(item_list *lp, size_t item_size, - const char *desc, int incr); +void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int incr); +void force_memzero(void *buf, size_t len); int msleep(int t); void *_new_array(unsigned long num, unsigned int size, int use_calloc); void *_realloc_array(void *ptr, unsigned int size, size_t num); @@ -423,8 +422,7 @@ int recv_xattr_request(struct file_struct *file, int f_in); void receive_xattr(int f, struct file_struct *file); void cache_tmp_xattr(struct file_struct *file, stat_x *sxp); void uncache_tmp_xattrs(void); -int set_xattr(const char *fname, const struct file_struct *file, - const char *fnamecmp, stat_x *sxp); +int set_xattr(const char *fname, const struct file_struct *file, const char *fnamecmp, stat_x *sxp); char *get_xattr_acl(const char *fname, int is_access_acl, size_t *len_p); int set_xattr_acl(const char *fname, int is_access_acl, const char *buf, size_t buf_len); int del_def_xattr_acl(const char *fname); @@ -3,7 +3,7 @@ * * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -39,6 +39,7 @@ extern int protocol_version; extern int relative_paths; extern int preserve_hard_links; extern int preserve_perms; +extern int write_devices; extern int preserve_xattrs; extern int basis_dir_cnt; extern int make_backups; @@ -51,9 +52,11 @@ extern int keep_partial; extern int checksum_seed; extern int whole_file; extern int inplace; +extern int inplace_partial; extern int allowed_lull; extern int delay_updates; extern int xfersum_type; +extern BOOL want_progress_now; extern mode_t orig_umask; extern struct stats stats; extern char *tmpdir; @@ -67,7 +70,7 @@ extern OFF_T preallocated_len; static struct bitbag *delayed_bits = NULL; static int phase = 0, redoing = 0; static flist_ndx_list batch_redo_list; -/* We're either updating the basis file or an identical copy: */ +/* This is non-0 when we are updating the basis file or an identical copy: */ static int updating_basis_or_equiv; #define TMPNAME_SUFFIX ".XXXXXX" @@ -231,13 +234,14 @@ int open_tmpfile(char *fnametmp, const char *fname, struct file_struct *file) } static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, - const char *fname, int fd, OFF_T total_size) + const char *fname, int fd, struct file_struct *file, int inplace_sizing) { static char file_sum1[MAX_DIGEST_LEN]; struct map_struct *mapbuf; struct sum_struct sum; int sum_len; int32 len; + OFF_T total_size = F_LENGTH(file); OFF_T offset = 0; OFF_T offset2; char *data; @@ -245,14 +249,14 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, char *map = NULL; #ifdef SUPPORT_PREALLOCATION - if (preallocate_files && fd != -1 && total_size > 0 && (!inplace || total_size > size_r)) { + if (preallocate_files && fd != -1 && total_size > 0 && (!inplace_sizing || total_size > size_r)) { /* Try to preallocate enough space for file's eventual length. Can * reduce fragmentation on filesystems like ext4, xfs, and NTFS. */ if ((preallocated_len = do_fallocate(fd, 0, total_size)) < 0) rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); } else #endif - if (inplace) { + if (inplace_sizing) { #ifdef HAVE_FTRUNCATE /* The most compatible way to create a sparse file is to start with no length. */ if (sparse_files > 0 && whole_file && fd >= 0 && do_ftruncate(fd, 0) == 0) @@ -379,9 +383,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, /* inplace: New data could be shorter than old data. * preallocate_files: total_size could have been an overestimate. * Cut off any extra preallocated zeros from dest file. */ - if ((inplace || preallocated_len > offset) && fd != -1 && do_ftruncate(fd, offset) < 0) { - rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", - full_fname(fname)); + if ((inplace_sizing || preallocated_len > offset) && fd != -1 && !IS_DEVICE(file->mode)) { + if (do_ftruncate(fd, offset) < 0) + rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", full_fname(fname)); } #endif @@ -402,9 +406,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, } -static void discard_receive_data(int f_in, OFF_T length) +static void discard_receive_data(int f_in, struct file_struct *file) { - receive_data(f_in, NULL, -1, 0, NULL, -1, length); + receive_data(f_in, NULL, -1, 0, NULL, -1, file, 0); } static void handle_delayed_updates(char *local_name) @@ -515,7 +519,7 @@ int recv_files(int f_in, int f_out, char *local_name) int iflags, xlen; char *fname, fbuf[MAXPATHLEN]; char xname[MAXPATHLEN]; - char fnametmp[MAXPATHLEN]; + char *fnametmp, fnametmpbuf[MAXPATHLEN]; char *fnamecmp, *partialptr; char fnamecmpbuf[MAXPATHLEN]; uchar fnamecmp_type; @@ -527,7 +531,7 @@ int recv_files(int f_in, int f_out, char *local_name) #ifdef SUPPORT_ACLS const char *parent_dirname = ""; #endif - int ndx, recv_ok; + int ndx, recv_ok, one_inplace; if (DEBUG_GTE(RECV, 1)) rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used); @@ -535,6 +539,8 @@ int recv_files(int f_in, int f_out, char *local_name) if (delay_updates) delayed_bits = bitbag_create(cur_flist->used + 1); + progress_init(); + while (1) { cleanup_disable(); @@ -542,9 +548,10 @@ int recv_files(int f_in, int f_out, char *local_name) ndx = read_ndx_and_attrs(f_in, f_out, &iflags, &fnamecmp_type, xname, &xlen); if (ndx == NDX_DONE) { - if (!am_server && INFO_GTE(PROGRESS, 2) && cur_flist) { + if (!am_server && cur_flist) { set_current_file_index(NULL, 0); - end_progress(0); + if (INFO_GTE(PROGRESS, 2)) + end_progress(0); } if (inc_recurse && first_flist) { if (read_batch) { @@ -644,7 +651,7 @@ int recv_files(int f_in, int f_out, char *local_name) stats.created_files++; } - if (!am_server && INFO_GTE(PROGRESS, 1)) + if (!am_server) set_current_file_index(file, ndx); stats.xferred_files++; stats.total_transferred_size += F_LENGTH(file); @@ -660,7 +667,7 @@ int recv_files(int f_in, int f_out, char *local_name) "(Skipping batched update for%s \"%s\")\n", redoing ? " resend of" : "", fname); - discard_receive_data(f_in, F_LENGTH(file)); + discard_receive_data(f_in, file); file->flags |= FLAG_FILE_SENT; continue; } @@ -671,13 +678,13 @@ int recv_files(int f_in, int f_out, char *local_name) if (!do_xfers) { /* log the transfer */ log_item(FCLIENT, file, iflags, NULL); if (read_batch) - discard_receive_data(f_in, F_LENGTH(file)); + discard_receive_data(f_in, file); continue; } if (write_batch < 0) { log_item(FCLIENT, file, iflags, NULL); if (!am_server) - discard_receive_data(f_in, F_LENGTH(file)); + discard_receive_data(f_in, file); if (inc_recurse) send_msg_int(MSG_SUCCESS, ndx); continue; @@ -746,6 +753,7 @@ int recv_files(int f_in, int f_out, char *local_name) if (fd1 == -1 && protocol_version < 29) { if (fnamecmp != fname) { fnamecmp = fname; + fnamecmp_type = FNAMECMP_FNAME; fd1 = do_open(fnamecmp, O_RDONLY, 0); } @@ -754,12 +762,14 @@ int recv_files(int f_in, int f_out, char *local_name) pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, basis_dir[0], fname); fnamecmp = fnamecmpbuf; + fnamecmp_type = FNAMECMP_BASIS_DIR_LOW; fd1 = do_open(fnamecmp, O_RDONLY, 0); } } - updating_basis_or_equiv = inplace - && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP); + one_inplace = inplace_partial && fnamecmp_type == FNAMECMP_PARTIAL_DIR; + updating_basis_or_equiv = one_inplace + || (inplace && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP)); if (fd1 == -1) { st.st_mode = 0; @@ -767,7 +777,7 @@ int recv_files(int f_in, int f_out, char *local_name) } else if (do_fstat(fd1,&st) != 0) { rsyserr(FERROR_XFER, errno, "fstat %s failed", full_fname(fnamecmp)); - discard_receive_data(f_in, F_LENGTH(file)); + discard_receive_data(f_in, file); close(fd1); if (inc_recurse) send_msg_int(MSG_NO_SEND, ndx); @@ -782,18 +792,21 @@ int recv_files(int f_in, int f_out, char *local_name) */ rprintf(FERROR_XFER, "recv_files: %s is a directory\n", full_fname(fnamecmp)); - discard_receive_data(f_in, F_LENGTH(file)); + discard_receive_data(f_in, file); close(fd1); if (inc_recurse) send_msg_int(MSG_NO_SEND, ndx); continue; } - if (fd1 != -1 && !S_ISREG(st.st_mode)) { + if (fd1 != -1 && !(S_ISREG(st.st_mode) || (write_devices && IS_DEVICE(st.st_mode)))) { close(fd1); fd1 = -1; } + if (fd1 != -1 && IS_DEVICE(st.st_mode) && st.st_size == 0) + st.st_size = get_device_size(fd1, fname); + /* If we're not preserving permissions, change the file-list's * mode based on the local permissions and some heuristics. */ if (!preserve_perms) { @@ -806,26 +819,27 @@ int recv_files(int f_in, int f_out, char *local_name) parent_dirname = dn; } #endif - file->mode = dest_mode(file->mode, st.st_mode, - dflt_perms, exists); + file->mode = dest_mode(file->mode, st.st_mode, dflt_perms, exists); } /* We now check to see if we are writing the file "inplace" */ - if (inplace) { - fd2 = do_open(fname, O_WRONLY|O_CREAT, 0600); + if (inplace || one_inplace) { + fnametmp = one_inplace ? partialptr : fname; + fd2 = do_open(fnametmp, O_WRONLY|O_CREAT, 0600); if (fd2 == -1) { rsyserr(FERROR_XFER, errno, "open %s failed", - full_fname(fname)); + full_fname(fnametmp)); } else if (updating_basis_or_equiv) cleanup_set(NULL, NULL, file, fd1, fd2); } else { + fnametmp = fnametmpbuf; fd2 = open_tmpfile(fnametmp, fname, file); if (fd2 != -1) cleanup_set(fnametmp, partialptr, file, fd1, fd2); } if (fd2 == -1) { - discard_receive_data(f_in, F_LENGTH(file)); + discard_receive_data(f_in, file); if (fd1 != -1) close(fd1); if (inc_recurse) @@ -840,10 +854,11 @@ int recv_files(int f_in, int f_out, char *local_name) rprintf(FINFO, "%s\n", fname); /* recv file data */ - recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, - fname, fd2, F_LENGTH(file)); + recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, file, inplace || one_inplace); log_item(log_code, file, iflags, NULL); + if (want_progress_now) + instant_progress(fname); if (fd1 != -1) close(fd1); @@ -856,19 +871,19 @@ int recv_files(int f_in, int f_out, char *local_name) if ((recv_ok && (!delay_updates || !partialptr)) || inplace) { if (partialptr == fname) partialptr = NULL; - if (!finish_transfer(fname, fnametmp, fnamecmp, - partialptr, file, recv_ok, 1)) + if (!finish_transfer(fname, fnametmp, fnamecmp, partialptr, file, recv_ok, 1)) recv_ok = -1; else if (fnamecmp == partialptr) { - do_unlink(partialptr); + if (!one_inplace) + do_unlink(partialptr); handle_partial_dir(partialptr, PDIR_DELETE); } - } else if (keep_partial && partialptr) { + } else if (keep_partial && partialptr && !one_inplace) { if (!handle_partial_dir(partialptr, PDIR_CREATE)) { rprintf(FERROR, - "Unable to create partial-dir for %s -- discarding %s.\n", - local_name ? local_name : f_name(file, NULL), - recv_ok ? "completed file" : "partial file"); + "Unable to create partial-dir for %s -- discarding %s.\n", + local_name ? local_name : f_name(file, NULL), + recv_ok ? "completed file" : "partial file"); do_unlink(fnametmp); recv_ok = -1; } else if (!finish_transfer(partialptr, fnametmp, fnamecmp, NULL, @@ -879,7 +894,7 @@ int recv_files(int f_in, int f_out, char *local_name) recv_ok = 2; } else partialptr = NULL; - } else + } else if (!one_inplace) do_unlink(fnametmp); cleanup_disable(); @@ -926,7 +941,7 @@ int recv_files(int f_in, int f_out, char *local_name) } else if (inc_recurse) send_msg_int(MSG_NO_SEND, ndx); break; - } + } case -1: if (inc_recurse) send_msg_int(MSG_NO_SEND, ndx); @@ -1,7 +1,7 @@ /* * A pre-compilation helper program to aid in the creation of rounding.h. * - * Copyright (C) 2007-2018 Wayne Davison + * Copyright (C) 2007-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,16 +23,16 @@ #define SIZEOF(x) ((long int)sizeof (x)) struct test { - union file_extras extras[ARRAY_LEN]; - struct file_struct file; + union file_extras extras[ARRAY_LEN]; + int64 test; }; #define ACTUAL_SIZE SIZEOF(struct test) -#define EXPECTED_SIZE (SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(struct file_struct)) +#define EXPECTED_SIZE (SIZEOF(union file_extras) * ARRAY_LEN + SIZEOF(int64)) int main(UNUSED(int argc), UNUSED(char *argv[])) { - static int test_array[1 - 2 * (ACTUAL_SIZE != EXPECTED_SIZE)]; - test_array[0] = 0; - return 0; + static int test_array[1 - 2 * (ACTUAL_SIZE != EXPECTED_SIZE)]; + test_array[0] = 0; + return 0; } diff --git a/rsync-ssl b/rsync-ssl new file mode 100755 index 00000000..8101975a --- /dev/null +++ b/rsync-ssl @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +# This script uses openssl, gnutls, or stunnel to secure an rsync daemon connection. + +# By default this script takes rsync args and hands them off to the actual +# rsync command with an --rsh option that makes it open an SSL connection to an +# rsync daemon. See the rsync-ssl manpage for usage details and env variables. + +# When the first arg is --HELPER, we are being used by rsync as an --rsh helper +# script, and the args are (note the trailing dot): +# +# rsync-ssl --HELPER HOSTNAME rsync --server --daemon . +# +# --HELPER is not a user-facing option, so it is not documented in the manpage. + +# The first SSL setup was based on: http://dozzie.jarowit.net/trac/wiki/RsyncSSL +# Note that an stunnel connection requires at least version 4.x of stunnel. + +function rsync_ssl_run { + case "$*" in + *rsync://*) ;; + *::*) ;; + *) + echo "You must use rsync-ssl with a daemon-style hostname." 1>&2 + exit 1 + ;; + esac + + exec rsync --rsh="$0 --HELPER" "${@}" +} + +function rsync_ssl_helper { + if [[ -z "$RSYNC_SSL_TYPE" ]]; then + found=`path_search openssl stunnel4 stunnel` || exit 1 + if [[ "$found" == */openssl ]]; then + RSYNC_SSL_TYPE=openssl + RSYNC_SSL_OPENSSL="$found" + elif [[ "$found" == */gnutls-cli ]]; then + RSYNC_SSL_TYPE=gnutls + RSYNC_SSL_GNUTLS="$found" + else + RSYNC_SSL_TYPE=stunnel + RSYNC_SSL_STUNNEL="$found" + fi + fi + + case "$RSYNC_SSL_TYPE" in + openssl) + if [[ -z "$RSYNC_SSL_OPENSSL" ]]; then + RSYNC_SSL_OPENSSL=`path_search openssl` || exit 1 + fi + optsep=' ' + ;; + gnutls) + if [[ -z "$RSYNC_SSL_GNUTLS" ]]; then + RSYNC_SSL_GNUTLS=`path_search gnutls-cli` || exit 1 + fi + optsep=' ' + ;; + stunnel) + if [[ -z "$RSYNC_SSL_STUNNEL" ]]; then + RSYNC_SSL_STUNNEL=`path_search stunnel4 stunnel` || exit 1 + fi + optsep=' = ' + ;; + *) + echo "The RSYNC_SSL_TYPE specifies an unknown type: $RSYNC_SSL_TYPE" 1>&2 + exit 1 + ;; + esac + + if [[ -z "$RSYNC_SSL_CERT" ]]; then + certopt="" + gnutls_cert_opt="" + else + certopt="cert$optsep$RSYNC_SSL_CERT" + gnutls_cert_opt="--x509keyfile=$RSYNC_SSL_CERT" + fi + + if [[ -z ${RSYNC_SSL_CA_CERT+x} ]]; then + # RSYNC_SSL_CA_CERT unset - default CA set AND verify: + # openssl: + caopt="-verify_return_error -verify 4" + # gnutls: + gnutls_opts="" + # stunnel: + # Since there is no way of using the default CA certificate collection, + # we cannot do any verification. Thus, stunnel should really only be + # used if nothing else is available. + cafile="" + verify="" + elif [[ "$RSYNC_SSL_CA_CERT" == "" ]]; then + # RSYNC_SSL_CA_CERT set but empty -do NO verifications: + # openssl: + caopt="-verify 1" + # gnutls: + gnutls_opts="--insecure" + # stunnel: + cafile="" + verify="verifyChain = no" + else + # RSYNC_SSL_CA_CERT set - use CA AND verify: + # openssl: + caopt="-CAfile $RSYNC_SSL_CA_CERT -verify_return_error -verify 4" + # gnutls: + gnutls_opts="--x509cafile=$RSYNC_SSL_CA_CERT" + # stunnel: + cafile="CAfile = $RSYNC_SSL_CA_CERT" + verify="verifyChain = yes" + fi + + port="${RSYNC_PORT:-0}" + if [[ "$port" == 0 ]]; then + port="${RSYNC_SSL_PORT:-874}" + fi + + # If the user specified USER@HOSTNAME::module, then rsync passes us + # the -l USER option too, so we must be prepared to ignore it. + if [[ "$1" == "-l" ]]; then + shift 2 + fi + + hostname="$1" + shift + + if [[ -z "$hostname" || "$1" != rsync || "$2" != --server || "$3" != --daemon ]]; then + echo "Usage: rsync-ssl --HELPER HOSTNAME rsync --server --daemon ." 1>&2 + exit 1 + fi + + if [[ $RSYNC_SSL_TYPE == openssl ]]; then + exec $RSYNC_SSL_OPENSSL s_client $caopt $certopt -quiet -verify_quiet -servername $hostname -connect $hostname:$port + elif [[ $RSYNC_SSL_TYPE == gnutls ]]; then + exec $RSYNC_SSL_GNUTLS --logfile=/dev/null $gnutls_cert_opt $gnutls_opts $hostname:$port + else + # devzero@web.de came up with this no-tmpfile calling syntax: + exec $RSYNC_SSL_STUNNEL -fd 10 11<&0 <<EOF 10<&0 0<&11 11<&- +foreground = yes +debug = crit +connect = $hostname:$port +client = yes +TIMEOUTclose = 0 +$verify +$certopt +$cafile +EOF + fi +} + +function path_search { + IFS_SAVE="$IFS" + IFS=: + for prog in "${@}"; do + for dir in $PATH; do + [[ -z "$dir" ]] && dir=. + if [[ -f "$dir/$prog" && -x "$dir/$prog" ]]; then + echo "$dir/$prog" + IFS="$IFS_SAVE" + return 0 + fi + done + done + + IFS="$IFS_SAVE" + echo "Failed to find on your path: $*" 1>&2 + echo "See the rsync-ssl manpage for configuration assistance." 1>&2 + return 1 +} + +if [[ "$#" == 0 ]]; then + echo "Usage: rsync-ssl [--type=SSL_TYPE] RSYNC_ARG [...]" 1>&2 + echo "The SSL_TYPE can be openssl or stunnel" + exit 1 +fi + +if [[ "$1" = --help || "$1" = -h ]]; then + exec rsync --help +fi + +if [[ "$1" == --HELPER ]]; then + shift + rsync_ssl_helper "${@}" +fi + +if [[ "$1" == --type=* ]]; then + export RSYNC_SSL_TYPE="${1/--type=/}" + shift +fi + +rsync_ssl_run "${@}" diff --git a/rsync-ssl.1 b/rsync-ssl.1 new file mode 100644 index 00000000..4677a57a --- /dev/null +++ b/rsync-ssl.1 @@ -0,0 +1,134 @@ +.TH "rsync-ssl" "1" "19 Jun 2020" "rsync-ssl 3.2.0" "User Commands" +.P +.SH "NAME" +.P +rsync-ssl \- a helper script for connecting to an ssl rsync daemon +.P +.SH "SYNOPSIS" +.P +.nf +rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS +.fi +.P +.SH "DESCRIPTION" +.P +The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon +that requires ssl connections. +.P +The script requires that you specify an rsync-daemon arg in the style of either +\fBhostname::\fP (with 2 colons) or \fBrsync://hostname/\fP. The default port used for +connecting is 874 (one higher than the normal 873) unless overridden in the +environment. You can specify an overriding port via \fB\-\-port\fP or by including +it in the normal spot in the URL format, though both of those require your +rsync version to be at least 3.2.0. +.P +.SH "OPTIONS" +.P +If the \fBfirst\fP arg is a \fB\-\-type=SSL_TYPE\fP option, the script will only use +that particular program to open an ssl connection instead of trying to find an +openssl or stunnel executable via a simple heuristic (assuming that the +\fBRSYNC_SSL_TYPE\fP environment variable is not set as well\ \-\- see below). This +option must specify one of \fBopenssl\fP or \fBstunnel\fP. The equal sign is +required for this particular option. +.P +All the other options are passed through to the rsync command, so consult the +\fBrsync\fP(1) manpage for more information on how it works. +.P +.SH "ENVIRONMENT VARIABLES" +.P +The ssl helper scripts are affected by the following environment variables: +.P +.IP "\fBRSYNC_SSL_TYPE\fP" +Specifies the program type that should be used to open the +ssl connection. It must be one of \fBopenssl\fP or \fBstunnel\fP. The +\fB\-\-type=SSL_TYPE\fP option overrides this, when specified. +.IP "\fBRSYNC_SSL_PORT\fP" +If specified, the value is the port number that is used as +the default when the user does not specify a port in their rsync command. +When not specified, the default port number is 874. (Note that older rsync +versions (prior to 3.2.0) did not communicate an overriding port number +value to the helper script.) +.IP "\fBRSYNC_SSL_CERT\fP" +If specified, the value is a filename that contains a +certificate to use for the connection. +.IP "\fBRSYNC_SSL_CA_CERT\fP" +If specified, the value is a filename that contains a +certificate authority certificate that is used to validate the connection. +.IP "\fBRSYNC_SSL_OPENSSL\fP" +Specifies the openssl executable to run when the +connection type is set to openssl. If unspecified, the $PATH is searched +for "openssl". +.IP "\fBRSYNC_SSL_GNUTLS\fP" +Specifies the gnutls-cli executable to run when the +connection type is set to gnutls. If unspecified, the $PATH is searched +for "gnutls-cli". +.IP "\fBRSYNC_SSL_STUNNEL\fP" +Specifies the stunnel executable to run when the +connection type is set to stunnel. If unspecified, the $PATH is searched +first for "stunnel4" and then for "stunnel". +.P +.SH "EXAMPLES" +.RS 4 +.P +.nf +rsync-ssl -aiv example.com::mod/ dest +.fi +.RE +.RS 4 +.P +.nf +rsync-ssl --type=openssl -aiv example.com::mod/ dest +.fi +.RE +.RS 4 +.P +.nf +rsync-ssl -aiv --port 9874 example.com::mod/ dest +.fi +.RE +.RS 4 +.P +.nf +rsync-ssl -aiv rsync://example.com:9874/mod/ dest +.fi +.RE +.P +.SH "SEE ALSO" +.P +\fBrsync\fP(1), \fBrsyncd.conf\fP(5) +.P +.SH "CAVEATS" +.P +Note that using an stunnel connection requires at least version 4 of stunnel, +which should be the case on modern systems. Also, it does not verify a +connection against the CA certificate collection, so it only encrypts the +connection without any cert validation unless you have specified the +certificate environment options. +.P +This script also supports a \fB\-\-type=gnutls\fP option, but at the time of this +release the gnutls-cli command was dropping output, making it unusable. If +that bug has been fixed in your version, feel free to put gnutls into an +exported RSYNC_SSL_TYPE environment variable to make its use the default. +.P +.SH "BUGS" +.P +Please report bugs! See the web site at http://rsync.samba.org/. +.P +.SH "VERSION" +.P +This man page is current for version 3.2.0 of rsync. +.P +.SH "CREDITS" +.P +rsync is distributed under the GNU General Public License. See the file +COPYING for details. +.P +A web site is available at http://rsync.samba.org/. The site includes an +FAQ-O-Matic which may cover questions unanswered by this manual page. +.P +.SH "AUTHOR" +.P +This manpage was written by Wayne Davison. +.P +Mailing lists for support and development are available at +http://lists.samba.org/. diff --git a/rsync-ssl.1.html b/rsync-ssl.1.html new file mode 100644 index 00000000..8f57e2cd --- /dev/null +++ b/rsync-ssl.1.html @@ -0,0 +1,120 @@ +<html><head> +<title>rsync-ssl(1) man page</title> +<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet"> +<style> +body { + max-width: 50em; + margin: auto; +} +body, b, strong, u { + font-family: 'Roboto', sans-serif; +} +code { + font-family: 'Roboto Mono', monospace; + font-weight: bold; + white-space: pre; +} +pre code { + display: block; + font-weight: normal; +} +blockquote pre code { + background: #f1f1f1; +} +dd p:first-of-type { + margin-block-start: 0em; +} +</style> +</head><body> +<h1>NAME</h1> +<p>rsync-ssl -⁠ a helper script for connecting to an ssl rsync daemon</p> +<h1>SYNOPSIS</h1> +<pre><code>rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS +</code></pre> +<h1>DESCRIPTION</h1> +<p>The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon +that requires ssl connections.</p> +<p>The script requires that you specify an rsync-daemon arg in the style of either +<code>hostname::</code> (with 2 colons) or <code>rsync://hostname/</code>. The default port used for +connecting is 874 (one higher than the normal 873) unless overridden in the +environment. You can specify an overriding port via <code>--port</code> or by including +it in the normal spot in the URL format, though both of those require your +rsync version to be at least 3.2.0.</p> +<h1>OPTIONS</h1> +<p>If the <strong>first</strong> arg is a <code>--type=SSL_TYPE</code> option, the script will only use +that particular program to open an ssl connection instead of trying to find an +openssl or stunnel executable via a simple heuristic (assuming that the +<code>RSYNC_SSL_TYPE</code> environment variable is not set as well -⁠-⁠ see below). This +option must specify one of <code>openssl</code> or <code>stunnel</code>. The equal sign is +required for this particular option.</p> +<p>All the other options are passed through to the rsync command, so consult the +<strong>rsync</strong>(1) manpage for more information on how it works.</p> +<h1>ENVIRONMENT VARIABLES</h1> +<p>The ssl helper scripts are affected by the following environment variables:</p> +<dl> +<dt><code>RSYNC_SSL_TYPE</code></dt><dd> Specifies the program type that should be used to open the +ssl connection. It must be one of <code>openssl</code> or <code>stunnel</code>. The +<code>--type=SSL_TYPE</code> option overrides this, when specified.</dd> +<dt><code>RSYNC_SSL_PORT</code></dt><dd> If specified, the value is the port number that is used as +the default when the user does not specify a port in their rsync command. +When not specified, the default port number is 874. (Note that older rsync +versions (prior to 3.2.0) did not communicate an overriding port number +value to the helper script.)</dd> +<dt><code>RSYNC_SSL_CERT</code></dt><dd> If specified, the value is a filename that contains a +certificate to use for the connection.</dd> +<dt><code>RSYNC_SSL_CA_CERT</code></dt><dd> If specified, the value is a filename that contains a +certificate authority certificate that is used to validate the connection.</dd> +<dt><code>RSYNC_SSL_OPENSSL</code></dt><dd> Specifies the openssl executable to run when the +connection type is set to openssl. If unspecified, the $PATH is searched +for "openssl".</dd> +<dt><code>RSYNC_SSL_GNUTLS</code></dt><dd> Specifies the gnutls-cli executable to run when the +connection type is set to gnutls. If unspecified, the $PATH is searched +for "gnutls-cli".</dd> +<dt><code>RSYNC_SSL_STUNNEL</code></dt><dd> Specifies the stunnel executable to run when the +connection type is set to stunnel. If unspecified, the $PATH is searched +first for "stunnel4" and then for "stunnel".</dd> +</dl> +<h1>EXAMPLES</h1> +<blockquote> +<pre><code>rsync-ssl -aiv example.com::mod/ dest +</code></pre> +</blockquote> +<blockquote> +<pre><code>rsync-ssl --type=openssl -aiv example.com::mod/ dest +</code></pre> +</blockquote> +<blockquote> +<pre><code>rsync-ssl -aiv --port 9874 example.com::mod/ dest +</code></pre> +</blockquote> +<blockquote> +<pre><code>rsync-ssl -aiv rsync://example.com:9874/mod/ dest +</code></pre> +</blockquote> +<h1>SEE ALSO</h1> +<p><strong>rsync</strong>(1), <strong>rsyncd.conf</strong>(5)</p> +<h1>CAVEATS</h1> +<p>Note that using an stunnel connection requires at least version 4 of stunnel, +which should be the case on modern systems. Also, it does not verify a +connection against the CA certificate collection, so it only encrypts the +connection without any cert validation unless you have specified the +certificate environment options.</p> +<p>This script also supports a <code>--type=gnutls</code> option, but at the time of this +release the gnutls-cli command was dropping output, making it unusable. If +that bug has been fixed in your version, feel free to put gnutls into an +exported RSYNC_SSL_TYPE environment variable to make its use the default.</p> +<h1>BUGS</h1> +<p>Please report bugs! See the web site at <a href="http://rsync.samba.org/">http://rsync.samba.org/</a>.</p> +<h1>VERSION</h1> +<p>This man page is current for version 3.2.0 of rsync.</p> +<h1>CREDITS</h1> +<p>rsync is distributed under the GNU General Public License. See the file +COPYING for details.</p> +<p>A web site is available at <a href="http://rsync.samba.org/">http://rsync.samba.org/</a>. The site includes an +FAQ-O-Matic which may cover questions unanswered by this manual page.</p> +<h1>AUTHOR</h1> +<p>This manpage was written by Wayne Davison.</p> +<p>Mailing lists for support and development are available at +<a href="http://lists.samba.org/">http://lists.samba.org/</a>.</p> +<div style="float: right"><p><i>19 Jun 2020</i></p></div> +</body></html> diff --git a/rsync-ssl.1.md b/rsync-ssl.1.md new file mode 100644 index 00000000..31cb465f --- /dev/null +++ b/rsync-ssl.1.md @@ -0,0 +1,109 @@ +# NAME + +rsync-ssl - a helper script for connecting to an ssl rsync daemon + +# SYNOPSIS + +``` +rsync-ssl [--type=SSL_TYPE] RSYNC_ARGS +``` + +# DESCRIPTION + +The rsync-ssl script helps you to run an rsync copy to/from an rsync daemon +that requires ssl connections. + +The script requires that you specify an rsync-daemon arg in the style of either +`hostname::` (with 2 colons) or `rsync://hostname/`. The default port used for +connecting is 874 (one higher than the normal 873) unless overridden in the +environment. You can specify an overriding port via `--port` or by including +it in the normal spot in the URL format, though both of those require your +rsync version to be at least 3.2.0. + +# OPTIONS + +If the **first** arg is a `--type=SSL_TYPE` option, the script will only use +that particular program to open an ssl connection instead of trying to find an +openssl or stunnel executable via a simple heuristic (assuming that the +`RSYNC_SSL_TYPE` environment variable is not set as well -- see below). This +option must specify one of `openssl` or `stunnel`. The equal sign is +required for this particular option. + +All the other options are passed through to the rsync command, so consult the +**rsync**(1) manpage for more information on how it works. + +# ENVIRONMENT VARIABLES + +The ssl helper scripts are affected by the following environment variables: + +0. `RSYNC_SSL_TYPE` Specifies the program type that should be used to open the + ssl connection. It must be one of `openssl` or `stunnel`. The + `--type=SSL_TYPE` option overrides this, when specified. +0. `RSYNC_SSL_PORT` If specified, the value is the port number that is used as + the default when the user does not specify a port in their rsync command. + When not specified, the default port number is 874. (Note that older rsync + versions (prior to 3.2.0) did not communicate an overriding port number + value to the helper script.) +0. `RSYNC_SSL_CERT` If specified, the value is a filename that contains a + certificate to use for the connection. +0. `RSYNC_SSL_CA_CERT` If specified, the value is a filename that contains a + certificate authority certificate that is used to validate the connection. +0. `RSYNC_SSL_OPENSSL` Specifies the openssl executable to run when the + connection type is set to openssl. If unspecified, the $PATH is searched + for "openssl". +0. `RSYNC_SSL_GNUTLS` Specifies the gnutls-cli executable to run when the + connection type is set to gnutls. If unspecified, the $PATH is searched + for "gnutls-cli". +0. `RSYNC_SSL_STUNNEL` Specifies the stunnel executable to run when the + connection type is set to stunnel. If unspecified, the $PATH is searched + first for "stunnel4" and then for "stunnel". + +# EXAMPLES + +> rsync-ssl -aiv example.com::mod/ dest + +> rsync-ssl --type=openssl -aiv example.com::mod/ dest + +> rsync-ssl -aiv --port 9874 example.com::mod/ dest + +> rsync-ssl -aiv rsync://example.com:9874/mod/ dest + +# SEE ALSO + +**rsync**(1), **rsyncd.conf**(5) + +# CAVEATS + +Note that using an stunnel connection requires at least version 4 of stunnel, +which should be the case on modern systems. Also, it does not verify a +connection against the CA certificate collection, so it only encrypts the +connection without any cert validation unless you have specified the +certificate environment options. + +This script also supports a `--type=gnutls` option, but at the time of this +release the gnutls-cli command was dropping output, making it unusable. If +that bug has been fixed in your version, feel free to put gnutls into an +exported RSYNC_SSL_TYPE environment variable to make its use the default. + +# BUGS + +Please report bugs! See the web site at <http://rsync.samba.org/>. + +# VERSION + +This man page is current for version @VERSION@ of rsync. + +# CREDITS + +rsync is distributed under the GNU General Public License. See the file +COPYING for details. + +A web site is available at <http://rsync.samba.org/>. The site includes an +FAQ-O-Matic which may cover questions unanswered by this manual page. + +# AUTHOR + +This manpage was written by Wayne Davison. + +Mailing lists for support and development are available at +<http://lists.samba.org/>. diff --git a/rsync-ssl.in b/rsync-ssl.in deleted file mode 100755 index da58d6af..00000000 --- a/rsync-ssl.in +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# This script supports using stunnel to secure an rsync daemon connection. -# Note that this requires at least version 4.x of stunnel. -case "$@" in -*rsync://*) ;; -*::*) ;; -*) - echo "You must use rsync-ssl with a daemon-style hostname." 0>&1 - exit 1 - ;; -esac -exec @bindir@/rsync --rsh=@bindir@/stunnel-rsync "${@}" @@ -1,2080 +1,2163 @@ -.TH "rsync" "1" "28 Jan 2018" "" "" +.TH "rsync" "1" "19 Jun 2020" "rsync 3.2.0" "User Commands" +.P .SH "NAME" -rsync \- a fast, versatile, remote (and local) file\-copying tool +.P +rsync \- a fast, versatile, remote (and local) file-copying tool +.P .SH "SYNOPSIS" - -.PP -.nf -Local: rsync [OPTION...] SRC... [DEST] +.P +.nf +Local: + rsync [OPTION...] SRC... [DEST] Access via remote shell: - Pull: rsync [OPTION...] [USER@]HOST:SRC... [DEST] - Push: rsync [OPTION...] SRC... [USER@]HOST:DEST + Pull: + rsync [OPTION...] [USER@]HOST:SRC... [DEST] + Push: + rsync [OPTION...] SRC... [USER@]HOST:DEST Access via rsync daemon: - Pull: rsync [OPTION...] [USER@]HOST::SRC... [DEST] + Pull: + rsync [OPTION...] [USER@]HOST::SRC... [DEST] rsync [OPTION...] rsync://[USER@]HOST[:PORT]/SRC... [DEST] - Push: rsync [OPTION...] SRC... [USER@]HOST::DEST - rsync [OPTION...] SRC... rsync://[USER@]HOST[:PORT]/DEST -.fi - -.PP -Usages with just one SRC arg and no DEST arg will list the source files -instead of copying. -.PP + Push: + rsync [OPTION...] SRC... [USER@]HOST::DEST + rsync [OPTION...] SRC... rsync://[USER@]HOST[:PORT]/DEST) +.fi +.P +Usages with just one SRC arg and no DEST arg will list the source files instead +of copying. +.P .SH "DESCRIPTION" - -.PP -Rsync is a fast and extraordinarily versatile file copying tool. It can -copy locally, to/from another host over any remote shell, or to/from a -remote rsync daemon. It offers a large number of options that control -every aspect of its behavior and permit very flexible specification of the -set of files to be copied. It is famous for its delta\-transfer algorithm, -which reduces the amount of data sent over the network by sending only the -differences between the source files and the existing files in the -destination. Rsync is widely used for backups and mirroring and as an -improved copy command for everyday use. -.PP -Rsync finds files that need to be transferred using a \(dq\&quick check\(dq\& -algorithm (by default) that looks for files that have changed in size or -in last\-modified time. Any changes in the other preserved attributes (as -requested by options) are made on the destination file directly when the -quick check indicates that the file\(cq\&s data does not need to be updated. -.PP +.P +Rsync is a fast and extraordinarily versatile file copying tool. It can copy +locally, to/from another host over any remote shell, or to/from a remote rsync +daemon. It offers a large number of options that control every aspect of its +behavior and permit very flexible specification of the set of files to be +copied. It is famous for its delta-transfer algorithm, which reduces the +amount of data sent over the network by sending only the differences between +the source files and the existing files in the destination. Rsync is widely +used for backups and mirroring and as an improved copy command for everyday +use. +.P +Rsync finds files that need to be transferred using a "quick check" algorithm +(by default) that looks for files that have changed in size or in last-modified +time. Any changes in the other preserved attributes (as requested by options) +are made on the destination file directly when the quick check indicates that +the file's data does not need to be updated. +.P Some of the additional features of rsync are: -.PP -.IP o +.P +.IP o support for copying links, devices, owners, groups, and permissions -.IP o -exclude and exclude\-from options similar to GNU tar -.IP o +.IP o +exclude and exclude-from options similar to GNU tar +.IP o a CVS exclude mode for ignoring the same files that CVS would ignore -.IP o +.IP o can use any transparent remote shell, including ssh or rsh -.IP o -does not require super\-user privileges -.IP o +.IP o +does not require super-user privileges +.IP o pipelining of file transfers to minimize latency costs -.IP o -support for anonymous or authenticated rsync daemons (ideal for -mirroring) - -.PP +.IP o +support for anonymous or authenticated rsync daemons (ideal for mirroring) +.P .SH "GENERAL" - -.PP -Rsync copies files either to or from a remote host, or locally on the -current host (it does not support copying files between two remote hosts). -.PP +.P +Rsync copies files either to or from a remote host, or locally on the current +host (it does not support copying files between two remote hosts). +.P There are two different ways for rsync to contact a remote system: using a -remote\-shell program as the transport (such as ssh or rsh) or contacting an -rsync daemon directly via TCP. The remote\-shell transport is used whenever -the source or destination path contains a single colon (:) separator after -a host specification. Contacting an rsync daemon directly happens when the -source or destination path contains a double colon (::) separator after a -host specification, OR when an rsync:// URL is specified (see also the -\(dq\&USING RSYNC\-DAEMON FEATURES VIA A REMOTE\-SHELL CONNECTION\(dq\& section for -an exception to this latter rule). -.PP -As a special case, if a single source arg is specified without a -destination, the files are listed in an output format similar to \(dq\&ls \-l\(dq\&. -.PP -As expected, if neither the source or destination path specify a remote -host, the copy occurs locally (see also the \fB\-\-list\-only\fP option). -.PP -Rsync refers to the local side as the \(dq\&client\(dq\& and the remote side as the -\(dq\&server\(dq\&. Don\(cq\&t confuse \(dq\&server\(dq\& with an rsync daemon \-\- a daemon is always a -server, but a server can be either a daemon or a remote\-shell spawned process. -.PP +remote-shell program as the transport (such as ssh or rsh) or contacting an +rsync daemon directly via TCP. The remote-shell transport is used whenever the +source or destination path contains a single colon (:) separator after a host +specification. Contacting an rsync daemon directly happens when the source or +destination path contains a double colon (::) separator after a host +specification, OR when an rsync:// URL is specified (see also the "USING +RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION" section for an exception +to this latter rule). +.P +As a special case, if a single source arg is specified without a destination, +the files are listed in an output format similar to "\fBls\ \-l\fP". +.P +As expected, if neither the source or destination path specify a remote host, +the copy occurs locally (see also the \fB\-\-list-only\fP option). +.P +Rsync refers to the local side as the client and the remote side as the server. +Don't confuse server with an rsync daemon. A daemon is always a server, but a +server can be either a daemon or a remote-shell spawned process. +.P .SH "SETUP" - -.PP -See the file README for installation instructions. -.PP -Once installed, you can use rsync to any machine that you can access via -a remote shell (as well as some that you can access using the rsync -daemon\-mode protocol). For remote transfers, a modern rsync uses ssh -for its communications, but it may have been configured to use a -different remote shell by default, such as rsh or remsh. -.PP +.P +See the file README.md for installation instructions. +.P +Once installed, you can use rsync to any machine that you can access via a +remote shell (as well as some that you can access using the rsync daemon-mode +protocol). For remote transfers, a modern rsync uses ssh for its +communications, but it may have been configured to use a different remote shell +by default, such as rsh or remsh. +.P You can also specify any remote shell you like, either by using the \fB\-e\fP command line option, or by setting the RSYNC_RSH environment variable. -.PP -Note that rsync must be installed on both the source and destination -machines. -.PP +.P +Note that rsync must be installed on both the source and destination machines. +.P .SH "USAGE" - -.PP -You use rsync in the same way you use rcp. You must specify a source -and a destination, one of which may be remote. -.PP +.P +You use rsync in the same way you use rcp. You must specify a source and a +destination, one of which may be remote. +.P Perhaps the best way to explain the syntax is with some examples: -.PP -.RS -\f(CWrsync \-t *.c foo:src/\fP +.RS 4 +.P +.nf +rsync -t *.c foo:src/ +.fi .RE - -.PP -This would transfer all files matching the pattern *.c from the -current directory to the directory src on the machine foo. If any of -the files already exist on the remote system then the rsync -remote\-update protocol is used to update the file by sending only the -differences in the data. Note that the expansion of wildcards on the -commandline (*.c) into a list of files is handled by the shell before -it runs rsync and not by rsync itself (exactly the same as all other -posix\-style programs). -.PP -.RS -\f(CWrsync \-avz foo:src/bar /data/tmp\fP +.P +This would transfer all files matching the pattern \fB*.c\fP from the current +directory to the directory src on the machine foo. If any of the files already +exist on the remote system then the rsync remote-update protocol is used to +update the file by sending only the differences in the data. Note that the +expansion of wildcards on the commandline (\fB*.c\fP) into a list of files is +handled by the shell before it runs rsync and not by rsync itself (exactly the +same as all other posix-style programs). +.RS 4 +.P +.nf +rsync -avz foo:src/bar /data/tmp +.fi .RE - -.PP +.P This would recursively transfer all files from the directory src/bar on the -machine foo into the /data/tmp/bar directory on the local machine. The -files are transferred in \(dq\&archive\(dq\& mode, which ensures that symbolic -links, devices, attributes, permissions, ownerships, etc. are preserved -in the transfer. Additionally, compression will be used to reduce the -size of data portions of the transfer. -.PP -.RS -\f(CWrsync \-avz foo:src/bar/ /data/tmp\fP +machine foo into the /data/tmp/bar directory on the local machine. The files +are transferred in archive mode, which ensures that symbolic links, devices, +attributes, permissions, ownerships, etc. are preserved in the transfer. +Additionally, compression will be used to reduce the size of data portions of +the transfer. +.RS 4 +.P +.nf +rsync -avz foo:src/bar/ /data/tmp +.fi .RE - -.PP +.P A trailing slash on the source changes this behavior to avoid creating an -additional directory level at the destination. You can think of a trailing -/ on a source as meaning \(dq\© the contents of this directory\(dq\& as opposed -to \(dq\© the directory by name\(dq\&, but in both cases the attributes of the +additional directory level at the destination. You can think of a trailing / +on a source as meaning "copy the contents of this directory" as opposed to +"copy the directory by name", but in both cases the attributes of the containing directory are transferred to the containing directory on the -destination. In other words, each of the following commands copies the -files in the same way, including their setting of the attributes of -/dest/foo: -.PP -.RS -\f(CWrsync \-av /src/foo /dest\fP -.br -\f(CWrsync \-av /src/foo/ /dest/foo\fP -.br +destination. In other words, each of the following commands copies the files +in the same way, including their setting of the attributes of /dest/foo: +.RS 4 +.P +.nf +rsync -av /src/foo /dest +rsync -av /src/foo/ /dest/foo +.fi .RE - -.PP -Note also that host and module references don\(cq\&t require a trailing slash to -copy the contents of the default directory. For example, both of these -copy the remote directory\(cq\&s contents into \(dq\&/dest\(dq\&: -.PP -.RS -\f(CWrsync \-av host: /dest\fP -.br -\f(CWrsync \-av host::module /dest\fP -.br +.P +Note also that host and module references don't require a trailing slash to +copy the contents of the default directory. For example, both of these copy +the remote directory's contents into "/dest": +.RS 4 +.P +.nf +rsync -av host: /dest +rsync -av host::module /dest +.fi .RE - -.PP -You can also use rsync in local\-only mode, where both the source and -destination don\(cq\&t have a \(cq\&:\(cq\& in the name. In this case it behaves like -an improved copy command. -.PP -Finally, you can list all the (listable) modules available from a -particular rsync daemon by leaving off the module name: -.PP -.RS -\f(CWrsync somehost.mydomain.com::\fP +.P +You can also use rsync in local-only mode, where both the source and +destination don't have a ':' in the name. In this case it behaves like an +improved copy command. +.P +Finally, you can list all the (listable) modules available from a particular +rsync daemon by leaving off the module name: +.RS 4 +.P +.nf +rsync somehost.mydomain.com:: +.fi .RE - -.PP +.P See the following section for more details. -.PP +.P .SH "ADVANCED USAGE" - -.PP +.P The syntax for requesting multiple files from a remote host is done by -specifying additional remote\-host args in the same style as the first, -or with the hostname omitted. For instance, all these work: -.PP -.RS -\f(CWrsync \-av host:file1 :file2 host:file{3,4} /dest/\fP -.br -\f(CWrsync \-av host::modname/file{1,2} host::modname/file3 /dest/\fP -.br -\f(CWrsync \-av host::modname/file1 ::modname/file{3,4}\fP +specifying additional remote-host args in the same style as the first, or with +the hostname omitted. For instance, all these work: +.RS 4 +.P +.nf +rsync -av host:file1 :file2 host:file{3,4} /dest/ +rsync -av host::modname/file{1,2} host::modname/file3 /dest/ +rsync -av host::modname/file1 ::modname/file{3,4} +.fi .RE - -.PP +.P Older versions of rsync required using quoted spaces in the SRC, like these examples: -.PP -.RS -\f(CWrsync \-av host:'\&dir1/file1 dir2/file2'\& /dest\fP -.br -\f(CWrsync host::'\&modname/dir1/file1 modname/dir2/file2'\& /dest\fP +.RS 4 +.P +.nf +rsync -av host:'dir1/file1 dir2/file2' /dest +rsync host::'modname/dir1/file1 modname/dir2/file2' /dest +.fi .RE - -.PP -This word\-splitting still works (by default) in the latest rsync, but is -not as easy to use as the first method. -.PP +.P +This word-splitting still works (by default) in the latest rsync, but is not as +easy to use as the first method. +.P If you need to transfer a filename that contains whitespace, you can either -specify the \fB\-\-protect\-args\fP (\fB\-s\fP) option, or you\(cq\&ll need to escape -the whitespace in a way that the remote shell will understand. For -instance: -.PP -.RS -\f(CWrsync \-av host:'\&file\e name\e with\e spaces'\& /dest\fP +specify the \fB\-\-protect-args\fP (\fB\-s\fP) option, or you'll need to escape the +whitespace in a way that the remote shell will understand. For instance: +.RS 4 +.P +.nf +rsync -av host:'file\\ name\\ with\\ spaces' /dest +.fi .RE - -.PP +.P .SH "CONNECTING TO AN RSYNC DAEMON" - -.PP -It is also possible to use rsync without a remote shell as the transport. -In this case you will directly connect to a remote rsync daemon, typically -using TCP port 873. (This obviously requires the daemon to be running on -the remote system, so refer to the STARTING AN RSYNC DAEMON TO ACCEPT -CONNECTIONS section below for information on that.) -.PP +.P +It is also possible to use rsync without a remote shell as the transport. In +this case you will directly connect to a remote rsync daemon, typically using +TCP port 873. (This obviously requires the daemon to be running on the remote +system, so refer to the STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS section +below for information on that.) +.P Using rsync in this way is the same as using it with a remote shell except that: -.PP -.IP o -you either use a double colon :: instead of a single colon to -separate the hostname from the path, or you use an rsync:// URL. -.IP o -the first word of the \(dq\&path\(dq\& is actually a module name. -.IP o -the remote daemon may print a message of the day when you -connect. -.IP o -if you specify no path name on the remote daemon then the -list of accessible paths on the daemon will be shown. -.IP o -if you specify no local destination then a listing of the -specified files on the remote daemon is provided. -.IP o -you must not specify the \fB\-\-rsh\fP (\fB\-e\fP) option. - -.PP -An example that copies all the files in a remote module named \(dq\&src\(dq\&: -.PP -.nf - rsync \-av host::src /dest -.fi - -.PP -Some modules on the remote daemon may require authentication. If so, -you will receive a password prompt when you connect. You can avoid the -password prompt by setting the environment variable RSYNC_PASSWORD to -the password you want to use or using the \fB\-\-password\-file\fP option. This -may be useful when scripting rsync. -.PP -WARNING: On some systems environment variables are visible to all -users. On those systems using \fB\-\-password\-file\fP is recommended. -.PP -You may establish the connection via a web proxy by setting the -environment variable RSYNC_PROXY to a hostname:port pair pointing to -your web proxy. Note that your web proxy\(cq\&s configuration must support -proxy connections to port 873. -.PP +.P +.IP o +you either use a double colon :: instead of a single colon to separate the +hostname from the path, or you use an rsync:// URL. +.IP o +the first word of the "path" is actually a module name. +.IP o +the remote daemon may print a message of the day when you connect. +.IP o +if you specify no path name on the remote daemon then the list of accessible +paths on the daemon will be shown. +.IP o +if you specify no local destination then a listing of the specified files on +the remote daemon is provided. +.IP o +you must not specify the \fB\-\-rsh\fP (\fB\-e\fP) option (since that overrides the +daemon connection to use ssh\ \-\- see USING RSYNC-DAEMON FEATURES VIA A +REMOTE-SHELL CONNECTION below). +.P +An example that copies all the files in a remote module named "src": +.RS 4 +.P +.nf +rsync -av host::src /dest +.fi +.RE +.P +Some modules on the remote daemon may require authentication. If so, you will +receive a password prompt when you connect. You can avoid the password prompt +by setting the environment variable RSYNC_PASSWORD to the password you want to +use or using the \fB\-\-password-file\fP option. This may be useful when scripting +rsync. +.P +WARNING: On some systems environment variables are visible to all users. On +those systems using \fB\-\-password-file\fP is recommended. +.P +You may establish the connection via a web proxy by setting the environment +variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note +that your web proxy's configuration must support proxy connections to port 873. +.P You may also establish a daemon connection using a program as a proxy by -setting the environment variable RSYNC_CONNECT_PROG to the commands you -wish to run in place of making a direct socket connection. The string may -contain the escape \(dq\&%H\(dq\& to represent the hostname specified in the rsync -command (so use \(dq\&%%\(dq\& if you need a single \(dq\&%\(dq\& in your string). For -example: -.PP -.nf - export RSYNC_CONNECT_PROG='\&ssh proxyhost nc %H 873'\& - rsync \-av targethost1::module/src/ /dest/ - rsync \-av rsync:://targethost2/module/src/ /dest/ -.fi - -.PP -The command specified above uses ssh to run nc (netcat) on a proxyhost, -which forwards all data to port 873 (the rsync daemon) on the targethost -(%H). -.PP -.SH "USING RSYNC\-DAEMON FEATURES VIA A REMOTE\-SHELL CONNECTION" - -.PP +setting the environment variable RSYNC_CONNECT_PROG to the commands you wish to +run in place of making a direct socket connection. The string may contain the +escape "%H" to represent the hostname specified in the rsync command (so use +"%%" if you need a single "%" in your string). For example: +.RS 4 +.P +.nf +export RSYNC_CONNECT_PROG='ssh proxyhost nc %H 873' +rsync -av targethost1::module/src/ /dest/ +rsync -av rsync://targethost2/module/src/ /dest/ +.fi +.RE +.P +The command specified above uses ssh to run nc (netcat) on a proxyhost, which +forwards all data to port 873 (the rsync daemon) on the targethost (%H). +.P +Note also that if the RSYNC_SHELL environment variable is set, that program +will be used to run the RSYNC_CONNECT_PROG command instead of using the default +shell of the \fBsystem()\fP call. +.P +.SH "USING RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION" +.P It is sometimes useful to use various features of an rsync daemon (such as named modules) without actually allowing any new socket connections into a -system (other than what is already required to allow remote\-shell access). -Rsync supports connecting to a host using a remote shell and then spawning -a single\-use \(dq\&daemon\(dq\& server that expects to read its config file in the -home dir of the remote user. This can be useful if you want to encrypt a -daemon\-style transfer\(cq\&s data, but since the daemon is started up fresh by -the remote user, you may not be able to use features such as chroot or -change the uid used by the daemon. (For another way to encrypt a daemon -transfer, consider using ssh to tunnel a local port to a remote machine and -configure a normal rsync daemon on that remote host to only allow -connections from \(dq\&localhost\(dq\&.) -.PP -From the user\(cq\&s perspective, a daemon transfer via a remote\-shell -connection uses nearly the same command\-line syntax as a normal -rsync\-daemon transfer, with the only exception being that you must -explicitly set the remote shell program on the command\-line with the -\fB\-\-rsh=COMMAND\fP option. (Setting the RSYNC_RSH in the environment -will not turn on this functionality.) For example: -.PP -.nf - rsync \-av \-\-rsh=ssh host::module /dest -.fi - -.PP -If you need to specify a different remote\-shell user, keep in mind that the -user@ prefix in front of the host is specifying the rsync\-user value (for a -module that requires user\-based authentication). This means that you must -give the \(cq\&\-l user\(cq\& option to ssh when specifying the remote\-shell, as in -this example that uses the short version of the \fB\-\-rsh\fP option: -.PP -.nf - rsync \-av \-e \(dq\&ssh \-l ssh\-user\(dq\& rsync\-user@host::module /dest -.fi - -.PP -The \(dq\&ssh\-user\(dq\& will be used at the ssh level; the \(dq\&rsync\-user\(dq\& will be -used to log\-in to the \(dq\&module\(dq\&. -.PP +system (other than what is already required to allow remote-shell access). +Rsync supports connecting to a host using a remote shell and then spawning a +single-use "daemon" server that expects to read its config file in the home dir +of the remote user. This can be useful if you want to encrypt a daemon-style +transfer's data, but since the daemon is started up fresh by the remote user, +you may not be able to use features such as chroot or change the uid used by +the daemon. (For another way to encrypt a daemon transfer, consider using ssh +to tunnel a local port to a remote machine and configure a normal rsync daemon +on that remote host to only allow connections from "localhost".) +.P +From the user's perspective, a daemon transfer via a remote-shell connection +uses nearly the same command-line syntax as a normal rsync-daemon transfer, +with the only exception being that you must explicitly set the remote shell +program on the command-line with the \fB\-\-rsh=COMMAND\fP option. (Setting the +RSYNC_RSH in the environment will not turn on this functionality.) For example: +.RS 4 +.P +.nf +rsync -av --rsh=ssh host::module /dest +.fi +.RE +.P +If you need to specify a different remote-shell user, keep in mind that the +user@ prefix in front of the host is specifying the rsync-user value (for a +module that requires user-based authentication). This means that you must give +the '\-l user' option to ssh when specifying the remote-shell, as in this +example that uses the short version of the \fB\-\-rsh\fP option: +.RS 4 +.P +.nf +rsync -av -e "ssh -l ssh-user" rsync-user@host::module /dest +.fi +.RE +.P +The "ssh-user" will be used at the ssh level; the "rsync-user" will be used to +log-in to the "module". +.P .SH "STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS" - -.PP +.P In order to connect to an rsync daemon, the remote system needs to have a -daemon already running (or it needs to have configured something like inetd -to spawn an rsync daemon for incoming connections on a particular port). -For full information on how to start a daemon that will handling incoming -socket connections, see the \fBrsyncd.conf\fP(5) man page \-\- that is the config -file for the daemon, and it contains the full details for how to run the -daemon (including stand\-alone and inetd configurations). -.PP -If you\(cq\&re using one of the remote\-shell transports for the transfer, there is +daemon already running (or it needs to have configured something like inetd to +spawn an rsync daemon for incoming connections on a particular port). For full +information on how to start a daemon that will handling incoming socket +connections, see the \fBrsyncd.conf\fP(5) man page\ \-\- that is the config file for +the daemon, and it contains the full details for how to run the daemon +(including stand-alone and inetd configurations). +.P +If you're using one of the remote-shell transports for the transfer, there is no need to manually start an rsync daemon. -.PP +.P .SH "SORTED TRANSFER ORDER" - -.PP +.P Rsync always sorts the specified filenames into its internal transfer list. This handles the merging together of the contents of identically named directories, makes it easy to remove duplicate filenames, and may confuse -someone when the files are transferred in a different order than what was -given on the command\-line. -.PP +someone when the files are transferred in a different order than what was given +on the command-line. +.P If you need a particular file to be transferred prior to another, either separate the files into different rsync calls, or consider using -\fB\-\-delay\-updates\fP (which doesn\(cq\&t affect the sorted transfer order, but -does make the final file\-updating phase happen much more rapidly). -.PP +\fB\-\-delay-updates\fP (which doesn't affect the sorted transfer order, but does +make the final file-updating phase happen much more rapidly). +.P .SH "EXAMPLES" - -.PP +.P Here are some examples of how I use rsync. -.PP -To backup my wife\(cq\&s home directory, which consists of large MS Word -files and mail folders, I use a cron job that runs -.PP -.RS -\f(CWrsync \-Cavz . arvidsjaur:backup\fP +.P +To backup my wife's home directory, which consists of large MS Word files and +mail folders, I use a cron job that runs +.RS 4 +.P +.nf +rsync -Cavz . arvidsjaur:backup +.fi .RE - -.PP +.P each night over a PPP connection to a duplicate directory on my machine -\(dq\&arvidsjaur\(dq\&. -.PP -To synchronize my samba source trees I use the following Makefile -targets: -.PP -.nf - get: - rsync \-avuzb \-\-exclude '\&*~'\& samba:samba/ . - put: - rsync \-Cavuzb . samba:samba/ - sync: get put -.fi - -.PP -this allows me to sync with a CVS directory at the other end of the -connection. I then do CVS operations on the remote machine, which saves a -lot of time as the remote CVS protocol isn\(cq\&t very efficient. -.PP -I mirror a directory between my \(dq\&old\(dq\& and \(dq\&new\(dq\& ftp sites with the -command: -.PP -\f(CWrsync \-az \-e ssh \-\-delete ~ftp/pub/samba nimbus:\(dq\&~ftp/pub/tridge\(dq\&\fP -.PP +"arvidsjaur". +.P +To synchronize my samba source trees I use the following Makefile targets: +.RS 4 +.P +.nf +get: + rsync -avuzb --exclude '*~' samba:samba/ . +put: + rsync -Cavuzb . samba:samba/ +sync: get put +.fi +.RE +.P +This allows me to sync with a CVS directory at the other end of the connection. +I then do CVS operations on the remote machine, which saves a lot of time as +the remote CVS protocol isn't very efficient. +.P +I mirror a directory between my "old" and "new" ftp sites with the command: +.RS 4 +.P +.nf +rsync -az -e ssh --delete ~ftp/pub/samba nimbus:"~ftp/pub/tridge" +.fi +.RE +.P This is launched from cron every few hours. -.PP -.SH "OPTIONS SUMMARY" - -.PP -Here is a short summary of the options available in rsync. Please refer -to the detailed description below for a complete description. -.nf - - \-v, \-\-verbose increase verbosity - \-\-info=FLAGS fine\-grained informational verbosity - \-\-debug=FLAGS fine\-grained debug verbosity - \-\-msgs2stderr special output handling for debugging - \-q, \-\-quiet suppress non\-error messages - \-\-no\-motd suppress daemon\-mode MOTD (see caveat) - \-c, \-\-checksum skip based on checksum, not mod\-time & size - \-a, \-\-archive archive mode; equals \-rlptgoD (no \-H,\-A,\-X) - \-\-no\-OPTION turn off an implied OPTION (e.g. \-\-no\-D) - \-r, \-\-recursive recurse into directories - \-R, \-\-relative use relative path names - \-\-no\-implied\-dirs don'\&t send implied dirs with \-\-relative - \-b, \-\-backup make backups (see \-\-suffix & \-\-backup\-dir) - \-\-backup\-dir=DIR make backups into hierarchy based in DIR - \-\-suffix=SUFFIX backup suffix (default ~ w/o \-\-backup\-dir) - \-u, \-\-update skip files that are newer on the receiver - \-\-inplace update destination files in\-place - \-\-append append data onto shorter files - \-\-append\-verify \-\-append w/old data in file checksum - \-d, \-\-dirs transfer directories without recursing - \-l, \-\-links copy symlinks as symlinks - \-L, \-\-copy\-links transform symlink into referent file/dir - \-\-copy\-unsafe\-links only \(dq\&unsafe\(dq\& symlinks are transformed - \-\-safe\-links ignore symlinks that point outside the tree - \-\-munge\-links munge symlinks to make them safer - \-k, \-\-copy\-dirlinks transform symlink to dir into referent dir - \-K, \-\-keep\-dirlinks treat symlinked dir on receiver as dir - \-H, \-\-hard\-links preserve hard links - \-p, \-\-perms preserve permissions - \-E, \-\-executability preserve executability - \-\-chmod=CHMOD affect file and/or directory permissions - \-A, \-\-acls preserve ACLs (implies \-p) - \-X, \-\-xattrs preserve extended attributes - \-o, \-\-owner preserve owner (super\-user only) - \-g, \-\-group preserve group - \-\-devices preserve device files (super\-user only) - \-\-specials preserve special files - \-D same as \-\-devices \-\-specials - \-t, \-\-times preserve modification times - \-O, \-\-omit\-dir\-times omit directories from \-\-times - \-J, \-\-omit\-link\-times omit symlinks from \-\-times - \-\-super receiver attempts super\-user activities - \-\-fake\-super store/recover privileged attrs using xattrs - \-S, \-\-sparse turn sequences of nulls into sparse blocks - \-\-preallocate allocate dest files before writing - \-n, \-\-dry\-run perform a trial run with no changes made - \-W, \-\-whole\-file copy files whole (w/o delta\-xfer algorithm) - \-\-checksum\-choice=STR choose the checksum algorithms - \-x, \-\-one\-file\-system don'\&t cross filesystem boundaries - \-B, \-\-block\-size=SIZE force a fixed checksum block\-size - \-e, \-\-rsh=COMMAND specify the remote shell to use - \-\-rsync\-path=PROGRAM specify the rsync to run on remote machine - \-\-existing skip creating new files on receiver - \-\-ignore\-existing skip updating files that exist on receiver - \-\-remove\-source\-files sender removes synchronized files (non\-dir) - \-\-del an alias for \-\-delete\-during - \-\-delete delete extraneous files from dest dirs - \-\-delete\-before receiver deletes before xfer, not during - \-\-delete\-during receiver deletes during the transfer - \-\-delete\-delay find deletions during, delete after - \-\-delete\-after receiver deletes after transfer, not during - \-\-delete\-excluded also delete excluded files from dest dirs - \-\-ignore\-missing\-args ignore missing source args without error - \-\-delete\-missing\-args delete missing source args from destination - \-\-ignore\-errors delete even if there are I/O errors - \-\-force force deletion of dirs even if not empty - \-\-max\-delete=NUM don'\&t delete more than NUM files - \-\-max\-size=SIZE don'\&t transfer any file larger than SIZE - \-\-min\-size=SIZE don'\&t transfer any file smaller than SIZE - \-\-partial keep partially transferred files - \-\-partial\-dir=DIR put a partially transferred file into DIR - \-\-delay\-updates put all updated files into place at end - \-m, \-\-prune\-empty\-dirs prune empty directory chains from file\-list - \-\-numeric\-ids don'\&t map uid/gid values by user/group name - \-\-usermap=STRING custom username mapping - \-\-groupmap=STRING custom groupname mapping - \-\-chown=USER:GROUP simple username/groupname mapping - \-\-timeout=SECONDS set I/O timeout in seconds - \-\-contimeout=SECONDS set daemon connection timeout in seconds - \-I, \-\-ignore\-times don'\&t skip files that match size and time - \-\-size\-only skip files that match in size - \-@, \-\-modify\-window=NUM set the accuracy for mod\-time comparisons - \-T, \-\-temp\-dir=DIR create temporary files in directory DIR - \-y, \-\-fuzzy find similar file for basis if no dest file - \-\-compare\-dest=DIR also compare received files relative to DIR - \-\-copy\-dest=DIR ... and include copies of unchanged files - \-\-link\-dest=DIR hardlink to files in DIR when unchanged - \-z, \-\-compress compress file data during the transfer - \-\-compress\-level=NUM explicitly set compression level - \-\-skip\-compress=LIST skip compressing files with suffix in LIST - \-C, \-\-cvs\-exclude auto\-ignore files in the same way CVS does - \-f, \-\-filter=RULE add a file\-filtering RULE - \-F same as \-\-filter='\&dir\-merge /.rsync\-filter'\& - repeated: \-\-filter='\&\- .rsync\-filter'\& - \-\-exclude=PATTERN exclude files matching PATTERN - \-\-exclude\-from=FILE read exclude patterns from FILE - \-\-include=PATTERN don'\&t exclude files matching PATTERN - \-\-include\-from=FILE read include patterns from FILE - \-\-files\-from=FILE read list of source\-file names from FILE - \-0, \-\-from0 all *from/filter files are delimited by 0s - \-s, \-\-protect\-args no space\-splitting; wildcard chars only - \-\-address=ADDRESS bind address for outgoing socket to daemon - \-\-port=PORT specify double\-colon alternate port number - \-\-sockopts=OPTIONS specify custom TCP options - \-\-blocking\-io use blocking I/O for the remote shell - \-\-outbuf=N|L|B set out buffering to None, Line, or Block - \-\-stats give some file\-transfer stats - \-8, \-\-8\-bit\-output leave high\-bit chars unescaped in output - \-h, \-\-human\-readable output numbers in a human\-readable format - \-\-progress show progress during transfer - \-P same as \-\-partial \-\-progress - \-i, \-\-itemize\-changes output a change\-summary for all updates - \-M, \-\-remote\-option=OPTION send OPTION to the remote side only - \-\-out\-format=FORMAT output updates using the specified FORMAT - \-\-log\-file=FILE log what we'\&re doing to the specified FILE - \-\-log\-file\-format=FMT log updates using the specified FMT - \-\-password\-file=FILE read daemon\-access password from FILE - \-\-list\-only list the files instead of copying them - \-\-bwlimit=RATE limit socket I/O bandwidth - \-\-write\-batch=FILE write a batched update to FILE - \-\-only\-write\-batch=FILE like \-\-write\-batch but w/o updating dest - \-\-read\-batch=FILE read a batched update from FILE - \-\-protocol=NUM force an older protocol version to be used - \-\-iconv=CONVERT_SPEC request charset conversion of filenames - \-\-checksum\-seed=NUM set block/file checksum seed (advanced) - \-4, \-\-ipv4 prefer IPv4 - \-6, \-\-ipv6 prefer IPv6 - \-\-version print version number -(\-h) \-\-help show this help (see below for \-h comment) -.fi - -.PP +.P +.SH "OPTION SUMMARY" +.P +Here is a short summary of the options available in rsync. Please refer to the +detailed description below for a complete description. +.P +.nf +--verbose, -v increase verbosity +--info=FLAGS fine-grained informational verbosity +--debug=FLAGS fine-grained debug verbosity +--msgs2stderr output messages directly to stderr +--quiet, -q suppress non-error messages +--no-motd suppress daemon-mode MOTD +--checksum, -c skip based on checksum, not mod-time & size +--archive, -a archive mode; equals -rlptgoD (no -H,-A,-X) +--no-OPTION turn off an implied OPTION (e.g. --no-D) +--recursive, -r recurse into directories +--relative, -R use relative path names +--no-implied-dirs don't send implied dirs with --relative +--backup, -b make backups (see --suffix & --backup-dir) +--backup-dir=DIR make backups into hierarchy based in DIR +--suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) +--update, -u skip files that are newer on the receiver +--inplace update destination files in-place +--append append data onto shorter files +--append-verify --append w/old data in file checksum +--dirs, -d transfer directories without recursing +--links, -l copy symlinks as symlinks +--copy-links, -L transform symlink into referent file/dir +--copy-unsafe-links only "unsafe" symlinks are transformed +--safe-links ignore symlinks that point outside the tree +--munge-links munge symlinks to make them safe & unusable +--copy-dirlinks, -k transform symlink to dir into referent dir +--keep-dirlinks, -K treat symlinked dir on receiver as dir +--hard-links, -H preserve hard links +--perms, -p preserve permissions +--executability, -E preserve executability +--chmod=CHMOD affect file and/or directory permissions +--acls, -A preserve ACLs (implies --perms) +--xattrs, -X preserve extended attributes +--owner, -o preserve owner (super-user only) +--group, -g preserve group +--devices preserve device files (super-user only) +--specials preserve special files +-D same as --devices --specials +--times, -t preserve modification times +--atimes, -U preserve access (use) times +--open-noatime avoid changing the atime on opened files +--omit-dir-times, -O omit directories from --times +--omit-link-times, -J omit symlinks from --times +--super receiver attempts super-user activities +--fake-super store/recover privileged attrs using xattrs +--sparse, -S turn sequences of nulls into sparse blocks +--preallocate allocate dest files before writing them +--write-devices write to devices as files (implies --inplace) +--dry-run, -n perform a trial run with no changes made +--whole-file, -W copy files whole (w/o delta-xfer algorithm) +--checksum-choice=STR choose the checksum algorithms +--one-file-system, -x don't cross filesystem boundaries +--block-size=SIZE, -B force a fixed checksum block-size +--rsh=COMMAND, -e specify the remote shell to use +--rsync-path=PROGRAM specify the rsync to run on remote machine +--existing skip creating new files on receiver +--ignore-existing skip updating files that exist on receiver +--remove-source-files sender removes synchronized files (non-dir) +--del an alias for --delete-during +--delete delete extraneous files from dest dirs +--delete-before receiver deletes before xfer, not during +--delete-during receiver deletes during the transfer +--delete-delay find deletions during, delete after +--delete-after receiver deletes after transfer, not during +--delete-excluded also delete excluded files from dest dirs +--ignore-missing-args ignore missing source args without error +--delete-missing-args delete missing source args from destination +--ignore-errors delete even if there are I/O errors +--force force deletion of dirs even if not empty +--max-delete=NUM don't delete more than NUM files +--max-size=SIZE don't transfer any file larger than SIZE +--min-size=SIZE don't transfer any file smaller than SIZE +--partial keep partially transferred files +--partial-dir=DIR put a partially transferred file into DIR +--delay-updates put all updated files into place at end +--prune-empty-dirs, -m prune empty directory chains from file-list +--numeric-ids don't map uid/gid values by user/group name +--usermap=STRING custom username mapping +--groupmap=STRING custom groupname mapping +--chown=USER:GROUP simple username/groupname mapping +--timeout=SECONDS set I/O timeout in seconds +--contimeout=SECONDS set daemon connection timeout in seconds +--ignore-times, -I don't skip files that match size and time +--size-only skip files that match in size +--modify-window=NUM, -@ set the accuracy for mod-time comparisons +--temp-dir=DIR, -T create temporary files in directory DIR +--fuzzy, -y find similar file for basis if no dest file +--compare-dest=DIR also compare destination files relative to DIR +--copy-dest=DIR ... and include copies of unchanged files +--link-dest=DIR hardlink to files in DIR when unchanged +--compress, -z compress file data during the transfer +--compress-level=NUM explicitly set compression level +--skip-compress=LIST skip compressing files with suffix in LIST +--cvs-exclude, -C auto-ignore files in the same way CVS does +--filter=RULE, -f add a file-filtering RULE +-F same as --filter='dir-merge /.rsync-filter' + repeated: --filter='- .rsync-filter' +--exclude=PATTERN exclude files matching PATTERN +--exclude-from=FILE read exclude patterns from FILE +--include=PATTERN don't exclude files matching PATTERN +--include-from=FILE read include patterns from FILE +--files-from=FILE read list of source-file names from FILE +--from0, -0 all *-from/filter files are delimited by 0s +--protect-args, -s no space-splitting; wildcard chars only +--copy-as=USER[:GROUP] specify user & optional group for the copy +--address=ADDRESS bind address for outgoing socket to daemon +--port=PORT specify double-colon alternate port number +--sockopts=OPTIONS specify custom TCP options +--blocking-io use blocking I/O for the remote shell +--outbuf=N|L|B set out buffering to None, Line, or Block +--stats give some file-transfer stats +--8-bit-output, -8 leave high-bit chars unescaped in output +--human-readable, -h output numbers in a human-readable format +--progress show progress during transfer +-P same as --partial --progress +--itemize-changes, -i output a change-summary for all updates +--remote-option=OPT, -M send OPTION to the remote side only +--out-format=FORMAT output updates using the specified FORMAT +--log-file=FILE log what we're doing to the specified FILE +--log-file-format=FMT log updates using the specified FMT +--password-file=FILE read daemon-access password from FILE +--list-only list the files instead of copying them +--bwlimit=RATE limit socket I/O bandwidth +--write-batch=FILE write a batched update to FILE +--only-write-batch=FILE like --write-batch but w/o updating dest +--read-batch=FILE read a batched update from FILE +--protocol=NUM force an older protocol version to be used +--iconv=CONVERT_SPEC request charset conversion of filenames +--checksum-seed=NUM set block/file checksum seed (advanced) +--ipv4, -4 prefer IPv4 +--ipv6, -6 prefer IPv6 +--version, -V print the version + other info and exit +--help, -h (*) show this help (* -h is help only on its own) +.fi +.P Rsync can also be run as a daemon, in which case the following options are -accepted: -.nf - - \-\-daemon run as an rsync daemon - \-\-address=ADDRESS bind to the specified address - \-\-bwlimit=RATE limit socket I/O bandwidth - \-\-config=FILE specify alternate rsyncd.conf file - \-M, \-\-dparam=OVERRIDE override global daemon config parameter - \-\-no\-detach do not detach from the parent - \-\-port=PORT listen on alternate port number - \-\-log\-file=FILE override the \(dq\&log file\(dq\& setting - \-\-log\-file\-format=FMT override the \(dq\&log format\(dq\& setting - \-\-sockopts=OPTIONS specify custom TCP options - \-v, \-\-verbose increase verbosity - \-4, \-\-ipv4 prefer IPv4 - \-6, \-\-ipv6 prefer IPv6 - \-h, \-\-help show this help (if used after \-\-daemon) -.fi - -.PP +accepted: +.P +.nf +--daemon run as an rsync daemon +--address=ADDRESS bind to the specified address +--bwlimit=RATE limit socket I/O bandwidth +--config=FILE specify alternate rsyncd.conf file +--dparam=OVERRIDE, -M override global daemon config parameter +--no-detach do not detach from the parent +--port=PORT listen on alternate port number +--log-file=FILE override the "log file" setting +--log-file-format=FMT override the "log format" setting +--sockopts=OPTIONS specify custom TCP options +--verbose, -v increase verbosity +--ipv4, -4 prefer IPv4 +--ipv6, -6 prefer IPv6 +--help, -h show this help (when used with --daemon) +.fi +.P .SH "OPTIONS" - -.PP -Rsync accepts both long (double\-dash + word) and short (single\-dash + letter) +.P +Rsync accepts both long (double-dash + word) and short (single-dash + letter) options. The full list of the available options are described below. If an -option can be specified in more than one way, the choices are comma\-separated. +option can be specified in more than one way, the choices are comma-separated. Some options only have a long variant, not a short. If the option takes a parameter, the parameter is only listed after the long variant, even though it must also be specified for the short. When specifying a parameter, you can -either use the form \-\-option=param or replace the \(cq\&=\(cq\& with whitespace. The -parameter may need to be quoted in some manner for it to survive the shell\(cq\&s -command\-line parsing. Keep in mind that a leading tilde (~) in a filename is -substituted by your shell, so \-\-option=~/foo will not change the tilde into -your home directory (remove the \(cq\&=\(cq\& for that). -.PP -.IP "\fB\-\-help\fP" -Print a short help page describing the options -available in rsync and exit. For backward\-compatibility with older -versions of rsync, the help will also be output if you use the \fB\-h\fP -option without any other args. -.IP -.IP "\fB\-\-version\fP" -print the rsync version number and exit. -.IP -.IP "\fB\-v, \-\-verbose\fP" -This option increases the amount of information you -are given during the transfer. By default, rsync works silently. A -single \fB\-v\fP will give you information about what files are being -transferred and a brief summary at the end. Two \fB\-v\fP options will give you -information on what files are being skipped and slightly more -information at the end. More than two \fB\-v\fP options should only be used if -you are debugging rsync. -.IP +either use the form \fB\-\-option=param\fP or replace the '=' with whitespace. The +parameter may need to be quoted in some manner for it to survive the shell's +command-line parsing. Keep in mind that a leading tilde (\fB~\fP) in a filename is +substituted by your shell, so \fB\-\-option=~/foo\fP will not change the tilde into +your home directory (remove the '=' for that). +.P +.IP "\fB\-\-help\fP, \fB\-h\fP \fB(*)\fP" +Print a short help page describing the options available in rsync and exit. +(*) The \fB\-h\fP short option will only invoke \fB\-\-help\fP when used without other +options since it normally means \fB\-\-human-readable\fP. +.IP "\fB\-\-version\fP, \fB\-V\fP" +Print the rsync version plus other info and exit. +.IP +The output includes the default list of checksum algorithms, the default +list of compression algorithms, a list of compiled-in capabilities, a link +to the rsync web site, and some license/copyright info. +.IP "\fB\-\-verbose\fP, \fB\-v\fP" +This option increases the amount of information you are given during the +transfer. By default, rsync works silently. A single \fB\-v\fP will give you +information about what files are being transferred and a brief summary at +the end. Two \fB\-v\fP options will give you information on what files are +being skipped and slightly more information at the end. More than two \fB\-v\fP +options should only be used if you are debugging rsync. +.IP In a modern rsync, the \fB\-v\fP option is equivalent to the setting of groups of \fB\-\-info\fP and \fB\-\-debug\fP options. You can choose to use these newer options in addition to, or in place of using \fB\-\-verbose\fP, as any -fine\-grained settings override the implied settings of \fB\-v\fP. Both -\fB\-\-info\fP and \fB\-\-debug\fP have a way to ask for help that tells you -exactly what flags are set for each increase in verbosity. -.IP -However, do keep in mind that a daemon\(cq\&s \(dq\&max verbosity\(dq\& setting will limit how -high of a level the various individual flags can be set on the daemon side. -For instance, if the max is 2, then any info and/or debug flag that is set to -a higher value than what would be set by \fB\-vv\fP will be downgraded to the -\fB\-vv\fP level in the daemon\(cq\&s logging. -.IP +fine-grained settings override the implied settings of \fB\-v\fP. Both \fB\-\-info\fP +and \fB\-\-debug\fP have a way to ask for help that tells you exactly what flags +are set for each increase in verbosity. +.IP +However, do keep in mind that a daemon's "\fBmax\ verbosity\fP" setting will limit +how high of a level the various individual flags can be set on the daemon +side. For instance, if the max is 2, then any info and/or debug flag that +is set to a higher value than what would be set by \fB\-vv\fP will be downgraded +to the \fB\-vv\fP level in the daemon's logging. .IP "\fB\-\-info=FLAGS\fP" -This option lets you have fine\-grained control over the -information -output you want to see. An individual flag name may be followed by a level +This option lets you have fine-grained control over the information output +you want to see. An individual flag name may be followed by a level number, with 0 meaning to silence that output, 1 being the default output level, and higher numbers increasing the output of that flag (for those -that support higher levels). Use -\fB\-\-info=help\fP -to see all the available flag names, what they output, and what flag names -are added for each increase in the verbose level. Some examples: -.IP -.nf - rsync \-a \-\-info=progress2 src/ dest/ - rsync \-avv \-\-info=stats2,misc1,flist0 src/ dest/ -.fi - -.IP -Note that \fB\-\-info=name\fP\(cq\&s output is affected by the \fB\-\-out\-format\fP and -\fB\-\-itemize\-changes\fP (\fB\-i\fP) options. See those options for more -information on what is output and when. -.IP +that support higher levels). Use \fB\-\-info=help\fP to see all the available +flag names, what they output, and what flag names are added for each +increase in the verbose level. Some examples: +.RS 4 +.IP +.nf +rsync -a --info=progress2 src/ dest/ +rsync -avv --info=stats2,misc1,flist0 src/ dest/ +.fi +.RE +.IP +Note that \fB\-\-info=name\fP's output is affected by the \fB\-\-out-format\fP and +\fB\-\-itemize-changes\fP (\fB\-i\fP) options. See those options for more information +on what is output and when. +.IP This option was added to 3.1.0, so an older rsync on the server side might -reject your attempts at fine\-grained control (if one or more flags needed +reject your attempts at fine-grained control (if one or more flags needed to be send to the server and the server was too old to understand them). -See also the \(dq\&max verbosity\(dq\& caveat above when dealing with a daemon. -.IP +See also the "\fBmax\ verbosity\fP" caveat above when dealing with a daemon. .IP "\fB\-\-debug=FLAGS\fP" -This option lets you have fine\-grained control over the debug -output you want to see. An individual flag name may be followed by a level -number, with 0 meaning to silence that output, 1 being the default output -level, and higher numbers increasing the output of that flag (for those -that support higher levels). Use -\fB\-\-debug=help\fP -to see all the available flag names, what they output, and what flag names -are added for each increase in the verbose level. Some examples: -.IP -.nf - rsync \-avvv \-\-debug=none src/ dest/ - rsync \-avA \-\-del \-\-debug=del2,acl src/ dest/ -.fi - -.IP +This option lets you have fine-grained control over the debug output you +want to see. An individual flag name may be followed by a level number, +with 0 meaning to silence that output, 1 being the default output level, +and higher numbers increasing the output of that flag (for those that +support higher levels). Use \fB\-\-debug=help\fP to see all the available flag +names, what they output, and what flag names are added for each increase in +the verbose level. Some examples: +.RS 4 +.IP +.nf +rsync -avvv --debug=none src/ dest/ +rsync -avA --del --debug=del2,acl src/ dest/ +.fi +.RE +.IP Note that some debug messages will only be output when \fB\-\-msgs2stderr\fP is specified, especially those pertaining to I/O and buffer debugging. -.IP -This option was added to 3.1.0, so an older rsync on the server side might -reject your attempts at fine\-grained control (if one or more flags needed -to be send to the server and the server was too old to understand them). -See also the \(dq\&max verbosity\(dq\& caveat above when dealing with a daemon. -.IP +.IP +Beginning in 3.2.0, this option is no longer auto-forwared to the server +side in order to allow you to specify different debug values for each side +of the transfer, as well as to specify a new debug option that is only +present in one of the rsync versions. If you want to duplicate the same +option on both sides, using brace expansion is an easy way to save you some +typing. This works in zsh and bash: +.RS 4 +.IP +.nf +rsync -aiv {-M,}--debug=del2 src/ dest/ +.fi +.RE .IP "\fB\-\-msgs2stderr\fP" -This option changes rsync to send all its output -directly to stderr rather than to send messages to the client side via the -protocol (which normally outputs info messages via stdout). This is mainly -intended for debugging in order to avoid changing the data sent via the -protocol, since the extra protocol data can change what is being tested. +This option changes rsync to send all its output directly to stderr rather +than to send messages to the client side via the protocol. The protocol +allows rsync to output normal messages via stdout and errors via stderr, +but it can delay messages behind a slew of data. +.IP +One case where this is helpful is when sending really large files, since +errors that happen on a remote receiver tend to get delayed until after the +file's data is fully sent. It is also helpful for debugging, since it +helps to avoid overpopulating the protocol data with extra message data. +.IP The option does not affect the remote side of a transfer without using -\fB\-\-remote\-option\fP \-\- e.g. \fB\-M\-\-msgs2stderr\fP. -Also keep in mind that a daemon connection does not have a stderr channel to send -messages back to the client side, so if you are doing any daemon\-transfer -debugging using this option, you should start up a daemon using \fB\-\-no\-detach\fP -so that you can see the stderr output on the daemon side. -.IP -This option has the side\-effect of making stderr output get line\-buffered so -that the merging of the output of 3 programs happens in a more readable manner. -.IP -.IP "\fB\-q, \-\-quiet\fP" -This option decreases the amount of information you -are given during the transfer, notably suppressing information messages -from the remote server. This option is useful when invoking rsync from -cron. -.IP -.IP "\fB\-\-no\-motd\fP" -This option affects the information that is output -by the client at the start of a daemon transfer. This suppresses the -message\-of\-the\-day (MOTD) text, but it also affects the list of modules -that the daemon sends in response to the \(dq\&rsync host::\(dq\& request (due to -a limitation in the rsync protocol), so omit this option if you want to -request the list of modules from the daemon. -.IP -.IP "\fB\-I, \-\-ignore\-times\fP" -Normally rsync will skip any files that are -already the same size and have the same modification timestamp. -This option turns off this \(dq\&quick check\(dq\& behavior, causing all files to -be updated. -.IP -.IP "\fB\-\-size\-only\fP" -This modifies rsync\(cq\&s \(dq\&quick check\(dq\& algorithm for -finding files that need to be transferred, changing it from the default of -transferring files with either a changed size or a changed last\-modified -time to just looking for files that have changed in size. This is useful -when starting to use rsync after using another mirroring system which may -not preserve timestamps exactly. -.IP -.IP "\fB\-@, \-\-modify\-window\fP" -When comparing two timestamps, rsync treats the -timestamps as being equal if they differ by no more than the modify\-window -value. The default is 0, which matches just integer seconds. If you specify a -negative value (and the receiver is at least version 3.1.3) then nanoseconds -will also be taken into account. Specifying 1 is useful for copies to/from MS -Windows FAT filesystems, because FAT represents times with a 2\-second -resolution (allowing times to differ from the original by up to 1 second). -.IP +\fB\-\-remote-option\fP\ \-\- e.g. \fB\-M\-\-msgs2stderr\fP or \fB{\-M,}\-\-msgs2stderr\fP. +.IP +Also keep in mind that connecting to a normal (non-remote-shell) daemon +does not have a stderr channel to send messages back to the client side, so +a modern rsync only allows the option on a remote-shell-run daemon. +.IP +This option has the side-effect of making stderr output get line-buffered +so that the merging of the output of 3 programs happens in a more readable +manner. +.IP "\fB\-\-quiet\fP, \fB\-q\fP" +This option decreases the amount of information you are given during the +transfer, notably suppressing information messages from the remote server. +This option is useful when invoking rsync from cron. +.IP "\fB\-\-no-motd\fP" +This option affects the information that is output by the client at the +start of a daemon transfer. This suppresses the message-of-the-day (MOTD) +text, but it also affects the list of modules that the daemon sends in +response to the "rsync host::" request (due to a limitation in the rsync +protocol), so omit this option if you want to request the list of modules +from the daemon. +.IP "\fB\-\-ignore-times\fP, \fB\-I\fP" +Normally rsync will skip any files that are already the same size and have +the same modification timestamp. This option turns off this "quick check" +behavior, causing all files to be updated. +.IP "\fB\-\-size-only\fP" +This modifies rsync's "quick check" algorithm for finding files that need +to be transferred, changing it from the default of transferring files with +either a changed size or a changed last-modified time to just looking for +files that have changed in size. This is useful when starting to use rsync +after using another mirroring system which may not preserve timestamps +exactly. +.IP "\fB\-\-modify-window\fP, \fB\-@\fP" +When comparing two timestamps, rsync treats the timestamps as being equal +if they differ by no more than the modify-window value. The default is 0, +which matches just integer seconds. If you specify a negative value (and +the receiver is at least version 3.1.3) then nanoseconds will also be taken +into account. Specifying 1 is useful for copies to/from MS Windows FAT +filesystems, because FAT represents times with a 2-second resolution +(allowing times to differ from the original by up to 1 second). +.IP If you want all your transfers to default to comparing nanoseconds, you can -create a ~/.popt file and put these lines in it: -.IP -.RS -\f(CW rsync alias \-a \-a@\-1\fP -.RE -.RS -\f(CW rsync alias \-t \-t@\-1\fP +create a \fB~/.popt\fP file and put these lines in it: +.RS 4 +.IP +.nf +rsync alias -a -a@-1 +rsync alias -t -t@-1 +.fi .RE - -.IP -With that as the default, you\(cq\&d need to specify \fB\-\-modify\-window=0\fP (aka -\fB\-@0\fP) to override it and ignore nanoseconds, e.g. if you\(cq\&re copying between -ext3 and ext4, or if the receiving rsync is older than 3.1.3. -.IP -.IP "\fB\-c, \-\-checksum\fP" -This changes the way rsync checks if the files have -been changed and are in need of a transfer. Without this option, rsync -uses a \(dq\&quick check\(dq\& that (by default) checks if each file\(cq\&s size and time -of last modification match between the sender and receiver. This option -changes this to compare a 128\-bit checksum for each file that has a -matching size. Generating the checksums means that both sides will expend -a lot of disk I/O reading all the data in the files in the transfer (and -this is prior to any reading that will be done to transfer changed files), -so this can slow things down significantly. -.IP -The sending side generates its checksums while it is doing the file\-system +.IP +With that as the default, you'd need to specify \fB\-\-modify-window=0\fP (aka +\fB\-@0\fP) to override it and ignore nanoseconds, e.g. if you're copying +between ext3 and ext4, or if the receiving rsync is older than 3.1.3. +.IP "\fB\-\-checksum\fP, \fB\-c\fP" +This changes the way rsync checks if the files have been changed and are in +need of a transfer. Without this option, rsync uses a "quick check" that +(by default) checks if each file's size and time of last modification match +between the sender and receiver. This option changes this to compare a +128-bit checksum for each file that has a matching size. Generating the +checksums means that both sides will expend a lot of disk I/O reading all +the data in the files in the transfer, so this can slow things down +significantly (and this is prior to any reading that will be done to +transfer changed files) +.IP +The sending side generates its checksums while it is doing the file-system scan that builds the list of the available files. The receiver generates its checksums when it is scanning for changed files, and will checksum any -file that has the same size as the corresponding sender\(cq\&s file: files with +file that has the same size as the corresponding sender's file: files with either a changed size or a changed checksum are selected for transfer. -.IP -Note that rsync always verifies that each \fItransferred\fP file was -correctly reconstructed on the receiving side by checking a whole\-file -checksum that is generated as the file is transferred, but that -automatic after\-the\-transfer verification has nothing to do with this -option\(cq\&s before\-the\-transfer \(dq\&Does this file need to be updated?\(dq\& check. -.IP -For protocol 30 and beyond (first supported in 3.0.0), the checksum used is -MD5. For older protocols, the checksum used is MD4. -.IP -.IP "\fB\-a, \-\-archive\fP" -This is equivalent to \fB\-rlptgoD\fP. It is a quick -way of saying you want recursion and want to preserve almost -everything (with \-H being a notable omission). -The only exception to the above equivalence is when \fB\-\-files\-from\fP is -specified, in which case \fB\-r\fP is not implied. -.IP -Note that \fB\-a\fP \fBdoes not preserve hardlinks\fP, because -finding multiply\-linked files is expensive. You must separately -specify \fB\-H\fP. -.IP -.IP "\-\-no\-OPTION" -You may turn off one or more implied options by prefixing -the option name with \(dq\&no\-\(dq\&. Not all options may be prefixed with a \(dq\&no\-\(dq\&: -only options that are implied by other options (e.g. \fB\-\-no\-D\fP, -\fB\-\-no\-perms\fP) or have different defaults in various circumstances -(e.g. \fB\-\-no\-whole\-file\fP, \fB\-\-no\-blocking\-io\fP, \fB\-\-no\-dirs\fP). You may -specify either the short or the long option name after the \(dq\&no\-\(dq\& prefix -(e.g. \fB\-\-no\-R\fP is the same as \fB\-\-no\-relative\fP). -.IP -For example: if you want to use \fB\-a\fP (\fB\-\-archive\fP) but don\(cq\&t want -\fB\-o\fP (\fB\-\-owner\fP), instead of converting \fB\-a\fP into \fB\-rlptgD\fP, you -could specify \fB\-a \-\-no\-o\fP (or \fB\-a \-\-no\-owner\fP). -.IP -The order of the options is important: if you specify \fB\-\-no\-r \-a\fP, the -\fB\-r\fP option would end up being turned on, the opposite of \fB\-a \-\-no\-r\fP. -Note also that the side\-effects of the \fB\-\-files\-from\fP option are NOT +.IP +Note that rsync always verifies that each \fItransferred\fP file was correctly +reconstructed on the receiving side by checking a whole-file checksum that +is generated as the file is transferred, but that automatic +after-the-transfer verification has nothing to do with this option's +before-the-transfer "Does this file need to be updated?" check. +.IP +The checksum used is auto-negotiated between the client and the server, but +can be overridden using either the \fB\-\-checksum-choice\fP option or an +environment variable that is discussed in that option's section. +.IP "\fB\-\-archive\fP, \fB\-a\fP" +This is equivalent to \fB\-rlptgoD\fP. It is a quick way of saying you want +recursion and want to preserve almost everything (with \fB\-H\fP being a notable +omission). The only exception to the above equivalence is when +\fB\-\-files-from\fP is specified, in which case \fB\-r\fP is not implied. +.IP +Note that \fB\-a\fP \fBdoes not preserve hardlinks\fP, because finding +multiply-linked files is expensive. You must separately specify \fB\-H\fP. +.IP "\fB\-\-no-OPTION\fP" +You may turn off one or more implied options by prefixing the option name +with "no-". Not all options may be prefixed with a "no-": only options that +are implied by other options (e.g. \fB\-\-no-D\fP, \fB\-\-no-perms\fP) or have +different defaults in various circumstances (e.g. \fB\-\-no-whole-file\fP, +\fB\-\-no-blocking-io\fP, \fB\-\-no-dirs\fP). You may specify either the short or the +long option name after the "no-" prefix (e.g. \fB\-\-no-R\fP is the same as +\fB\-\-no-relative\fP). +.IP +For example: if you want to use \fB\-a\fP (\fB\-\-archive\fP) but don't want \fB\-o\fP +(\fB\-\-owner\fP), instead of converting \fB\-a\fP into \fB\-rlptgD\fP, you could specify +\fB\-a\ \-\-no-o\fP (or \fB\-a\ \-\-no-owner\fP). +.IP +The order of the options is important: if you specify \fB\-\-no-r\ \-a\fP, the +\fB\-r\fP option would end up being turned on, the opposite of \fB\-a\ \-\-no-r\fP. +Note also that the side-effects of the \fB\-\-files-from\fP option are NOT positional, as it affects the default state of several options and slightly -changes the meaning of \fB\-a\fP (see the \fB\-\-files\-from\fP option for more +changes the meaning of \fB\-a\fP (see the \fB\-\-files-from\fP option for more details). -.IP -.IP "\fB\-r, \-\-recursive\fP" -This tells rsync to copy directories -recursively. See also \fB\-\-dirs\fP (\fB\-d\fP). -.IP +.IP "\fB\-\-recursive\fP, \fB\-r\fP" +This tells rsync to copy directories recursively. See also \fB\-\-dirs\fP (\fB\-d\fP). +.IP Beginning with rsync 3.0.0, the recursive algorithm used is now an incremental scan that uses much less memory than before and begins the transfer after the scanning of the first few directories have been completed. This incremental scan only affects our recursion algorithm, and -does not change a non\-recursive transfer. It is also only possible when +does not change a non-recursive transfer. It is also only possible when both ends of the transfer are at least version 3.0.0. -.IP +.IP Some options require rsync to know the full file list, so these options -disable the incremental recursion mode. These include: \fB\-\-delete\-before\fP, -\fB\-\-delete\-after\fP, \fB\-\-prune\-empty\-dirs\fP, and \fB\-\-delay\-updates\fP. -Because of this, the default delete mode when you specify \fB\-\-delete\fP is now -\fB\-\-delete\-during\fP when both ends of the connection are at least 3.0.0 -(use \fB\-\-del\fP or \fB\-\-delete\-during\fP to request this improved deletion mode -explicitly). See also the \fB\-\-delete\-delay\fP option that is a better choice -than using \fB\-\-delete\-after\fP. -.IP -Incremental recursion can be disabled using the \fB\-\-no\-inc\-recursive\fP -option or its shorter \fB\-\-no\-i\-r\fP alias. -.IP -.IP "\fB\-R, \-\-relative\fP" -Use relative paths. This means that the full path -names specified on the command line are sent to the server rather than -just the last parts of the filenames. This is particularly useful when -you want to send several different directories at the same time. For -example, if you used this command: -.IP -.RS -\f(CW rsync \-av /foo/bar/baz.c remote:/tmp/\fP +disable the incremental recursion mode. These include: \fB\-\-delete-before\fP, +\fB\-\-delete-after\fP, \fB\-\-prune-empty-dirs\fP, and \fB\-\-delay-updates\fP. Because of +this, the default delete mode when you specify \fB\-\-delete\fP is now +\fB\-\-delete-during\fP when both ends of the connection are at least 3.0.0 (use +\fB\-\-del\fP or \fB\-\-delete-during\fP to request this improved deletion mode +explicitly). See also the \fB\-\-delete-delay\fP option that is a better choice +than using \fB\-\-delete-after\fP. +.IP +Incremental recursion can be disabled using the \fB\-\-no-inc-recursive\fP option +or its shorter \fB\-\-no-i-r\fP alias. +.IP "\fB\-\-relative\fP, \fB\-R\fP" +Use relative paths. This means that the full path names specified on the +command line are sent to the server rather than just the last parts of the +filenames. This is particularly useful when you want to send several +different directories at the same time. For example, if you used this +command: +.RS 4 +.IP +.nf +rsync -av /foo/bar/baz.c remote:/tmp/ +.fi .RE - -.IP -\&... this would create a file named baz.c in /tmp/ on the remote -machine. If instead you used -.IP -.RS -\f(CW rsync \-avR /foo/bar/baz.c remote:/tmp/\fP +.IP +would create a file named baz.c in /tmp/ on the remote machine. If instead +you used +.RS 4 +.IP +.nf +rsync -avR /foo/bar/baz.c remote:/tmp/ +.fi .RE - -.IP +.IP then a file named /tmp/foo/bar/baz.c would be created on the remote machine, preserving its full path. These extra path elements are called -\(dq\&implied directories\(dq\& (i.e. the \(dq\&foo\(dq\& and the \(dq\&foo/bar\(dq\& directories in the +"implied directories" (i.e. the "foo" and the "foo/bar" directories in the above example). -.IP +.IP Beginning with rsync 3.0.0, rsync always sends these implied directories as real directories in the file list, even if a path element is really a -symlink on the sending side. This prevents some really unexpected -behaviors when copying the full path of a file that you didn\(cq\&t realize had -a symlink in its path. If you want to duplicate a server\-side symlink, -include both the symlink via its path, and referent directory via its real -path. If you\(cq\&re dealing with an older rsync on the sending side, you may -need to use the \fB\-\-no\-implied\-dirs\fP option. -.IP +symlink on the sending side. This prevents some really unexpected behaviors +when copying the full path of a file that you didn't realize had a symlink +in its path. If you want to duplicate a server-side symlink, include both +the symlink via its path, and referent directory via its real path. If +you're dealing with an older rsync on the sending side, you may need to use +the \fB\-\-no-implied-dirs\fP option. +.IP It is also possible to limit the amount of path information that is sent as implied directories for each path you specify. With a modern rsync on the sending side (beginning with 2.6.7), you can insert a dot and a slash into the source path, like this: -.IP -.RS -\f(CW rsync \-avR /foo/./bar/baz.c remote:/tmp/\fP +.RS 4 +.IP +.nf +rsync -avR /foo/./bar/baz.c remote:/tmp/ +.fi .RE - -.IP -That would create /tmp/bar/baz.c on the remote machine. (Note that the -dot must be followed by a slash, so \(dq\&/foo/.\(dq\& would not be abbreviated.) -For older rsync versions, you would need to use a chdir to limit the -source path. For example, when pushing files: -.IP -.RS -\f(CW (cd /foo; rsync \-avR bar/baz.c remote:/tmp/) \fP +.IP +That would create /tmp/bar/baz.c on the remote machine. (Note that the dot +must be followed by a slash, so "/foo/." would not be abbreviated.) For +older rsync versions, you would need to use a chdir to limit the source +path. For example, when pushing files: +.RS 4 +.IP +.nf +(cd /foo; rsync -avR bar/baz.c remote:/tmp/) +.fi .RE - -.IP -(Note that the parens put the two commands into a sub\-shell, so that the -\(dq\&cd\(dq\& command doesn\(cq\&t remain in effect for future commands.) -If you\(cq\&re pulling files from an older rsync, use this idiom (but only -for a non\-daemon transfer): -.IP -.RS -\f(CW rsync \-avR \-\-rsync\-path=\(dq\&cd /foo; rsync\(dq\& \e \fP -.br -\f(CW remote:bar/baz.c /tmp/\fP +.IP +(Note that the parens put the two commands into a sub-shell, so that the +"cd" command doesn't remain in effect for future commands.) If you're +pulling files from an older rsync, use this idiom (but only for a +non-daemon transfer): +.RS 4 +.IP +.nf +rsync -avR --rsync-path="cd /foo; rsync" \\ + remote:bar/baz.c /tmp/ +.fi .RE - -.IP -.IP "\fB\-\-no\-implied\-dirs\fP" -This option affects the default behavior of the -\fB\-\-relative\fP option. When it is specified, the attributes of the implied -directories from the source names are not included in the transfer. This -means that the corresponding path elements on the destination system are -left unchanged if they exist, and any missing implied directories are -created with default attributes. This even allows these implied path -elements to have big differences, such as being a symlink to a directory on -the receiving side. -.IP -For instance, if a command\-line arg or a files\-from entry told rsync to -transfer the file \(dq\&path/foo/file\(dq\&, the directories \(dq\&path\(dq\& and \(dq\&path/foo\(dq\& -are implied when \fB\-\-relative\fP is used. If \(dq\&path/foo\(dq\& is a symlink to -\(dq\&bar\(dq\& on the destination system, the receiving rsync would ordinarily -delete \(dq\&path/foo\(dq\&, recreate it as a directory, and receive the file into -the new directory. With \fB\-\-no\-implied\-dirs\fP, the receiving rsync updates -\(dq\&path/foo/file\(dq\& using the existing path elements, which means that the file -ends up being created in \(dq\&path/bar\(dq\&. Another way to accomplish this link -preservation is to use the \fB\-\-keep\-dirlinks\fP option (which will also -affect symlinks to directories in the rest of the transfer). -.IP +.IP "\fB\-\-no-implied-dirs\fP" +This option affects the default behavior of the \fB\-\-relative\fP option. When +it is specified, the attributes of the implied directories from the source +names are not included in the transfer. This means that the corresponding +path elements on the destination system are left unchanged if they exist, +and any missing implied directories are created with default attributes. +This even allows these implied path elements to have big differences, such +as being a symlink to a directory on the receiving side. +.IP +For instance, if a command-line arg or a files-from entry told rsync to +transfer the file "path/foo/file", the directories "path" and "path/foo" +are implied when \fB\-\-relative\fP is used. If "path/foo" is a symlink to "bar" +on the destination system, the receiving rsync would ordinarily delete +"path/foo", recreate it as a directory, and receive the file into the new +directory. With \fB\-\-no-implied-dirs\fP, the receiving rsync updates +"path/foo/file" using the existing path elements, which means that the file +ends up being created in "path/bar". Another way to accomplish this link +preservation is to use the \fB\-\-keep-dirlinks\fP option (which will also affect +symlinks to directories in the rest of the transfer). +.IP When pulling files from an rsync older than 3.0.0, you may need to use this option if the sending side has a symlink in the path you request and you wish the implied directories to be transferred as normal directories. -.IP -.IP "\fB\-b, \-\-backup\fP" -With this option, preexisting destination files are -renamed as each file is transferred or deleted. You can control where the -backup file goes and what (if any) suffix gets appended using the -\fB\-\-backup\-dir\fP and \fB\-\-suffix\fP options. -.IP -Note that if you don\(cq\&t specify \fB\-\-backup\-dir\fP, (1) the -\fB\-\-omit\-dir\-times\fP option will be forced on, and (2) if \fB\-\-delete\fP is -also in effect (without \fB\-\-delete\-excluded\fP), rsync will add a \(dq\&protect\(dq\& -filter\-rule for the backup suffix to the end of all your existing excludes -(e.g. \fB\-f \(dq\&P *~\(dq\&\fP). This will prevent previously backed\-up files from being -deleted. Note that if you are supplying your own filter rules, you may -need to manually insert your own exclude/protect rule somewhere higher up -in the list so that it has a high enough priority to be effective (e.g., if -your rules specify a trailing inclusion/exclusion of \(cq\&*\(cq\&, the auto\-added -rule would never be reached). -.IP -.IP "\fB\-\-backup\-dir=DIR\fP" -In combination with the \fB\-\-backup\fP option, this -tells rsync to store all backups in the specified directory on the receiving -side. This can be used for incremental backups. You can additionally -specify a backup suffix using the \fB\-\-suffix\fP option -(otherwise the files backed up in the specified directory -will keep their original filenames). -.IP +.IP "\fB\-\-backup\fP, \fB\-b\fP" +With this option, preexisting destination files are renamed as each file is +transferred or deleted. You can control where the backup file goes and +what (if any) suffix gets appended using the \fB\-\-backup-dir\fP and \fB\-\-suffix\fP +options. +.IP +Note that if you don't specify \fB\-\-backup-dir\fP, (1) the \fB\-\-omit-dir-times\fP +option will be forced on, and (2) if \fB\-\-delete\fP is also in effect (without +\fB\-\-delete-excluded\fP), rsync will add a "protect" filter-rule for the backup +suffix to the end of all your existing excludes (e.g. \fB\-f\ "P\ *~"\fP). This +will prevent previously backed-up files from being deleted. Note that if +you are supplying your own filter rules, you may need to manually insert +your own exclude/protect rule somewhere higher up in the list so that it +has a high enough priority to be effective (e.g., if your rules specify a +trailing inclusion/exclusion of \fB*\fP, the auto-added rule would never be +reached). +.IP "\fB\-\-backup-dir=DIR\fP" +In combination with the \fB\-\-backup\fP option, this tells rsync to store all +backups in the specified directory on the receiving side. This can be used +for incremental backups. You can additionally specify a backup suffix +using the \fB\-\-suffix\fP option (otherwise the files backed up in the specified +directory will keep their original filenames). +.IP Note that if you specify a relative path, the backup directory will be relative to the destination directory, so you probably want to specify -either an absolute path or a path that starts with \(dq\&../\(dq\&. If an rsync -daemon is the receiver, the backup dir cannot go outside the module\(cq\&s path +either an absolute path or a path that starts with "../". If an rsync +daemon is the receiver, the backup dir cannot go outside the module's path hierarchy, so take extra care not to delete it or copy into it. -.IP .IP "\fB\-\-suffix=SUFFIX\fP" -This option allows you to override the default -backup suffix used with the \fB\-\-backup\fP (\fB\-b\fP) option. The default suffix is a ~ -if no \-\fB\-backup\-dir\fP was specified, otherwise it is an empty string. -.IP -.IP "\fB\-u, \-\-update\fP" -This forces rsync to skip any files which exist on -the destination and have a modified time that is newer than the source -file. (If an existing destination file has a modification time equal to the -source file\(cq\&s, it will be updated if the sizes are different.) -.IP -Note that this does not affect the copying of dirs, symlinks, or other special -files. Also, a difference of file format between the sender and receiver -is always considered to be important enough for an update, no matter what -date is on the objects. In other words, if the source has a directory -where the destination has a file, the transfer would occur regardless of -the timestamps. -.IP -This option is a transfer rule, not an exclude, so it doesn\(cq\&t affect the -data that goes into the file\-lists, and thus it doesn\(cq\&t affect deletions. +This option allows you to override the default backup suffix used with the +\fB\-\-backup\fP (\fB\-b\fP) option. The default suffix is a \fB~\fP if no \fB\-\-backup-dir\fP +was specified, otherwise it is an empty string. +.IP "\fB\-\-update\fP, \fB\-u\fP" +This forces rsync to skip any files which exist on the destination and have +a modified time that is newer than the source file. (If an existing +destination file has a modification time equal to the source file's, it +will be updated if the sizes are different.) +.IP +Note that this does not affect the copying of dirs, symlinks, or other +special files. Also, a difference of file format between the sender and +receiver is always considered to be important enough for an update, no +matter what date is on the objects. In other words, if the source has a +directory where the destination has a file, the transfer would occur +regardless of the timestamps. +.IP +This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. -.IP .IP "\fB\-\-inplace\fP" -This option changes how rsync transfers a file when -its data needs to be updated: instead of the default method of creating -a new copy of the file and moving it into place when it is complete, rsync -instead writes the updated data directly to the destination file. -.IP +This option changes how rsync transfers a file when its data needs to be +updated: instead of the default method of creating a new copy of the file +and moving it into place when it is complete, rsync instead writes the +updated data directly to the destination file. +.IP This has several effects: -.IP -.RS -.IP o +.IP +.RS +.IP o Hard links are not broken. This means the new data will be visible through other hard links to the destination file. Moreover, attempts to -copy differing source files onto a multiply\-linked destination file will -result in a \(dq\&tug of war\(dq\& with the destination data changing back and forth. -.IP o -In\-use binaries cannot be updated (either the OS will prevent this from -happening, or binaries that attempt to swap\-in their data will misbehave or -crash). -.IP o -The file\(cq\&s data will be in an inconsistent state during the transfer -and will be left that way if the transfer is interrupted or if an update +copy differing source files onto a multiply-linked destination file will +result in a "tug of war" with the destination data changing back and +forth. +.IP o +In-use binaries cannot be updated (either the OS will prevent this from +happening, or binaries that attempt to swap-in their data will misbehave +or crash). +.IP o +The file's data will be in an inconsistent state during the transfer and +will be left that way if the transfer is interrupted or if an update fails. -.IP o -A file that rsync cannot write to cannot be updated. While a super user -can update any file, a normal user needs to be granted write permission for -the open of the file for writing to be successful. -.IP o -The efficiency of rsync\(cq\&s delta\-transfer algorithm may be reduced if -some data in the destination file is overwritten before it can be copied to -a position later in the file. This does not apply if you use \fB\-\-backup\fP, -since rsync is smart enough to use the backup file as the basis file for the -transfer. +.IP o +A file that rsync cannot write to cannot be updated. While a super user +can update any file, a normal user needs to be granted write permission +for the open of the file for writing to be successful. +.IP o +The efficiency of rsync's delta-transfer algorithm may be reduced if some +data in the destination file is overwritten before it can be copied to a +position later in the file. This does not apply if you use \fB\-\-backup\fP, +since rsync is smart enough to use the backup file as the basis file for +the transfer. .RE - -.IP +.IP WARNING: you should not use this option to update files that are being accessed by others, so be careful when choosing to use this for a copy. -.IP -This option is useful for transferring large files with block\-based changes +.IP +This option is useful for transferring large files with block-based changes or appended data, and also on systems that are disk bound, not network -bound. It can also help keep a copy\-on\-write filesystem snapshot from +bound. It can also help keep a copy-on-write filesystem snapshot from diverging the entire contents of a file that only has minor changes. -.IP -The option implies \fB\-\-partial\fP (since an interrupted transfer does not delete -the file), but conflicts with \fB\-\-partial\-dir\fP and \fB\-\-delay\-updates\fP. -Prior to rsync 2.6.4 \fB\-\-inplace\fP was also incompatible with \fB\-\-compare\-dest\fP -and \fB\-\-link\-dest\fP. -.IP +.IP +The option implies \fB\-\-partial\fP (since an interrupted transfer does not +delete the file), but conflicts with \fB\-\-partial-dir\fP and \fB\-\-delay-updates\fP. +Prior to rsync 2.6.4 \fB\-\-inplace\fP was also incompatible with +\fB\-\-compare-dest\fP and \fB\-\-link-dest\fP. .IP "\fB\-\-append\fP" -This causes rsync to update a file by appending data onto -the end of the file, which presumes that the data that already exists on -the receiving side is identical with the start of the file on the sending -side. If a file needs to be transferred and its size on the receiver is -the same or longer than the size on the sender, the file is skipped. This -does not interfere with the updating of a file\(cq\&s non\-content attributes -(e.g. permissions, ownership, etc.) when the file does not need to be -transferred, nor does it affect the updating of any non\-regular files. +This causes rsync to update a file by appending data onto the end of the +file, which presumes that the data that already exists on the receiving +side is identical with the start of the file on the sending side. If a +file needs to be transferred and its size on the receiver is the same or +longer than the size on the sender, the file is skipped. This does not +interfere with the updating of a file's non-content attributes (e.g. +permissions, ownership, etc.) when the file does not need to be +transferred, nor does it affect the updating of any non-regular files. Implies \fB\-\-inplace\fP. -.IP -The use of \fB\-\-append\fP can be dangerous if you aren\(cq\&t 100% sure that the files -that are longer have only grown by the appending of data onto the end. You -should thus use include/exclude/filter rules to ensure that such a transfer is -only affecting files that you know to be growing via appended data. -.IP -.IP "\fB\-\-append\-verify\fP" -This works just like the \fB\-\-append\fP option, but -the existing data on the receiving side is included in the full\-file -checksum verification step, which will cause a file to be resent if the -final verification step fails (rsync uses a normal, non\-appending -\fB\-\-inplace\fP transfer for the resend). -.IP +.IP +The use of \fB\-\-append\fP can be dangerous if you aren't 100% sure that the +files that are longer have only grown by the appending of data onto the +end. You should thus use include/exclude/filter rules to ensure that such +a transfer is only affecting files that you know to be growing via appended +data. +.IP "\fB\-\-append-verify\fP" +This works just like the \fB\-\-append\fP option, but the existing data on the +receiving side is included in the full-file checksum verification step, +which will cause a file to be resent if the final verification step fails +(rsync uses a normal, non-appending \fB\-\-inplace\fP transfer for the resend). +It otherwise has the exact same caveats for files that have not grown +larger, so don't use this for a general copy. +.IP Note: prior to rsync 3.0.0, the \fB\-\-append\fP option worked like -\fB\-\-append\-verify\fP, so if you are interacting with an older rsync (or the +\fB\-\-append-verify\fP, so if you are interacting with an older rsync (or the transfer is using a protocol prior to 30), specifying either append option -will initiate an \fB\-\-append\-verify\fP transfer. -.IP -.IP "\fB\-d, \-\-dirs\fP" -Tell the sending side to include any directories that -are encountered. Unlike \fB\-\-recursive\fP, a directory\(cq\&s contents are not copied -unless the directory name specified is \(dq\&.\(dq\& or ends with a trailing slash -(e.g. \(dq\&.\(dq\&, \(dq\&dir/.\(dq\&, \(dq\&dir/\(dq\&, etc.). Without this option or the -\fB\-\-recursive\fP option, rsync will skip all directories it encounters (and -output a message to that effect for each one). If you specify both -\fB\-\-dirs\fP and \fB\-\-recursive\fP, \fB\-\-recursive\fP takes precedence. -.IP -The \fB\-\-dirs\fP option is implied by the \fB\-\-files\-from\fP option -or the \fB\-\-list\-only\fP option (including an implied -\fB\-\-list\-only\fP usage) if \fB\-\-recursive\fP wasn\(cq\&t specified (so that -directories are seen in the listing). Specify \fB\-\-no\-dirs\fP (or \fB\-\-no\-d\fP) -if you want to turn this off. -.IP -There is also a backward\-compatibility helper option, \fB\-\-old\-dirs\fP (or -\fB\-\-old\-d\fP) that tells rsync to use a hack of \(dq\&\-r \-\-exclude=\(cq\&/*/*\(cq\&\(dq\& to get +will initiate an \fB\-\-append-verify\fP transfer. +.IP "\fB\-\-dirs\fP, \fB\-d\fP" +Tell the sending side to include any directories that are encountered. +Unlike \fB\-\-recursive\fP, a directory's contents are not copied unless the +directory name specified is "." or ends with a trailing slash (e.g. ".", +"dir/.", "dir/", etc.). Without this option or the \fB\-\-recursive\fP option, +rsync will skip all directories it encounters (and output a message to that +effect for each one). If you specify both \fB\-\-dirs\fP and \fB\-\-recursive\fP, +\fB\-\-recursive\fP takes precedence. +.IP +The \fB\-\-dirs\fP option is implied by the \fB\-\-files-from\fP option or the +\fB\-\-list-only\fP option (including an implied \fB\-\-list-only\fP usage) if +\fB\-\-recursive\fP wasn't specified (so that directories are seen in the +listing). Specify \fB\-\-no-dirs\fP (or \fB\-\-no-d\fP) if you want to turn this off. +.IP +There is also a backward-compatibility helper option, \fB\-\-old-dirs\fP (or +\fB\-\-old-d\fP) that tells rsync to use a hack of \fB\-r\ \-\-exclude='/*/*'\fP to get an older rsync to list a single directory without recursing. -.IP -.IP "\fB\-l, \-\-links\fP" -When symlinks are encountered, recreate the -symlink on the destination. -.IP -.IP "\fB\-L, \-\-copy\-links\fP" -When symlinks are encountered, the item that -they point to (the referent) is copied, rather than the symlink. In older -versions of rsync, this option also had the side\-effect of telling the -receiving side to follow symlinks, such as symlinks to directories. In a -modern rsync such as this one, you\(cq\&ll need to specify \fB\-\-keep\-dirlinks\fP (\fB\-K\fP) -to get this extra behavior. The only exception is when sending files to -an rsync that is too old to understand \fB\-K\fP \-\- in that case, the \fB\-L\fP option -will still have the side\-effect of \fB\-K\fP on that older receiving rsync. -.IP -.IP "\fB\-\-copy\-unsafe\-links\fP" -This tells rsync to copy the referent of -symbolic links that point outside the copied tree. Absolute symlinks -are also treated like ordinary files, and so are any symlinks in the -source path itself when \fB\-\-relative\fP is used. This option has no -additional effect if \fB\-\-copy\-links\fP was also specified. -.IP -.IP "\fB\-\-safe\-links\fP" -This tells rsync to ignore any symbolic links -which point outside the copied tree. All absolute symlinks are -also ignored. Using this option in conjunction with \fB\-\-relative\fP may -give unexpected results. -.IP -.IP "\fB\-\-munge\-links\fP" -This option tells rsync to (1) modify all symlinks on -the receiving side in a way that makes them unusable but recoverable (see -below), or (2) to unmunge symlinks on the sending side that had been stored in -a munged state. This is useful if you don\(cq\&t quite trust the source of the data -to not try to slip in a symlink to a unexpected place. -.IP +.IP "\fB\-\-links\fP, \fB\-l\fP" +When symlinks are encountered, recreate the symlink on the destination. +.IP "\fB\-\-copy-links\fP, \fB\-L\fP" +When symlinks are encountered, the item that they point to (the referent) +is copied, rather than the symlink. In older versions of rsync, this +option also had the side-effect of telling the receiving side to follow +symlinks, such as symlinks to directories. In a modern rsync such as this +one, you'll need to specify \fB\-\-keep-dirlinks\fP (\fB\-K\fP) to get this extra +behavior. The only exception is when sending files to an rsync that is too +old to understand \fB\-K\fP\ \-\- in that case, the \fB\-L\fP option will still have the +side-effect of \fB\-K\fP on that older receiving rsync. +.IP "\fB\-\-copy-unsafe-links\fP" +This tells rsync to copy the referent of symbolic links that point outside +the copied tree. Absolute symlinks are also treated like ordinary files, +and so are any symlinks in the source path itself when \fB\-\-relative\fP is +used. This option has no additional effect if \fB\-\-copy-links\fP was also +specified. +.IP +Note that the cut-off point is the top of the transfer, which is the part +of the path that rsync isn't mentioning in the verbose output. If you copy +"/src/subdir" to "/dest/" then the "subdir" directory is a name inside the +transfer tree, not the top of the transfer (which is /src) so it is legal +for created relative symlinks to refer to other names inside the /src and +/dest directories. If you instead copy "/src/subdir/" (with a trailing +slash) to "/dest/subdir" that would not allow symlinks to any files outside +of "subdir". +.IP "\fB\-\-safe-links\fP" +This tells rsync to ignore any symbolic links which point outside the +copied tree. All absolute symlinks are also ignored. Using this option in +conjunction with \fB\-\-relative\fP may give unexpected results. +.IP "\fB\-\-munge-links\fP" +This option tells rsync to (1) modify all symlinks on the receiving side in +a way that makes them unusable but recoverable (see below), or (2) to +unmunge symlinks on the sending side that had been stored in a munged +state. This is useful if you don't quite trust the source of the data to +not try to slip in a symlink to a unexpected place. +.IP The way rsync disables the use of symlinks is to prefix each one with the -string \(dq\&/rsyncd\-munged/\(dq\&. This prevents the links from being used as long as -that directory does not exist. When this option is enabled, rsync will refuse -to run if that path is a directory or a symlink to a directory. -.IP -The option only affects the client side of the transfer, so if you need it to -affect the server, specify it via \fB\-\-remote\-option\fP. (Note that in a local -transfer, the client side is the sender.) -.IP -This option has no affect on a daemon, since the daemon configures whether it -wants munged symlinks via its \(dq\&munge symlinks\(dq\& parameter. See also the -\(dq\&munge\-symlinks\(dq\& perl script in the support directory of the source code. -.IP -.IP "\fB\-k, \-\-copy\-dirlinks\fP" -This option causes the sending side to treat -a symlink to a directory as though it were a real directory. This is -useful if you don\(cq\&t want symlinks to non\-directories to be affected, as -they would be using \fB\-\-copy\-links\fP. -.IP +string "/rsyncd-munged/". This prevents the links from being used as long +as that directory does not exist. When this option is enabled, rsync will +refuse to run if that path is a directory or a symlink to a directory. +.IP +The option only affects the client side of the transfer, so if you need it +to affect the server, specify it via \fB\-\-remote-option\fP. (Note that in a +local transfer, the client side is the sender.) +.IP +This option has no affect on a daemon, since the daemon configures whether +it wants munged symlinks via its "\fBmunge\ symlinks\fP" parameter. See also the +"munge-symlinks" perl script in the support directory of the source code. +.IP "\fB\-\-copy-dirlinks\fP, \fB\-k\fP" +This option causes the sending side to treat a symlink to a directory as +though it were a real directory. This is useful if you don't want symlinks +to non-directories to be affected, as they would be using \fB\-\-copy-links\fP. +.IP Without this option, if the sending side has replaced a directory with a symlink to a directory, the receiving side will delete anything that is in the way of the new symlink, including a directory hierarchy (as long as \fB\-\-force\fP or \fB\-\-delete\fP is in effect). -.IP -See also \fB\-\-keep\-dirlinks\fP for an analogous option for the receiving -side. -.IP -\fB\-\-copy\-dirlinks\fP applies to all symlinks to directories in the source. If +.IP +See also \fB\-\-keep-dirlinks\fP for an analogous option for the receiving side. +.IP +\fB\-\-copy-dirlinks\fP applies to all symlinks to directories in the source. If you want to follow only a few specified symlinks, a trick you can use is to -pass them as additional source args with a trailing slash, using \fB\-\-relative\fP -to make the paths match up right. For example: -.IP -.RS -\f(CWrsync \-r \-\-relative src/./ src/./follow\-me/ dest/\fP +pass them as additional source args with a trailing slash, using +\fB\-\-relative\fP to make the paths match up right. For example: +.RS 4 +.IP +.nf +rsync -r --relative src/./ src/./follow-me/ dest/ +.fi .RE - -.IP -This works because rsync calls \fBlstat\fP(2) on the source arg as given, and the -trailing slash makes \fBlstat\fP(2) follow the symlink, giving rise to a directory -in the file\-list which overrides the symlink found during the scan of \(dq\&src/./\(dq\&. -.IP -.IP "\fB\-K, \-\-keep\-dirlinks\fP" -This option causes the receiving side to treat -a symlink to a directory as though it were a real directory, but only if it -matches a real directory from the sender. Without this option, the -receiver\(cq\&s symlink would be deleted and replaced with a real directory. -.IP -For example, suppose you transfer a directory \(dq\&foo\(dq\& that contains a file -\(dq\&file\(dq\&, but \(dq\&foo\(dq\& is a symlink to directory \(dq\&bar\(dq\& on the receiver. Without -\fB\-\-keep\-dirlinks\fP, the receiver deletes symlink \(dq\&foo\(dq\&, recreates it as a +.IP +This works because rsync calls \fBlstat\fP(2) on the source arg as given, and +the trailing slash makes \fBlstat\fP(2) follow the symlink, giving rise to a +directory in the file-list which overrides the symlink found during the +scan of "src/./". +.IP "\fB\-\-keep-dirlinks\fP, \fB\-K\fP" +This option causes the receiving side to treat a symlink to a directory as +though it were a real directory, but only if it matches a real directory +from the sender. Without this option, the receiver's symlink would be +deleted and replaced with a real directory. +.IP +For example, suppose you transfer a directory "foo" that contains a file +"file", but "foo" is a symlink to directory "bar" on the receiver. Without +\fB\-\-keep-dirlinks\fP, the receiver deletes symlink "foo", recreates it as a directory, and receives the file into the new directory. With -\fB\-\-keep\-dirlinks\fP, the receiver keeps the symlink and \(dq\&file\(dq\& ends up in -\(dq\&bar\(dq\&. -.IP -One note of caution: if you use \fB\-\-keep\-dirlinks\fP, you must trust all -the symlinks in the copy! If it is possible for an untrusted user to -create their own symlink to any directory, the user could then (on a -subsequent copy) replace the symlink with a real directory and affect the -content of whatever directory the symlink references. For backup copies, -you are better off using something like a bind mount instead of a symlink -to modify your receiving hierarchy. -.IP -See also \fB\-\-copy\-dirlinks\fP for an analogous option for the sending side. -.IP -.IP "\fB\-H, \-\-hard\-links\fP" -This tells rsync to look for hard\-linked files in -the source and link together the corresponding files on the destination. -Without this option, hard\-linked files in the source are treated -as though they were separate files. -.IP -This option does NOT necessarily ensure that the pattern of hard links on the -destination exactly matches that on the source. Cases in which the +\fB\-\-keep-dirlinks\fP, the receiver keeps the symlink and "file" ends up in +"bar". +.IP +One note of caution: if you use \fB\-\-keep-dirlinks\fP, you must trust all the +symlinks in the copy! If it is possible for an untrusted user to create +their own symlink to any directory, the user could then (on a subsequent +copy) replace the symlink with a real directory and affect the content of +whatever directory the symlink references. For backup copies, you are +better off using something like a bind mount instead of a symlink to modify +your receiving hierarchy. +.IP +See also \fB\-\-copy-dirlinks\fP for an analogous option for the sending side. +.IP "\fB\-\-hard-links\fP, \fB\-H\fP" +This tells rsync to look for hard-linked files in the source and link +together the corresponding files on the destination. Without this option, +hard-linked files in the source are treated as though they were separate +files. +.IP +This option does NOT necessarily ensure that the pattern of hard links on +the destination exactly matches that on the source. Cases in which the destination may end up with extra hard links include the following: -.IP -.RS -.IP o -If the destination contains extraneous hard\-links (more linking than -what is present in the source file list), the copying algorithm will not -break them explicitly. However, if one or more of the paths have content -differences, the normal file\-update process will break those extra links +.IP +.RS +.IP o +If the destination contains extraneous hard-links (more linking than what +is present in the source file list), the copying algorithm will not break +them explicitly. However, if one or more of the paths have content +differences, the normal file-update process will break those extra links (unless you are using the \fB\-\-inplace\fP option). -.IP o -If you specify a \fB\-\-link\-dest\fP directory that contains hard links, -the linking of the destination files against the \fB\-\-link\-dest\fP files can +.IP o +If you specify a \fB\-\-link-dest\fP directory that contains hard links, the +linking of the destination files against the \fB\-\-link-dest\fP files can cause some paths in the destination to become linked together due to the -\fB\-\-link\-dest\fP associations. +\fB\-\-link-dest\fP associations. .RE - -.IP +.IP Note that rsync can only detect hard links between files that are inside -the transfer set. If rsync updates a file that has extra hard\-link +the transfer set. If rsync updates a file that has extra hard-link connections to files outside the transfer, that linkage will be broken. If you are tempted to use the \fB\-\-inplace\fP option to avoid this breakage, be very careful that you know how your files are being updated so that you are certain that no unintended changes happen due to lingering hard links (and see the \fB\-\-inplace\fP option for more caveats). -.IP +.IP If incremental recursion is active (see \fB\-\-recursive\fP), rsync may transfer -a missing hard\-linked file before it finds that another link for that contents -exists elsewhere in the hierarchy. This does not affect the accuracy of -the transfer (i.e. which files are hard\-linked together), just its efficiency -(i.e. copying the data for a new, early copy of a hard\-linked file that could -have been found later in the transfer in another member of the hard\-linked -set of files). One way to avoid this inefficiency is to disable -incremental recursion using the \fB\-\-no\-inc\-recursive\fP option. -.IP -.IP "\fB\-p, \-\-perms\fP" -This option causes the receiving rsync to set the -destination permissions to be the same as the source permissions. (See -also the \fB\-\-chmod\fP option for a way to modify what rsync considers to -be the source permissions.) -.IP +a missing hard-linked file before it finds that another link for that +contents exists elsewhere in the hierarchy. This does not affect the +accuracy of the transfer (i.e. which files are hard-linked together), just +its efficiency (i.e. copying the data for a new, early copy of a +hard-linked file that could have been found later in the transfer in +another member of the hard-linked set of files). One way to avoid this +inefficiency is to disable incremental recursion using the +\fB\-\-no-inc-recursive\fP option. +.IP "\fB\-\-perms\fP, \fB\-p\fP" +This option causes the receiving rsync to set the destination permissions +to be the same as the source permissions. (See also the \fB\-\-chmod\fP option +for a way to modify what rsync considers to be the source permissions.) +.IP When this option is \fIoff\fP, permissions are set as follows: -.IP -.RS -.IP o +.IP +.RS +.IP o Existing files (including updated files) retain their existing -permissions, though the \fB\-\-executability\fP option might change just -the execute permission for the file. -.IP o -New files get their \(dq\&normal\(dq\& permission bits set to the source -file\(cq\&s permissions masked with the receiving directory\(cq\&s default -permissions (either the receiving process\(cq\&s umask, or the permissions -specified via the destination directory\(cq\&s default ACL), and -their special permission bits disabled except in the case where a new -directory inherits a setgid bit from its parent directory. +permissions, though the \fB\-\-executability\fP option might change just the +execute permission for the file. +.IP o +New files get their "normal" permission bits set to the source file's +permissions masked with the receiving directory's default permissions +(either the receiving process's umask, or the permissions specified via +the destination directory's default ACL), and their special permission +bits disabled except in the case where a new directory inherits a setgid +bit from its parent directory. .RE - -.IP -Thus, when \fB\-\-perms\fP and \fB\-\-executability\fP are both disabled, -rsync\(cq\&s behavior is the same as that of other file\-copy utilities, -such as \fBcp\fP(1) and \fBtar\fP(1). -.IP +.IP +Thus, when \fB\-\-perms\fP and \fB\-\-executability\fP are both disabled, rsync's +behavior is the same as that of other file-copy utilities, such as \fBcp\fP(1) +and \fBtar\fP(1). +.IP In summary: to give destination files (both old and new) the source -permissions, use \fB\-\-perms\fP. To give new files the destination\-default +permissions, use \fB\-\-perms\fP. To give new files the destination-default permissions (while leaving existing files unchanged), make sure that the -\fB\-\-perms\fP option is off and use \fB\-\-chmod=ugo=rwX\fP (which ensures that -all non\-masked bits get enabled). If you\(cq\&d care to make this latter -behavior easier to type, you could define a popt alias for it, such as -putting this line in the file ~/.popt (the following defines the \fB\-Z\fP option, -and includes \-\-no\-g to use the default group of the destination dir): -.IP -.RS -\f(CW rsync alias \-Z \-\-no\-p \-\-no\-g \-\-chmod=ugo=rwX\fP +\fB\-\-perms\fP option is off and use \fB\-\-chmod=ugo=rwX\fP (which ensures that all +non-masked bits get enabled). If you'd care to make this latter behavior +easier to type, you could define a popt alias for it, such as putting this +line in the file \fB~/.popt\fP (the following defines the \fB\-Z\fP option, and +includes \fB\-\-no-g\fP to use the default group of the destination dir): +.RS 4 +.IP +.nf +rsync alias -Z --no-p --no-g --chmod=ugo=rwX +.fi .RE - -.IP +.IP You could then use this new option in a command such as this one: -.IP -.RS -\f(CW rsync \-avZ src/ dest/\fP +.RS 4 +.IP +.nf +rsync -avZ src/ dest/ +.fi .RE - -.IP -(Caveat: make sure that \fB\-a\fP does not follow \fB\-Z\fP, or it will re\-enable -the two \(dq\&\-\-no\-*\(dq\& options mentioned above.) -.IP -The preservation of the destination\(cq\&s setgid bit on newly\-created +.IP +(Caveat: make sure that \fB\-a\fP does not follow \fB\-Z\fP, or it will re-enable the +two \fB\-\-no-*\fP options mentioned above.) +.IP +The preservation of the destination's setgid bit on newly-created directories when \fB\-\-perms\fP is off was added in rsync 2.6.7. Older rsync versions erroneously preserved the three special permission bits for -newly\-created files when \fB\-\-perms\fP was off, while overriding the -destination\(cq\&s setgid bit setting on a newly\-created directory. Default ACL +newly-created files when \fB\-\-perms\fP was off, while overriding the +destination's setgid bit setting on a newly-created directory. Default ACL observance was added to the ACL patch for rsync 2.6.7, so older (or -non\-ACL\-enabled) rsyncs use the umask even if default ACLs are present. +non-ACL-enabled) rsyncs use the umask even if default ACLs are present. (Keep in mind that it is the version of the receiving rsync that affects these behaviors.) -.IP -.IP "\fB\-E, \-\-executability\fP" -This option causes rsync to preserve the -executability (or non\-executability) of regular files when \fB\-\-perms\fP is -not enabled. A regular file is considered to be executable if at least one -\(cq\&x\(cq\& is turned on in its permissions. When an existing destination file\(cq\&s -executability differs from that of the corresponding source file, rsync -modifies the destination file\(cq\&s permissions as follows: -.IP -.RS -.IP o -To make a file non\-executable, rsync turns off all its \(cq\&x\(cq\& -permissions. -.IP o -To make a file executable, rsync turns on each \(cq\&x\(cq\& permission that -has a corresponding \(cq\&r\(cq\& permission enabled. +.IP "\fB\-\-executability\fP, \fB\-E\fP" +This option causes rsync to preserve the executability (or +non-executability) of regular files when \fB\-\-perms\fP is not enabled. A +regular file is considered to be executable if at least one 'x' is turned +on in its permissions. When an existing destination file's executability +differs from that of the corresponding source file, rsync modifies the +destination file's permissions as follows: +.IP +.RS +.IP o +To make a file non-executable, rsync turns off all its 'x' permissions. +.IP o +To make a file executable, rsync turns on each 'x' permission that has a +corresponding 'r' permission enabled. .RE - -.IP +.IP If \fB\-\-perms\fP is enabled, this option is ignored. -.IP -.IP "\fB\-A, \-\-acls\fP" -This option causes rsync to update the destination -ACLs to be the same as the source ACLs. -The option also implies \fB\-\-perms\fP. -.IP -The source and destination systems must have compatible ACL entries for this -option to work properly. See the \fB\-\-fake\-super\fP option for a way to backup -and restore ACLs that are not compatible. -.IP -.IP "\fB\-X, \-\-xattrs\fP" -This option causes rsync to update the destination -extended attributes to be the same as the source ones. -.IP -For systems that support extended\-attribute namespaces, a copy being done by a -super\-user copies all namespaces except system.*. A normal user only copies -the user.* namespace. To be able to backup and restore non\-user namespaces as -a normal user, see the \fB\-\-fake\-super\fP option. -.IP -The above name filtering can be overridden by using one or more filter options -with the \fBx\fP modifier. When you specify an xattr\-affecting filter rule, rsync -requires that you do your own system/user filtering, as well as any additional -filtering for what xattr names are copied and what names are allowed to be -deleted. For example, to skip the system namespace, you could specify: -.IP -.RS -\-\-filter=\(cq\&\-x system.*\(cq\& +.IP "\fB\-\-acls\fP, \fB\-A\fP" +This option causes rsync to update the destination ACLs to be the same as +the source ACLs. The option also implies \fB\-\-perms\fP. +.IP +The source and destination systems must have compatible ACL entries for +this option to work properly. See the \fB\-\-fake-super\fP option for a way to +backup and restore ACLs that are not compatible. +.IP "\fB\-\-xattrs\fP, \fB\-X\fP" +This option causes rsync to update the destination extended attributes to +be the same as the source ones. +.IP +For systems that support extended-attribute namespaces, a copy being done +by a super-user copies all namespaces except system.*. A normal user only +copies the user.* namespace. To be able to backup and restore non-user +namespaces as a normal user, see the \fB\-\-fake-super\fP option. +.IP +The above name filtering can be overridden by using one or more filter +options with the \fBx\fP modifier. When you specify an xattr-affecting +filter rule, rsync requires that you do your own system/user filtering, as +well as any additional filtering for what xattr names are copied and what +names are allowed to be deleted. For example, to skip the system +namespace, you could specify: +.RS 4 +.IP +.nf +--filter='-x system.*' +.fi .RE - -.IP +.IP To skip all namespaces except the user namespace, you could specify a -negated\-user match: -.IP -.RS -\-\-filter=\(cq\&\-x! user.*\(cq\& +negated-user match: +.RS 4 +.IP +.nf +--filter='-x! user.*' +.fi .RE - -.IP -To prevent any attributes from being deleted, you could specify a receiver\-only -rule that excludes all names: -.IP -.RS -\-\-filter=\(cq\&\-xr *\(cq\& +.IP +To prevent any attributes from being deleted, you could specify a +receiver-only rule that excludes all names: +.RS 4 +.IP +.nf +--filter='-xr *' +.fi .RE - -.IP -Note that the \fB\-X\fP option does not copy rsync\(cq\&s special xattr values (e.g. -those used by \fB\-\-fake\-super\fP) unless you repeat the option (e.g. \-XX). -This \(dq\© all xattrs\(dq\& mode cannot be used with \fB\-\-fake\-super\fP. -.IP +.IP +Note that the \fB\-X\fP option does not copy rsync's special xattr values (e.g. +those used by \fB\-\-fake-super\fP) unless you repeat the option (e.g. \fB\-XX\fP). +This "copy all xattrs" mode cannot be used with \fB\-\-fake-super\fP. .IP "\fB\-\-chmod\fP" -This option tells rsync to apply one or more -comma\-separated \(dq\&chmod\(dq\& modes to the permission of the files in the -transfer. The resulting value is treated as though it were the permissions -that the sending side supplied for the file, which means that this option -can seem to have no effect on existing files if \fB\-\-perms\fP is not enabled. -.IP +This option tells rsync to apply one or more comma-separated "chmod" modes +to the permission of the files in the transfer. The resulting value is +treated as though it were the permissions that the sending side supplied +for the file, which means that this option can seem to have no effect on +existing files if \fB\-\-perms\fP is not enabled. +.IP In addition to the normal parsing rules specified in the \fBchmod\fP(1) manpage, you can specify an item that should only apply to a directory by -prefixing it with a \(cq\&D\(cq\&, or specify an item that should only apply to a -file by prefixing it with a \(cq\&F\(cq\&. For example, the following will ensure -that all directories get marked set\-gid, that no files are other\-writable, -that both are user\-writable and group\-writable, and that both have +prefixing it with a 'D', or specify an item that should only apply to a +file by prefixing it with a 'F'. For example, the following will ensure +that all directories get marked set-gid, that no files are other-writable, +that both are user-writable and group-writable, and that both have consistent executability across all bits: -.IP -.RS -\-\-chmod=Dg+s,ug+w,Fo\-w,+X +.RS 4 +.IP +.nf +--chmod=Dg+s,ug+w,Fo-w,+X +.fi .RE - -.IP +.IP Using octal mode numbers is also allowed: -.IP -.RS -\-\-chmod=D2775,F664 +.RS 4 +.IP +.nf +--chmod=D2775,F664 +.fi .RE - -.IP -It is also legal to specify multiple \fB\-\-chmod\fP options, as each -additional option is just appended to the list of changes to make. -.IP +.IP +It is also legal to specify multiple \fB\-\-chmod\fP options, as each additional +option is just appended to the list of changes to make. +.IP See the \fB\-\-perms\fP and \fB\-\-executability\fP options for how the resulting permission value can be applied to the files in the transfer. -.IP -.IP "\fB\-o, \-\-owner\fP" -This option causes rsync to set the owner of the -destination file to be the same as the source file, but only if the -receiving rsync is being run as the super\-user (see also the \fB\-\-super\fP -and \fB\-\-fake\-super\fP options). -Without this option, the owner of new and/or transferred files are set to -the invoking user on the receiving side. -.IP +.IP "\fB\-\-owner\fP, \fB\-o\fP" +This option causes rsync to set the owner of the destination file to be the +same as the source file, but only if the receiving rsync is being run as +the super-user (see also the \fB\-\-super\fP and \fB\-\-fake-super\fP options). Without +this option, the owner of new and/or transferred files are set to the +invoking user on the receiving side. +.IP The preservation of ownership will associate matching names by default, but may fall back to using the ID number in some circumstances (see also the -\fB\-\-numeric\-ids\fP option for a full discussion). -.IP -.IP "\fB\-g, \-\-group\fP" -This option causes rsync to set the group of the -destination file to be the same as the source file. If the receiving -program is not running as the super\-user (or if \fB\-\-no\-super\fP was -specified), only groups that the invoking user on the receiving side -is a member of will be preserved. +\fB\-\-numeric-ids\fP option for a full discussion). +.IP "\fB\-\-group\fP, \fB\-g\fP" +This option causes rsync to set the group of the destination file to be the +same as the source file. If the receiving program is not running as the +super-user (or if \fB\-\-no-super\fP was specified), only groups that the +invoking user on the receiving side is a member of will be preserved. Without this option, the group is set to the default group of the invoking user on the receiving side. -.IP +.IP The preservation of group information will associate matching names by default, but may fall back to using the ID number in some circumstances -(see also the \fB\-\-numeric\-ids\fP option for a full discussion). -.IP +(see also the \fB\-\-numeric-ids\fP option for a full discussion). .IP "\fB\-\-devices\fP" -This option causes rsync to transfer character and -block device files to the remote system to recreate these devices. -This option has no effect if the receiving rsync is not run as the -super\-user (see also the \fB\-\-super\fP and \fB\-\-fake\-super\fP options). -.IP +This option causes rsync to transfer character and block device files to +the remote system to recreate these devices. This option has no effect if +the receiving rsync is not run as the super-user (see also the \fB\-\-super\fP +and \fB\-\-fake-super\fP options). .IP "\fB\-\-specials\fP" -This option causes rsync to transfer special files -such as named sockets and fifos. -.IP +This option causes rsync to transfer special files such as named sockets +and fifos. .IP "\fB\-D\fP" -The \fB\-D\fP option is equivalent to \fB\-\-devices\fP \fB\-\-specials\fP. -.IP -.IP "\fB\-t, \-\-times\fP" -This tells rsync to transfer modification times along -with the files and update them on the remote system. Note that if this -option is not used, the optimization that excludes files that have not been -modified cannot be effective; in other words, a missing \fB\-t\fP or \fB\-a\fP will -cause the next transfer to behave as if it used \fB\-I\fP, causing all files to be -updated (though rsync\(cq\&s delta\-transfer algorithm will make the update fairly efficient -if the files haven\(cq\&t actually changed, you\(cq\&re much better off using \fB\-t\fP). -.IP -.IP "\fB\-O, \-\-omit\-dir\-times\fP" -This tells rsync to omit directories when -it is preserving modification times (see \fB\-\-times\fP). If NFS is sharing -the directories on the receiving side, it is a good idea to use \fB\-O\fP. -This option is inferred if you use \fB\-\-backup\fP without \fB\-\-backup\-dir\fP. -.IP -This option also has the side\-effect of avoiding early creation of directories -in incremental recursion copies. The default \fB\-\-inc\-recursive\fP copying -normally does an early\-create pass of all the sub\-directories in a parent -directory in order for it to be able to then set the modify time of the parent -directory right away (without having to delay that until a bunch of recursive -copying has finished). This early\-create idiom is not necessary if directory -modify times are not being preserved, so it is skipped. Since early\-create -directories don\(cq\&t have accurate mode, mtime, or ownership, the use of this -option can help when someone wants to avoid these partially\-finished -directories. -.IP -.IP "\fB\-J, \-\-omit\-link\-times\fP" -This tells rsync to omit symlinks when -it is preserving modification times (see \fB\-\-times\fP). -.IP +The \fB\-D\fP option is equivalent to \fB\-\-devices\ \-\-specials\fP. +.IP "\fB\-\-write-devices\fP" +This tells rsync to treat a device on the receiving side as a regular file, +allowing the writing of file data into a device. +.IP +This option implies the \fB\-\-inplace\fP option. +.IP +Be careful using this, as you should know what devices are present on the +receiving side of the transfer, especially if running rsync as root. +.IP +This option is refused by an rsync daemon. +.IP "\fB\-\-times\fP, \fB\-t\fP" +This tells rsync to transfer modification times along with the files and +update them on the remote system. Note that if this option is not used, +the optimization that excludes files that have not been modified cannot be +effective; in other words, a missing \fB\-t\fP or \fB\-a\fP will cause the next +transfer to behave as if it used \fB\-I\fP, causing all files to be updated +(though rsync's delta-transfer algorithm will make the update fairly +efficient if the files haven't actually changed, you're much better off +using \fB\-t\fP). +.IP "\fB\-\-atimes\fP, \fB\-U\fP" +This tells rsync to set the access (use) times of the destination files to +the same value as the source files. +.IP +If repeated, it also sets the \fB\-\-open-noatime\fP option, which can help you +to make the sending and receiving systems have the same access times on the +transferred files without needing to run rsync an extra time after a file +is transferred. +.IP +Note that some older rsync versions (prior to 3.2.0) may have been built +with a pre-release \fB\-\-atimes\fP patch that does not imply \fB\-\-open-noatime\fP +when this option is repeated. +.IP "\fB\-\-open-noatime\fP" +This tells rsync to open files with the O_NOATIME flag (on systems that +support it) to avoid changing the access time of the files that are being +transferred. If your OS does not support the O_NOATIME flag then rsync +will silently ignore this option. Note also that some filesystems are +mounted to avoid updating the atime on read access even without the +O_NOATIME flag being set. +.IP "\fB\-\-omit-dir-times\fP, \fB\-O\fP" +This tells rsync to omit directories when it is preserving modification +times (see \fB\-\-times\fP). If NFS is sharing the directories on the receiving +side, it is a good idea to use \fB\-O\fP. This option is inferred if you use +\fB\-\-backup\fP without \fB\-\-backup-dir\fP. +.IP +This option also has the side-effect of avoiding early creation of +directories in incremental recursion copies. The default \fB\-\-inc-recursive\fP +copying normally does an early-create pass of all the sub-directories in a +parent directory in order for it to be able to then set the modify time of +the parent directory right away (without having to delay that until a bunch +of recursive copying has finished). This early-create idiom is not +necessary if directory modify times are not being preserved, so it is +skipped. Since early-create directories don't have accurate mode, mtime, +or ownership, the use of this option can help when someone wants to avoid +these partially-finished directories. +.IP "\fB\-\-omit-link-times\fP, \fB\-J\fP" +This tells rsync to omit symlinks when it is preserving modification times +(see \fB\-\-times\fP). .IP "\fB\-\-super\fP" -This tells the receiving side to attempt super\-user -activities even if the receiving rsync wasn\(cq\&t run by the super\-user. These -activities include: preserving users via the \fB\-\-owner\fP option, preserving -all groups (not just the current user\(cq\&s groups) via the \fB\-\-groups\fP -option, and copying devices via the \fB\-\-devices\fP option. This is useful -for systems that allow such activities without being the super\-user, and -also for ensuring that you will get errors if the receiving side isn\(cq\&t -being run as the super\-user. To turn off super\-user activities, the -super\-user can use \fB\-\-no\-super\fP. -.IP -.IP "\fB\-\-fake\-super\fP" -When this option is enabled, rsync simulates -super\-user activities by saving/restoring the privileged attributes via -special extended attributes that are attached to each file (as needed). This -includes the file\(cq\&s owner and group (if it is not the default), the file\(cq\&s -device info (device & special files are created as empty text files), and -any permission bits that we won\(cq\&t allow to be set on the real file (e.g. -the real file gets u\-s,g\-s,o\-t for safety) or that would limit the owner\(cq\&s -access (since the real super\-user can always access/change a file, the -files we create can always be accessed/changed by the creating user). -This option also handles ACLs (if \fB\-\-acls\fP was specified) and non\-user -extended attributes (if \fB\-\-xattrs\fP was specified). -.IP -This is a good way to backup data without using a super\-user, and to store +This tells the receiving side to attempt super-user activities even if the +receiving rsync wasn't run by the super-user. These activities include: +preserving users via the \fB\-\-owner\fP option, preserving all groups (not just +the current user's groups) via the \fB\-\-groups\fP option, and copying devices +via the \fB\-\-devices\fP option. This is useful for systems that allow such +activities without being the super-user, and also for ensuring that you +will get errors if the receiving side isn't being run as the super-user. +To turn off super-user activities, the super-user can use \fB\-\-no-super\fP. +.IP "\fB\-\-fake-super\fP" +When this option is enabled, rsync simulates super-user activities by +saving/restoring the privileged attributes via special extended attributes +that are attached to each file (as needed). This includes the file's owner +and group (if it is not the default), the file's device info (device & +special files are created as empty text files), and any permission bits +that we won't allow to be set on the real file (e.g. the real file gets +u-s,g-s,o-t for safety) or that would limit the owner's access (since the +real super-user can always access/change a file, the files we create can +always be accessed/changed by the creating user). This option also handles +ACLs (if \fB\-\-acls\fP was specified) and non-user extended attributes (if +\fB\-\-xattrs\fP was specified). +.IP +This is a good way to backup data without using a super-user, and to store ACLs from incompatible systems. -.IP -The \fB\-\-fake\-super\fP option only affects the side where the option is used. -To affect the remote side of a remote\-shell connection, use the -\fB\-\-remote\-option\fP (\fB\-M\fP) option: -.IP -.RS -\f(CW rsync \-av \-M\-\-fake\-super /src/ host:/dest/\fP +.IP +The \fB\-\-fake-super\fP option only affects the side where the option is used. +To affect the remote side of a remote-shell connection, use the +\fB\-\-remote-option\fP (\fB\-M\fP) option: +.RS 4 +.IP +.nf +rsync -av -M--fake-super /src/ host:/dest/ +.fi .RE - -.IP +.IP For a local copy, this option affects both the source and the destination. If you wish a local copy to enable this option just for the destination -files, specify \fB\-M\-\-fake\-super\fP. If you wish a local copy to enable -this option just for the source files, combine \fB\-\-fake\-super\fP with -\fB\-M\-\-super\fP. -.IP -This option is overridden by both \fB\-\-super\fP and \fB\-\-no\-super\fP. -.IP -See also the \(dq\&fake super\(dq\& setting in the daemon\(cq\&s rsyncd.conf file. -.IP -.IP "\fB\-S, \-\-sparse\fP" -Try to handle sparse files efficiently so they take -up less space on the destination. If combined with \fB\-\-inplace\fP the -file created might not end up with sparse blocks with some combinations -of kernel version and/or filesystem type. If \fB\-\-whole\-file\fP is in -effect (e.g. for a local copy) then it will always work because rsync -truncates the file prior to writing out the updated version. -.IP +files, specify \fB\-M\-\-fake-super\fP. If you wish a local copy to enable this +option just for the source files, combine \fB\-\-fake-super\fP with \fB\-M\-\-super\fP. +.IP +This option is overridden by both \fB\-\-super\fP and \fB\-\-no-super\fP. +.IP +See also the "\fBfake\ super\fP" setting in the daemon's rsyncd.conf file. +.IP "\fB\-\-sparse\fP, \fB\-S\fP" +Try to handle sparse files efficiently so they take up less space on the +destination. If combined with \fB\-\-inplace\fP the file created might not end +up with sparse blocks with some combinations of kernel version and/or +filesystem type. If \fB\-\-whole-file\fP is in effect (e.g. for a local copy) +then it will always work because rsync truncates the file prior to writing +out the updated version. +.IP Note that versions of rsync older than 3.1.3 will reject the combination of \fB\-\-sparse\fP and \fB\-\-inplace\fP. -.IP .IP "\fB\-\-preallocate\fP" -This tells the receiver to allocate each destination -file to its eventual size before writing data to the file. Rsync will only -use the real filesystem\-level preallocation support provided by Linux\(cq\&s -\fBfallocate\fP(2) system call or Cygwin\(cq\&s \fBposix_fallocate\fP(3), not the slow -glibc implementation that writes a null byte into each block. -.IP +This tells the receiver to allocate each destination file to its eventual +size before writing data to the file. Rsync will only use the real +filesystem-level preallocation support provided by Linux's \fBfallocate\fP(2) +system call or Cygwin's \fBposix_fallocate\fP(3), not the slow glibc +implementation that writes a null byte into each block. +.IP Without this option, larger files may not be entirely contiguous on the -filesystem, but with this option rsync will probably copy more slowly. If the -destination is not an extent\-supporting filesystem (such as ext4, xfs, NTFS, -etc.), this option may have no positive effect at all. -.IP +filesystem, but with this option rsync will probably copy more slowly. If +the destination is not an extent-supporting filesystem (such as ext4, xfs, +NTFS, etc.), this option may have no positive effect at all. +.IP If combined with \fB\-\-sparse\fP, the file will only have sparse blocks (as opposed to allocated sequences of null bytes) if the kernel version and filesystem type support creating holes in the allocated data. -.IP -.IP "\fB\-n, \-\-dry\-run\fP" -This makes rsync perform a trial run that doesn\(cq\&t -make any changes (and produces mostly the same output as a real run). It -is most commonly used in combination with the \fB\-v, \-\-verbose\fP and/or -\fB\-i, \-\-itemize\-changes\fP options to see what an rsync command is going -to do before one actually runs it. -.IP -The output of \fB\-\-itemize\-changes\fP is supposed to be exactly the same on a +.IP "\fB\-\-dry-run\fP, \fB\-n\fP" +This makes rsync perform a trial run that doesn't make any changes (and +produces mostly the same output as a real run). It is most commonly used +in combination with the \fB\-\-verbose\fP, \fB\-v\fP and/or \fB\-\-itemize-changes\fP, \fB\-i\fP +options to see what an rsync command is going to do before one actually +runs it. +.IP +The output of \fB\-\-itemize-changes\fP is supposed to be exactly the same on a dry run and a subsequent real run (barring intentional trickery and system -call failures); if it isn\(cq\&t, that\(cq\&s a bug. Other output should be mostly -unchanged, but may differ in some areas. Notably, a dry run does not -send the actual data for file transfers, so \fB\-\-progress\fP has no effect, -the \(dq\&bytes sent\(dq\&, \(dq\&bytes received\(dq\&, \(dq\&literal data\(dq\&, and \(dq\&matched data\(dq\& -statistics are too small, and the \(dq\&speedup\(dq\& value is equivalent to a run +call failures); if it isn't, that's a bug. Other output should be mostly +unchanged, but may differ in some areas. Notably, a dry run does not send +the actual data for file transfers, so \fB\-\-progress\fP has no effect, the +"bytes sent", "bytes received", "literal data", and "matched data" +statistics are too small, and the "speedup" value is equivalent to a run where no file transfers were needed. -.IP -.IP "\fB\-W, \-\-whole\-file\fP" -This option disables rsync\(cq\&s delta\-transfer algorithm, -which causes all transferred files to be sent whole. The transfer may be -faster if this option is used when the bandwidth between the source and -destination machines is higher than the bandwidth to disk (especially when the -\(dq\&disk\(dq\& is actually a networked filesystem). This is the default when both -the source and destination are specified as local paths, but only if no -batch\-writing option is in effect. -.IP -.IP "\fB\-\-checksum\-choice=STR\fP" -This option overrides the checksum algoriths. -If one algorithm name is specified, it is used for both the transfer checksums -and (assuming \fB\-\-checksum\fP is specifed) the pre\-transfer checksumming. If two -comma\-separated names are supplied, the first name affects the transfer -checksums, and the second name affects the pre\-transfer checksumming. -.IP -The algorithm choices are \(dq\&auto\(dq\&, \(dq\&md4\(dq\&, \(dq\&md5\(dq\&, and \(dq\&none\(dq\&. If \(dq\&none\(dq\& is -specified for the first name, the \fB\-\-whole\-file\fP option is forced on and no -checksum verification is performed on the transferred data. If \(dq\&none\(dq\& is -specified for the second name, the \fB\-\-checksum\fP option cannot be used. The -\(dq\&auto\(dq\& option is the default, where rsync bases its algorithm choice on the -protocol version (for backward compatibility with older rsync versions). -.IP -.IP "\fB\-x, \-\-one\-file\-system\fP" -This tells rsync to avoid crossing a -filesystem boundary when recursing. This does not limit the user\(cq\&s ability -to specify items to copy from multiple filesystems, just rsync\(cq\&s recursion -through the hierarchy of each directory that the user specified, and also -the analogous recursion on the receiving side during deletion. Also keep -in mind that rsync treats a \(dq\&bind\(dq\& mount to the same device as being on the -same filesystem. -.IP -If this option is repeated, rsync omits all mount\-point directories from -the copy. Otherwise, it includes an empty directory at each mount\-point it +.IP "\fB\-\-whole-file\fP, \fB\-W\fP" +This option disables rsync's delta-transfer algorithm, which causes all +transferred files to be sent whole. The transfer may be faster if this +option is used when the bandwidth between the source and destination +machines is higher than the bandwidth to disk (especially when the "disk" +is actually a networked filesystem). This is the default when both the +source and destination are specified as local paths, but only if no +batch-writing option is in effect. +.IP "\fB\-\-checksum-choice=STR\fP, \fB\-\-cc=STR\fP" +This option overrides the checksum algorithms. If one algorithm name is +specified, it is used for both the transfer checksums and (assuming +\fB\-\-checksum\fP is specified) the pre-transfer checksums. If two +comma-separated names are supplied, the first name affects the transfer +checksums, and the second name affects the pre-transfer checksums (\fB\-c\fP). +.IP +The checksum options that you may be able to use are: +.IP +.RS +.IP o +\fBauto\fP (the default) +.IP o +\fBxxh64\fP (aka xxhash) +.IP o +\fBmd5\fP +.IP o +\fBmd4\fP +.IP o +\fBnone\fP +.RE +.IP +Run \fBrsync\ \-V\fP to see the default checksum list compiled into your version. +.IP +If "none" is specified for the first (or only) name, the \fB\-\-whole-file\fP +option is forced on and no checksum verification is performed on the +transferred data. If "none" is specified for the second (or only) name, +the \fB\-\-checksum\fP option cannot be used. +.IP +The "auto" option is the default, where rsync bases its algorithm choice on +a negotiation between the client and the server as follows: +.IP +If both the client and the server are at least version 3.2.0, they will +exchange a list of checksum names and choose the first one in the list that +they have in common. This typically means that they will choose xxh64 if +they both support it and fall back to MD5. If one side of the transfer is +not new enough to support this checksum negotiation, then a value is chosen +based on the protocol version (which chooses between MD5 and various +flavors of MD4 based on protocol age). +.IP +You can also override the checksum using the RSYNC_CHECKSUM_LIST +environment variable by setting it to a space-separated list of checksum +names that you consider acceptable. If no common checksum is found, the +client exits with an error. This method does not allow you to specify the +transfer checksum separately from the pre-transfer checksum, and it ignores +"auto" and all unknown checksum names. If the remote rsync is not new +enough to handle a checksum negotiation list, the list is silently ignored +unless it contains the string "FAIL". +.IP +The use of the \fB\-\-checksum-choice\fP option overrides this environment list. +.IP "\fB\-\-one-file-system\fP, \fB\-x\fP" +This tells rsync to avoid crossing a filesystem boundary when recursing. +This does not limit the user's ability to specify items to copy from +multiple filesystems, just rsync's recursion through the hierarchy of each +directory that the user specified, and also the analogous recursion on the +receiving side during deletion. Also keep in mind that rsync treats a +"bind" mount to the same device as being on the same filesystem. +.IP +If this option is repeated, rsync omits all mount-point directories from +the copy. Otherwise, it includes an empty directory at each mount-point it encounters (using the attributes of the mounted directory because those of -the underlying mount\-point directory are inaccessible). -.IP -If rsync has been told to collapse symlinks (via \fB\-\-copy\-links\fP or -\fB\-\-copy\-unsafe\-links\fP), a symlink to a directory on another device is -treated like a mount\-point. Symlinks to non\-directories are unaffected -by this option. -.IP -.IP "\fB\-\-existing, \-\-ignore\-non\-existing\fP" -This tells rsync to skip -creating files (including directories) that do not exist -yet on the destination. If this option is -combined with the \fB\-\-ignore\-existing\fP option, no files will be updated -(which can be useful if all you want to do is delete extraneous files). -.IP -This option is a transfer rule, not an exclude, so it doesn\(cq\&t affect the -data that goes into the file\-lists, and thus it doesn\(cq\&t affect deletions. +the underlying mount-point directory are inaccessible). +.IP +If rsync has been told to collapse symlinks (via \fB\-\-copy-links\fP or +\fB\-\-copy-unsafe-links\fP), a symlink to a directory on another device is +treated like a mount-point. Symlinks to non-directories are unaffected by +this option. +.IP "\fB\-\-existing\fP, \fB\-\-ignore-non-existing\fP" +This tells rsync to skip creating files (including directories) that do not +exist yet on the destination. If this option is combined with the +\fB\-\-ignore-existing\fP option, no files will be updated (which can be useful +if all you want to do is delete extraneous files). +.IP +This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. -.IP -.IP "\fB\-\-ignore\-existing\fP" -This tells rsync to skip updating files that -already exist on the destination (this does \fInot\fP ignore existing -directories, or nothing would get done). See also \fB\-\-existing\fP. -.IP -This option is a transfer rule, not an exclude, so it doesn\(cq\&t affect the -data that goes into the file\-lists, and thus it doesn\(cq\&t affect deletions. +.IP "\fB\-\-ignore-existing\fP" +This tells rsync to skip updating files that already exist on the +destination (this does \fInot\fP ignore existing directories, or nothing would +get done). See also \fB\-\-existing\fP. +.IP +This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. -.IP -This option can be useful for those doing backups using the \fB\-\-link\-dest\fP +.IP +This option can be useful for those doing backups using the \fB\-\-link-dest\fP option when they need to continue a backup run that got interrupted. Since -a \fB\-\-link\-dest\fP run is copied into a new directory hierarchy (when it is -used properly), using \fB\-\-ignore existing\fP will ensure that the -already\-handled files don\(cq\&t get tweaked (which avoids a change in -permissions on the hard\-linked files). This does mean that this option -is only looking at the existing files in the destination hierarchy itself. -.IP -.IP "\fB\-\-remove\-source\-files\fP" -This tells rsync to remove from the sending -side the files (meaning non\-directories) that are a part of the transfer -and have been successfully duplicated on the receiving side. -.IP -Note that you should only use this option on source files that are quiescent. -If you are using this to move files that show up in a particular directory over -to another host, make sure that the finished files get renamed into the source -directory, not directly written into it, so that rsync can\(cq\&t possibly transfer -a file that is not yet fully written. If you can\(cq\&t first write the files into -a different directory, you should use a naming idiom that lets rsync avoid -transferring files that are not yet finished (e.g. name the file \(dq\&foo.new\(dq\& when -it is written, rename it to \(dq\&foo\(dq\& when it is done, and then use the option -\fB\-\-exclude='\&*.new'\&\fP for the rsync transfer). -.IP -Starting with 3.1.0, rsync will skip the sender\-side removal (and output an -error) if the file\(cq\&s size or modify time has not stayed unchanged. -.IP +a \fB\-\-link-dest\fP run is copied into a new directory hierarchy (when it is +used properly), using \fB\-\-ignore-existing\fP will ensure that the +already-handled files don't get tweaked (which avoids a change in +permissions on the hard-linked files). This does mean that this option is +only looking at the existing files in the destination hierarchy itself. +.IP "\fB\-\-remove-source-files\fP" +This tells rsync to remove from the sending side the files (meaning +non-directories) that are a part of the transfer and have been successfully +duplicated on the receiving side. +.IP +Note that you should only use this option on source files that are +quiescent. If you are using this to move files that show up in a +particular directory over to another host, make sure that the finished +files get renamed into the source directory, not directly written into it, +so that rsync can't possibly transfer a file that is not yet fully written. +If you can't first write the files into a different directory, you should +use a naming idiom that lets rsync avoid transferring files that are not +yet finished (e.g. name the file "foo.new" when it is written, rename it to +"foo" when it is done, and then use the option \fB\-\-exclude='*.new'\fP for the +rsync transfer). +.IP +Starting with 3.1.0, rsync will skip the sender-side removal (and output an +error) if the file's size or modify time has not stayed unchanged. .IP "\fB\-\-delete\fP" -This tells rsync to delete extraneous files from the -receiving side (ones that aren\(cq\&t on the sending side), but only for the -directories that are being synchronized. You must have asked rsync to -send the whole directory (e.g. \(dq\&dir\(dq\& or \(dq\&dir/\(dq\&) without using a wildcard -for the directory\(cq\&s contents (e.g. \(dq\&dir/*\(dq\&) since the wildcard is expanded -by the shell and rsync thus gets a request to transfer individual files, not -the files\(cq\& parent directory. Files that are excluded from the transfer are -also excluded from being deleted unless you use the \fB\-\-delete\-excluded\fP -option or mark the rules as only matching on the sending side (see the +This tells rsync to delete extraneous files from the receiving side (ones +that aren't on the sending side), but only for the directories that are +being synchronized. You must have asked rsync to send the whole directory +(e.g. "\fBdir\fP" or "\fBdir/\fP") without using a wildcard for the directory's +contents (e.g. "\fBdir/*\fP") since the wildcard is expanded by the shell and +rsync thus gets a request to transfer individual files, not the files' +parent directory. Files that are excluded from the transfer are also +excluded from being deleted unless you use the \fB\-\-delete-excluded\fP option +or mark the rules as only matching on the sending side (see the include/exclude modifiers in the FILTER RULES section). -.IP +.IP Prior to rsync 2.6.7, this option would have no effect unless \fB\-\-recursive\fP was enabled. Beginning with 2.6.7, deletions will also occur when \fB\-\-dirs\fP -(\fB\-d\fP) is enabled, but only for directories whose contents are being copied. -.IP -This option can be dangerous if used incorrectly! It is a very good idea to -first try a run using the \fB\-\-dry\-run\fP option (\fB\-n\fP) to see what files are +(\fB\-d\fP) is enabled, but only for directories whose contents are being +copied. +.IP +This option can be dangerous if used incorrectly! It is a very good idea to +first try a run using the \fB\-\-dry-run\fP option (\fB\-n\fP) to see what files are going to be deleted. -.IP -If the sending side detects any I/O errors, then the deletion of any -files at the destination will be automatically disabled. This is to -prevent temporary filesystem failures (such as NFS errors) on the -sending side from causing a massive deletion of files on the -destination. You can override this with the \fB\-\-ignore\-errors\fP option. -.IP -The \fB\-\-delete\fP option may be combined with one of the \-\-delete\-WHEN options -without conflict, as well as \fB\-\-delete\-excluded\fP. However, if none of the -\-\-delete\-WHEN options are specified, rsync will choose the -\fB\-\-delete\-during\fP algorithm when talking to rsync 3.0.0 or newer, and -the \fB\-\-delete\-before\fP algorithm when talking to an older rsync. See also -\fB\-\-delete\-delay\fP and \fB\-\-delete\-after\fP. -.IP -.IP "\fB\-\-delete\-before\fP" -Request that the file\-deletions on the receiving -side be done before the transfer starts. -See \fB\-\-delete\fP (which is implied) for more details on file\-deletion. -.IP -Deleting before the transfer is helpful if the filesystem is tight for space -and removing extraneous files would help to make the transfer possible. -However, it does introduce a delay before the start of the transfer, -and this delay might cause the transfer to timeout (if \fB\-\-timeout\fP was -specified). It also forces rsync to use the old, non\-incremental recursion -algorithm that requires rsync to scan all the files in the transfer into -memory at once (see \fB\-\-recursive\fP). -.IP -.IP "\fB\-\-delete\-during, \-\-del\fP" -Request that the file\-deletions on the -receiving side be done incrementally as the transfer happens. The -per\-directory delete scan is done right before each directory is checked -for updates, so it behaves like a more efficient \fB\-\-delete\-before\fP, -including doing the deletions prior to any per\-directory filter files -being updated. This option was first added in rsync version 2.6.4. -See \fB\-\-delete\fP (which is implied) for more details on file\-deletion. -.IP -.IP "\fB\-\-delete\-delay\fP" -Request that the file\-deletions on the receiving -side be computed during the transfer (like \fB\-\-delete\-during\fP), and then -removed after the transfer completes. This is useful when combined with -\fB\-\-delay\-updates\fP and/or \fB\-\-fuzzy\fP, and is more efficient than using -\fB\-\-delete\-after\fP (but can behave differently, since \fB\-\-delete\-after\fP -computes the deletions in a separate pass after all updates are done). -If the number of removed files overflows an internal buffer, a -temporary file will be created on the receiving side to hold the names (it -is removed while open, so you shouldn\(cq\&t see it during the transfer). If -the creation of the temporary file fails, rsync will try to fall back to -using \fB\-\-delete\-after\fP (which it cannot do if \fB\-\-recursive\fP is doing an -incremental scan). -See \fB\-\-delete\fP (which is implied) for more details on file\-deletion. -.IP -.IP "\fB\-\-delete\-after\fP" -Request that the file\-deletions on the receiving -side be done after the transfer has completed. This is useful if you -are sending new per\-directory merge files as a part of the transfer and -you want their exclusions to take effect for the delete phase of the -current transfer. It also forces rsync to use the old, non\-incremental -recursion algorithm that requires rsync to scan all the files in the -transfer into memory at once (see \fB\-\-recursive\fP). -See \fB\-\-delete\fP (which is implied) for more details on file\-deletion. -.IP -.IP "\fB\-\-delete\-excluded\fP" -In addition to deleting the files on the -receiving side that are not on the sending side, this tells rsync to also -delete any files on the receiving side that are excluded (see \fB\-\-exclude\fP). -See the FILTER RULES section for a way to make individual exclusions behave -this way on the receiver, and for a way to protect files from -\fB\-\-delete\-excluded\fP. -See \fB\-\-delete\fP (which is implied) for more details on file\-deletion. -.IP -.IP "\fB\-\-ignore\-missing\-args\fP" -When rsync is first processing the explicitly -requested source files (e.g. command\-line arguments or \fB\-\-files\-from\fP -entries), it is normally an error if the file cannot be found. This option -suppresses that error, and does not try to transfer the file. This does not -affect subsequent vanished\-file errors if a file was initially found to be -present and later is no longer there. -.IP -.IP "\fB\-\-delete\-missing\-args\fP" -This option takes the behavior of (the implied) -\fB\-\-ignore\-missing\-args\fP option a step farther: each missing arg will become -a deletion request of the corresponding destination file on the receiving side -(should it exist). If the destination file is a non\-empty directory, it will -only be successfully deleted if \-\-force or \-\-delete are in effect. Other than +.IP +If the sending side detects any I/O errors, then the deletion of any files +at the destination will be automatically disabled. This is to prevent +temporary filesystem failures (such as NFS errors) on the sending side from +causing a massive deletion of files on the destination. You can override +this with the \fB\-\-ignore-errors\fP option. +.IP +The \fB\-\-delete\fP option may be combined with one of the \-\-delete-WHEN options +without conflict, as well as \fB\-\-delete-excluded\fP. However, if none of the +\fB\-\-delete-WHEN\fP options are specified, rsync will choose the +\fB\-\-delete-during\fP algorithm when talking to rsync 3.0.0 or newer, and the +\fB\-\-delete-before\fP algorithm when talking to an older rsync. See also +\fB\-\-delete-delay\fP and \fB\-\-delete-after\fP. +.IP "\fB\-\-delete-before\fP" +Request that the file-deletions on the receiving side be done before the +transfer starts. See \fB\-\-delete\fP (which is implied) for more details on +file-deletion. +.IP +Deleting before the transfer is helpful if the filesystem is tight for +space and removing extraneous files would help to make the transfer +possible. However, it does introduce a delay before the start of the +transfer, and this delay might cause the transfer to timeout (if +\fB\-\-timeout\fP was specified). It also forces rsync to use the old, +non-incremental recursion algorithm that requires rsync to scan all the +files in the transfer into memory at once (see \fB\-\-recursive\fP). +.IP "\fB\-\-delete-during\fP, \fB\-\-del\fP" +Request that the file-deletions on the receiving side be done incrementally +as the transfer happens. The per-directory delete scan is done right +before each directory is checked for updates, so it behaves like a more +efficient \fB\-\-delete-before\fP, including doing the deletions prior to any +per-directory filter files being updated. This option was first added in +rsync version 2.6.4. See \fB\-\-delete\fP (which is implied) for more details on +file-deletion. +.IP "\fB\-\-delete-delay\fP" +Request that the file-deletions on the receiving side be computed during +the transfer (like \fB\-\-delete-during\fP), and then removed after the transfer +completes. This is useful when combined with \fB\-\-delay-updates\fP and/or +\fB\-\-fuzzy\fP, and is more efficient than using \fB\-\-delete-after\fP (but can +behave differently, since \fB\-\-delete-after\fP computes the deletions in a +separate pass after all updates are done). If the number of removed files +overflows an internal buffer, a temporary file will be created on the +receiving side to hold the names (it is removed while open, so you +shouldn't see it during the transfer). If the creation of the temporary +file fails, rsync will try to fall back to using \fB\-\-delete-after\fP (which it +cannot do if \fB\-\-recursive\fP is doing an incremental scan). See \fB\-\-delete\fP +(which is implied) for more details on file-deletion. +.IP "\fB\-\-delete-after\fP" +Request that the file-deletions on the receiving side be done after the +transfer has completed. This is useful if you are sending new +per-directory merge files as a part of the transfer and you want their +exclusions to take effect for the delete phase of the current transfer. It +also forces rsync to use the old, non-incremental recursion algorithm that +requires rsync to scan all the files in the transfer into memory at once +(see \fB\-\-recursive\fP). See \fB\-\-delete\fP (which is implied) for more details on +file-deletion. +.IP "\fB\-\-delete-excluded\fP" +In addition to deleting the files on the receiving side that are not on the +sending side, this tells rsync to also delete any files on the receiving +side that are excluded (see \fB\-\-exclude\fP). See the FILTER RULES section for +a way to make individual exclusions behave this way on the receiver, and +for a way to protect files from \fB\-\-delete-excluded\fP. See \fB\-\-delete\fP (which +is implied) for more details on file-deletion. +.IP "\fB\-\-ignore-missing-args\fP" +When rsync is first processing the explicitly requested source files (e.g. +command-line arguments or \fB\-\-files-from\fP entries), it is normally an error +if the file cannot be found. This option suppresses that error, and does +not try to transfer the file. This does not affect subsequent +vanished-file errors if a file was initially found to be present and later +is no longer there. +.IP "\fB\-\-delete-missing-args\fP" +This option takes the behavior of (the implied) \fB\-\-ignore-missing-args\fP +option a step farther: each missing arg will become a deletion request of +the corresponding destination file on the receiving side (should it exist). +If the destination file is a non-empty directory, it will only be +successfully deleted if \fB\-\-force\fP or \fB\-\-delete\fP are in effect. Other than that, this option is independent of any other type of delete processing. -.IP -The missing source files are represented by special file\-list entries which -display as a \(dq\&*missing\(dq\& entry in the \fB\-\-list\-only\fP output. -.IP -.IP "\fB\-\-ignore\-errors\fP" -Tells \fB\-\-delete\fP to go ahead and delete files -even when there are I/O errors. -.IP +.IP +The missing source files are represented by special file-list entries which +display as a "\fB*missing\fP" entry in the \fB\-\-list-only\fP output. +.IP "\fB\-\-ignore-errors\fP" +Tells \fB\-\-delete\fP to go ahead and delete files even when there are I/O +errors. .IP "\fB\-\-force\fP" -This option tells rsync to delete a non\-empty directory -when it is to be replaced by a non\-directory. This is only relevant if -deletions are not active (see \fB\-\-delete\fP for details). -.IP +This option tells rsync to delete a non-empty directory when it is to be +replaced by a non-directory. This is only relevant if deletions are not +active (see \fB\-\-delete\fP for details). +.IP Note for older rsync versions: \fB\-\-force\fP used to still be required when -using \fB\-\-delete\-after\fP, and it used to be non\-functional unless the +using \fB\-\-delete-after\fP, and it used to be non-functional unless the \fB\-\-recursive\fP option was also enabled. -.IP -.IP "\fB\-\-max\-delete=NUM\fP" -This tells rsync not to delete more than NUM -files or directories. If that limit is exceeded, all further deletions are -skipped through the end of the transfer. At the end, rsync outputs a warning -(including a count of the skipped deletions) and exits with an error code -of 25 (unless some more important error condition also occurred). -.IP -Beginning with version 3.0.0, you may specify \fB\-\-max\-delete=0\fP to be warned +.IP "\fB\-\-max-delete=NUM\fP" +This tells rsync not to delete more than NUM files or directories. If that +limit is exceeded, all further deletions are skipped through the end of the +transfer. At the end, rsync outputs a warning (including a count of the +skipped deletions) and exits with an error code of 25 (unless some more +important error condition also occurred). +.IP +Beginning with version 3.0.0, you may specify \fB\-\-max-delete=0\fP to be warned about any extraneous files in the destination without removing any of them. -Older clients interpreted this as \(dq\&unlimited\(dq\&, so if you don\(cq\&t know what -version the client is, you can use the less obvious \fB\-\-max\-delete=\-1\fP as -a backward\-compatible way to specify that no deletions be allowed (though -really old versions didn\(cq\&t warn when the limit was exceeded). -.IP -.IP "\fB\-\-max\-size=SIZE\fP" -This tells rsync to avoid transferring any -file that is larger than the specified SIZE. The SIZE value can be -suffixed with a string to indicate a size multiplier, and -may be a fractional value (e.g. \(dq\&\fB\-\-max\-size=1.5m\fP\(dq\&). -.IP -This option is a transfer rule, not an exclude, so it doesn\(cq\&t affect the -data that goes into the file\-lists, and thus it doesn\(cq\&t affect deletions. +Older clients interpreted this as "unlimited", so if you don't know what +version the client is, you can use the less obvious \fB\-\-max-delete=\-1\fP as a +backward-compatible way to specify that no deletions be allowed (though +really old versions didn't warn when the limit was exceeded). +.IP "\fB\-\-max-size=SIZE\fP" +This tells rsync to avoid transferring any file that is larger than the +specified SIZE. The SIZE value can be suffixed with a string to indicate a +size multiplier, and may be a fractional value (e.g. \fB\-\-max-size=1.5m\fP). +.IP +This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. -.IP -The suffixes are as follows: \(dq\&K\(dq\& (or \(dq\&KiB\(dq\&) is a kibibyte (1024), -\(dq\&M\(dq\& (or \(dq\&MiB\(dq\&) is a mebibyte (1024*1024), and \(dq\&G\(dq\& (or \(dq\&GiB\(dq\&) is a -gibibyte (1024*1024*1024). -If you want the multiplier to be 1000 instead of 1024, use \(dq\&KB\(dq\&, -\(dq\&MB\(dq\&, or \(dq\&GB\(dq\&. (Note: lower\-case is also accepted for all values.) -Finally, if the suffix ends in either \(dq\&+1\(dq\& or \(dq\&\-1\(dq\&, the value will +.IP +The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or +"MiB") is a mebibyte (1024*1024), and "G" (or "GiB") is a gibibyte +(1024*1024*1024). If you want the multiplier to be 1000 instead of 1024, +use "KB", "MB", or "GB". (Note: lower-case is also accepted for all +values.) Finally, if the suffix ends in either "+1" or "\-1", the value will be offset by one byte in the indicated direction. -.IP -Examples: \-\-max\-size=1.5mb\-1 is 1499999 bytes, and \-\-max\-size=2g+1 is +.IP +Examples: \fB\-\-max-size=1.5mb-1\fP is 1499999 bytes, and \fB\-\-max-size=2g+1\fP is 2147483649 bytes. -.IP -Note that rsync versions prior to 3.1.0 did not allow \fB\-\-max\-size=0\fP. -.IP -.IP "\fB\-\-min\-size=SIZE\fP" -This tells rsync to avoid transferring any -file that is smaller than the specified SIZE, which can help in not -transferring small, junk files. -See the \fB\-\-max\-size\fP option for a description of SIZE and other information. -.IP -Note that rsync versions prior to 3.1.0 did not allow \fB\-\-min\-size=0\fP. -.IP -.IP "\fB\-B, \-\-block\-size=BLOCKSIZE\fP" -This forces the block size used in -rsync\(cq\&s delta\-transfer algorithm to a fixed value. It is normally selected based on -the size of each file being updated. See the technical report for details. -.IP -.IP "\fB\-e, \-\-rsh=COMMAND\fP" -This option allows you to choose an alternative -remote shell program to use for communication between the local and -remote copies of rsync. Typically, rsync is configured to use ssh by -default, but you may prefer to use rsh on a local network. -.IP -If this option is used with \fB[user@]host::module/path\fP, then the -remote shell \fICOMMAND\fP will be used to run an rsync daemon on the -remote host, and all data will be transmitted through that remote -shell connection, rather than through a direct socket connection to a -running rsync daemon on the remote host. See the section \(dq\&USING -RSYNC\-DAEMON FEATURES VIA A REMOTE\-SHELL CONNECTION\(dq\& above. -.IP -Command\-line arguments are permitted in COMMAND provided that COMMAND is -presented to rsync as a single argument. You must use spaces (not tabs -or other whitespace) to separate the command and args from each other, -and you can use single\- and/or double\-quotes to preserve spaces in an -argument (but not backslashes). Note that doubling a single\-quote -inside a single\-quoted string gives you a single\-quote; likewise for -double\-quotes (though you need to pay attention to which quotes your -shell is parsing and which quotes rsync is parsing). Some examples: -.IP -.RS -\f(CW \-e '\&ssh \-p 2234'\&\fP -.br -\f(CW \-e '\&ssh \-o \(dq\&ProxyCommand nohup ssh firewall nc \-w1 %h %p\(dq\&'\&\fP -.br +.IP +Note that rsync versions prior to 3.1.0 did not allow \fB\-\-max-size=0\fP. +.IP "\fB\-\-min-size=SIZE\fP" +This tells rsync to avoid transferring any file that is smaller than the +specified SIZE, which can help in not transferring small, junk files. See +the \fB\-\-max-size\fP option for a description of SIZE and other information. +.IP +Note that rsync versions prior to 3.1.0 did not allow \fB\-\-min-size=0\fP. +.IP "\fB\-\-block-size=BLOCKSIZE\fP, \fB\-B\fP" +This forces the block size used in rsync's delta-transfer algorithm to a +fixed value. It is normally selected based on the size of each file being +updated. See the technical report for details. +.IP "\fB\-\-rsh=COMMAND\fP, \fB\-e\fP" +This option allows you to choose an alternative remote shell program to use +for communication between the local and remote copies of rsync. Typically, +rsync is configured to use ssh by default, but you may prefer to use rsh on +a local network. +.IP +If this option is used with \fB[user@]host::module/path\fP, then the remote +shell \fICOMMAND\fP will be used to run an rsync daemon on the remote host, and +all data will be transmitted through that remote shell connection, rather +than through a direct socket connection to a running rsync daemon on the +remote host. See the section "USING RSYNC-DAEMON FEATURES VIA A +REMOTE-SHELL CONNECTION" above. +.IP +Beginning with rsync 3.2.0, the RSYNC_PORT environment variable will be set +when a daemon connection is being made via a remote-shell connection. It +is set to 0 if the default daemon port is being assumed, or it is set to +the value of the rsync port that was specified via either the \fB\-\-port\fP +option or a non-empty port value in an rsync:// URL. This allows the +script to discern if a non-default port is being requested, allowing for +things such as an SSL or stunnel helper script to connect to a default or +alternate port. +.IP +Command-line arguments are permitted in COMMAND provided that COMMAND is +presented to rsync as a single argument. You must use spaces (not tabs or +other whitespace) to separate the command and args from each other, and you +can use single- and/or double-quotes to preserve spaces in an argument (but +not backslashes). Note that doubling a single-quote inside a single-quoted +string gives you a single-quote; likewise for double-quotes (though you +need to pay attention to which quotes your shell is parsing and which +quotes rsync is parsing). Some examples: +.RS 4 +.IP +.nf +-e 'ssh -p 2234' +-e 'ssh -o "ProxyCommand nohup ssh firewall nc -w1 %h %p"' +.fi .RE - -.IP -(Note that ssh users can alternately customize site\-specific connect +.IP +(Note that ssh users can alternately customize site-specific connect options in their .ssh/config file.) -.IP +.IP You can also choose the remote shell program using the RSYNC_RSH environment variable, which accepts the same range of values as \fB\-e\fP. -.IP -See also the \fB\-\-blocking\-io\fP option which is affected by this option. -.IP -.IP "\fB\-\-rsync\-path=PROGRAM\fP" -Use this to specify what program is to be run -on the remote machine to start\-up rsync. Often used when rsync is not in -the default remote\-shell\(cq\&s path (e.g. \-\-rsync\-path=/usr/local/bin/rsync). -Note that PROGRAM is run with the help of a shell, so it can be any -program, script, or command sequence you\(cq\&d care to run, so long as it does -not corrupt the standard\-in & standard\-out that rsync is using to -communicate. -.IP +.IP +See also the \fB\-\-blocking-io\fP option which is affected by this option. +.IP "\fB\-\-rsync-path=PROGRAM\fP" +Use this to specify what program is to be run on the remote machine to +start-up rsync. Often used when rsync is not in the default remote-shell's +path (e.g. \fB\-\-rsync-path=/usr/local/bin/rsync\fP). Note that PROGRAM is run +with the help of a shell, so it can be any program, script, or command +sequence you'd care to run, so long as it does not corrupt the standard-in +& standard-out that rsync is using to communicate. +.IP One tricky example is to set a different default directory on the remote machine for use with the \fB\-\-relative\fP option. For instance: -.IP -.RS -\f(CW rsync \-avR \-\-rsync\-path=\(dq\&cd /a/b && rsync\(dq\& host:c/d /e/\fP +.RS 4 +.IP +.nf +rsync -avR --rsync-path="cd /a/b && rsync" host:c/d /e/ +.fi .RE - -.IP -.IP "\fB\-M, \-\-remote\-option=OPTION\fP" -This option is used for more advanced -situations where you want certain effects to be limited to one side of the -transfer only. For instance, if you want to pass \fB\-\-log\-file=FILE\fP and -\fB\-\-fake\-super\fP to the remote system, specify it like this: -.IP -.RS -\f(CW rsync \-av \-M \-\-log\-file=foo \-M\-\-fake\-super src/ dest/\fP +.IP "\fB\-\-remote-option=OPTION\fP, \fB\-M\fP" +This option is used for more advanced situations where you want certain +effects to be limited to one side of the transfer only. For instance, if +you want to pass \fB\-\-log-file=FILE\fP and \fB\-\-fake-super\fP to the remote system, +specify it like this: +.RS 4 +.IP +.nf +rsync -av -M --log-file=foo -M--fake-super src/ dest/ +.fi .RE - -.IP +.IP If you want to have an option affect only the local side of a transfer when it normally affects both sides, send its negation to the remote side. Like this: -.IP -.RS -\f(CW rsync \-av \-x \-M\-\-no\-x src/ dest/\fP +.RS 4 +.IP +.nf +rsync -av -x -M--no-x src/ dest/ +.fi .RE - -.IP -Be cautious using this, as it is possible to toggle an option that will cause -rsync to have a different idea about what data to expect next over the socket, -and that will make it fail in a cryptic fashion. -.IP -Note that it is best to use a separate \fB\-\-remote\-option\fP for each option you -want to pass. This makes your useage compatible with the \fB\-\-protect\-args\fP -option. If that option is off, any spaces in your remote options will be split -by the remote shell unless you take steps to protect them. -.IP -When performing a local transfer, the \(dq\&local\(dq\& side is the sender and the -\(dq\&remote\(dq\& side is the receiver. -.IP -Note some versions of the popt option\-parsing library have a bug in them that -prevents you from using an adjacent arg with an equal in it next to a short -option letter (e.g. \f(CW\-M\-\-log\-file=/tmp/foo\fP). If this bug affects your -version of popt, you can use the version of popt that is included with rsync. -.IP -.IP "\fB\-C, \-\-cvs\-exclude\fP" -This is a useful shorthand for excluding a -broad range of files that you often don\(cq\&t want to transfer between -systems. It uses a similar algorithm to CVS to determine if -a file should be ignored. -.IP +.IP +Be cautious using this, as it is possible to toggle an option that will +cause rsync to have a different idea about what data to expect next over +the socket, and that will make it fail in a cryptic fashion. +.IP +Note that it is best to use a separate \fB\-\-remote-option\fP for each option +you want to pass. This makes your usage compatible with the +\fB\-\-protect-args\fP option. If that option is off, any spaces in your remote +options will be split by the remote shell unless you take steps to protect +them. +.IP +When performing a local transfer, the "local" side is the sender and the +"remote" side is the receiver. +.IP +Note some versions of the popt option-parsing library have a bug in them +that prevents you from using an adjacent arg with an equal in it next to a +short option letter (e.g. \fB\-M\-\-log-file=/tmp/foo\fP). If this bug affects +your version of popt, you can use the version of popt that is included with +rsync. +.IP "\fB\-\-cvs-exclude\fP, \fB\-C\fP" +This is a useful shorthand for excluding a broad range of files that you +often don't want to transfer between systems. It uses a similar algorithm +to CVS to determine if a file should be ignored. +.IP The exclude list is initialized to exclude the following items (these -initial items are marked as perishable \-\- see the FILTER RULES section): -.IP -.RS -.RS -\f(CWRCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS .make.state -\&.nse_depinfo *~ #* .#* ,* _$* *$ *.old *.bak *.BAK *.orig *.rej .del\-* -*.a *.olb *.o *.obj *.so *.exe *.Z *.elc *.ln core .svn/ .git/ .hg/ .bzr/\fP -.RE +initial items are marked as perishable\ \-\- see the FILTER RULES section): +.RS 4 +.IP +\fBRCS\fP +\fBSCCS\fP +\fBCVS\fP +\fBCVS.adm\fP +\fBRCSLOG\fP +\fBcvslog.*\fP +\fBtags\fP +\fBTAGS\fP +\fB.make.state\fP +\fB.nse_depinfo\fP +\fB*~\fP +\fB#*\fP +\fB.#*\fP +\fB,*\fP +\fB_$*\fP +\fB*$\fP +\fB*.old\fP +\fB*.bak\fP +\fB*.BAK\fP +\fB*.orig\fP +\fB*.rej\fP +\fB.del-*\fP +\fB*.a\fP +\fB*.olb\fP +\fB*.o\fP +\fB*.obj\fP +\fB*.so\fP +\fB*.exe\fP +\fB*.Z\fP +\fB*.elc\fP +\fB*.ln\fP +\fBcore\fP +\fB.svn/\fP +\fB.git/\fP +\fB.hg/\fP +\fB.bzr/\fP .RE - -.IP +.IP then, files listed in a $HOME/.cvsignore are added to the list and any -files listed in the CVSIGNORE environment variable (all cvsignore names -are delimited by whitespace). -.IP -Finally, any file is ignored if it is in the same directory as a -\&.cvsignore file and matches one of the patterns listed therein. Unlike -rsync\(cq\&s filter/exclude files, these patterns are split on whitespace. -See the \fBcvs\fP(1) manual for more information. -.IP -If you\(cq\&re combining \fB\-C\fP with your own \fB\-\-filter\fP rules, you should -note that these CVS excludes are appended at the end of your own rules, -regardless of where the \fB\-C\fP was placed on the command\-line. This makes them -a lower priority than any rules you specified explicitly. If you want to -control where these CVS excludes get inserted into your filter rules, you -should omit the \fB\-C\fP as a command\-line option and use a combination of -\fB\-\-filter=:C\fP and \fB\-\-filter=\-C\fP (either on your command\-line or by -putting the \(dq\&:C\(dq\& and \(dq\&\-C\(dq\& rules into a filter file with your other rules). -The first option turns on the per\-directory scanning for the .cvsignore -file. The second option does a one\-time import of the CVS excludes -mentioned above. -.IP -.IP "\fB\-f, \-\-filter=RULE\fP" -This option allows you to add rules to selectively -exclude certain files from the list of files to be transferred. This is -most useful in combination with a recursive transfer. -.IP -You may use as many \fB\-\-filter\fP options on the command line as you like -to build up the list of files to exclude. If the filter contains whitespace, +files listed in the CVSIGNORE environment variable (all cvsignore names are +delimited by whitespace). +.IP +Finally, any file is ignored if it is in the same directory as a .cvsignore +file and matches one of the patterns listed therein. Unlike rsync's +filter/exclude files, these patterns are split on whitespace. See the +\fBcvs\fP(1) manual for more information. +.IP +If you're combining \fB\-C\fP with your own \fB\-\-filter\fP rules, you should note +that these CVS excludes are appended at the end of your own rules, +regardless of where the \fB\-C\fP was placed on the command-line. This makes +them a lower priority than any rules you specified explicitly. If you want +to control where these CVS excludes get inserted into your filter rules, +you should omit the \fB\-C\fP as a command-line option and use a combination of +\fB\-\-filter=:C\fP and \fB\-\-filter=\-C\fP (either on your command-line or by putting +the ":C" and "\-C" rules into a filter file with your other rules). The +first option turns on the per-directory scanning for the .cvsignore file. +The second option does a one-time import of the CVS excludes mentioned +above. +.IP "\fB\-\-filter=RULE\fP, \fB\-f\fP" +This option allows you to add rules to selectively exclude certain files +from the list of files to be transferred. This is most useful in +combination with a recursive transfer. +.IP +You may use as many \fB\-\-filter\fP options on the command line as you like to +build up the list of files to exclude. If the filter contains whitespace, be sure to quote it so that the shell gives the rule to rsync as a single argument. The text below also mentions that you can use an underscore to replace the space that separates a rule from its arg. -.IP +.IP See the FILTER RULES section for detailed information on this option. -.IP .IP "\fB\-F\fP" -The \fB\-F\fP option is a shorthand for adding two \fB\-\-filter\fP rules to -your command. The first time it is used is a shorthand for this rule: -.IP -.RS -\f(CW \-\-filter='\&dir\-merge /.rsync\-filter'\&\fP +The \fB\-F\fP option is a shorthand for adding two \fB\-\-filter\fP rules to your +command. The first time it is used is a shorthand for this rule: +.RS 4 +.IP +.nf +--filter='dir-merge /.rsync-filter' +.fi .RE - -.IP -This tells rsync to look for per\-directory .rsync\-filter files that have +.IP +This tells rsync to look for per-directory .rsync-filter files that have been sprinkled through the hierarchy and use their rules to filter the files in the transfer. If \fB\-F\fP is repeated, it is a shorthand for this rule: -.IP -.RS -\f(CW \-\-filter='\&exclude .rsync\-filter'\&\fP +.RS 4 +.IP +.nf +--filter='exclude .rsync-filter' +.fi .RE - -.IP -This filters out the .rsync\-filter files themselves from the transfer. -.IP +.IP +This filters out the .rsync-filter files themselves from the transfer. +.IP See the FILTER RULES section for detailed information on how these options work. -.IP .IP "\fB\-\-exclude=PATTERN\fP" -This option is a simplified form of the -\fB\-\-filter\fP option that defaults to an exclude rule and does not allow -the full rule\-parsing syntax of normal filter rules. -.IP +This option is a simplified form of the \fB\-\-filter\fP option that defaults to +an exclude rule and does not allow the full rule-parsing syntax of normal +filter rules. +.IP See the FILTER RULES section for detailed information on this option. -.IP -.IP "\fB\-\-exclude\-from=FILE\fP" -This option is related to the \fB\-\-exclude\fP -option, but it specifies a FILE that contains exclude patterns (one per line). -Blank lines in the file and lines starting with \(cq\&;\(cq\& or \(cq\&#\(cq\& are ignored. -If \fIFILE\fP is \fB\-\fP, the list will be read from standard input. -.IP +.IP "\fB\-\-exclude-from=FILE\fP" +This option is related to the \fB\-\-exclude\fP option, but it specifies a FILE +that contains exclude patterns (one per line). Blank lines in the file and +lines starting with '\fB;\fP' or '\fB#\fP' are ignored. If \fIFILE\fP is '\fB\-\fP', the +list will be read from standard input. .IP "\fB\-\-include=PATTERN\fP" -This option is a simplified form of the -\fB\-\-filter\fP option that defaults to an include rule and does not allow -the full rule\-parsing syntax of normal filter rules. -.IP +This option is a simplified form of the \fB\-\-filter\fP option that defaults to +an include rule and does not allow the full rule-parsing syntax of normal +filter rules. +.IP See the FILTER RULES section for detailed information on this option. -.IP -.IP "\fB\-\-include\-from=FILE\fP" -This option is related to the \fB\-\-include\fP -option, but it specifies a FILE that contains include patterns (one per line). -Blank lines in the file and lines starting with \(cq\&;\(cq\& or \(cq\&#\(cq\& are ignored. -If \fIFILE\fP is \fB\-\fP, the list will be read from standard input. -.IP -.IP "\fB\-\-files\-from=FILE\fP" -Using this option allows you to specify the -exact list of files to transfer (as read from the specified FILE or \fB\-\fP -for standard input). It also tweaks the default behavior of rsync to make -transferring just the specified files and directories easier: -.IP -.RS -.IP o +.IP "\fB\-\-include-from=FILE\fP" +This option is related to the \fB\-\-include\fP option, but it specifies a FILE +that contains include patterns (one per line). Blank lines in the file and +lines starting with '\fB;\fP' or '\fB#\fP' are ignored. If \fIFILE\fP is '\fB\-\fP', the +list will be read from standard input. +.IP "\fB\-\-files-from=FILE\fP" +Using this option allows you to specify the exact list of files to transfer +(as read from the specified FILE or '\fB\-\fP' for standard input). It also +tweaks the default behavior of rsync to make transferring just the +specified files and directories easier: +.IP +.RS +.IP o The \fB\-\-relative\fP (\fB\-R\fP) option is implied, which preserves the path information that is specified for each item in the file (use -\fB\-\-no\-relative\fP or \fB\-\-no\-R\fP if you want to turn that off). -.IP o +\fB\-\-no-relative\fP or \fB\-\-no-R\fP if you want to turn that off). +.IP o The \fB\-\-dirs\fP (\fB\-d\fP) option is implied, which will create directories specified in the list on the destination rather than noisily skipping -them (use \fB\-\-no\-dirs\fP or \fB\-\-no\-d\fP if you want to turn that off). -.IP o -The \fB\-\-archive\fP (\fB\-a\fP) option\(cq\&s behavior does not imply \fB\-\-recursive\fP +them (use \fB\-\-no-dirs\fP or \fB\-\-no-d\fP if you want to turn that off). +.IP o +The \fB\-\-archive\fP (\fB\-a\fP) option's behavior does not imply \fB\-\-recursive\fP (\fB\-r\fP), so specify it explicitly, if you want it. -.IP o -These side\-effects change the default state of rsync, so the position -of the \fB\-\-files\-from\fP option on the command\-line has no bearing on how -other options are parsed (e.g. \fB\-a\fP works the same before or after -\fB\-\-files\-from\fP, as does \fB\-\-no\-R\fP and all other options). +.IP o +These side-effects change the default state of rsync, so the position of +the \fB\-\-files-from\fP option on the command-line has no bearing on how other +options are parsed (e.g. \fB\-a\fP works the same before or after +\fB\-\-files-from\fP, as does \fB\-\-no-R\fP and all other options). .RE - -.IP -The filenames that are read from the FILE are all relative to the -source dir \-\- any leading slashes are removed and no \(dq\&..\(dq\& references are -allowed to go higher than the source dir. For example, take this -command: -.IP -.RS -\f(CW rsync \-a \-\-files\-from=/tmp/foo /usr remote:/backup\fP +.IP +The filenames that are read from the FILE are all relative to the source +dir\ \-\- any leading slashes are removed and no ".." references are allowed +to go higher than the source dir. For example, take this command: +.RS 4 +.IP +.nf +rsync -a --files-from=/tmp/foo /usr remote:/backup +.fi .RE - -.IP -If /tmp/foo contains the string \(dq\&bin\(dq\& (or even \(dq\&/bin\(dq\&), the /usr/bin +.IP +If /tmp/foo contains the string "bin" (or even "/bin"), the /usr/bin directory will be created as /backup/bin on the remote host. If it -contains \(dq\&bin/\(dq\& (note the trailing slash), the immediate contents of -the directory would also be sent (without needing to be explicitly -mentioned in the file \-\- this began in version 2.6.4). In both cases, -if the \fB\-r\fP option was enabled, that dir\(cq\&s entire hierarchy would -also be transferred (keep in mind that \fB\-r\fP needs to be specified -explicitly with \fB\-\-files\-from\fP, since it is not implied by \fB\-a\fP). -Also note -that the effect of the (enabled by default) \fB\-\-relative\fP option is to -duplicate only the path info that is read from the file \-\- it does not -force the duplication of the source\-spec path (/usr in this case). -.IP -In addition, the \fB\-\-files\-from\fP file can be read from the remote host -instead of the local host if you specify a \(dq\&host:\(dq\& in front of the file -(the host must match one end of the transfer). As a short\-cut, you can -specify just a prefix of \(dq\&:\(dq\& to mean \(dq\&use the remote end of the -transfer\(dq\&. For example: -.IP -.RS -\f(CW rsync \-a \-\-files\-from=:/path/file\-list src:/ /tmp/copy\fP +contains "bin/" (note the trailing slash), the immediate contents of the +directory would also be sent (without needing to be explicitly mentioned in +the file\ \-\- this began in version 2.6.4). In both cases, if the \fB\-r\fP +option was enabled, that dir's entire hierarchy would also be transferred +(keep in mind that \fB\-r\fP needs to be specified explicitly with +\fB\-\-files-from\fP, since it is not implied by \fB\-a\fP). Also note that the +effect of the (enabled by default) \fB\-\-relative\fP option is to duplicate only +the path info that is read from the file\ \-\- it does not force the +duplication of the source-spec path (/usr in this case). +.IP +In addition, the \fB\-\-files-from\fP file can be read from the remote host +instead of the local host if you specify a "host:" in front of the file +(the host must match one end of the transfer). As a short-cut, you can +specify just a prefix of ":" to mean "use the remote end of the transfer". +For example: +.RS 4 +.IP +.nf +rsync -a --files-from=:/path/file-list src:/ /tmp/copy +.fi .RE - -.IP -This would copy all the files specified in the /path/file\-list file that -was located on the remote \(dq\&src\(dq\& host. -.IP -If the \fB\-\-iconv\fP and \fB\-\-protect\-args\fP options are specified and the -\fB\-\-files\-from\fP filenames are being sent from one host to another, the -filenames will be translated from the sending host\(cq\&s charset to the -receiving host\(cq\&s charset. -.IP -NOTE: sorting the list of files in the \-\-files\-from input helps rsync to be -more efficient, as it will avoid re\-visiting the path elements that are shared -between adjacent entries. If the input is not sorted, some path elements -(implied directories) may end up being scanned multiple times, and rsync will -eventually unduplicate them after they get turned into file\-list elements. -.IP -.IP "\fB\-0, \-\-from0\fP" -This tells rsync that the rules/filenames it reads from a -file are terminated by a null (\(cq\&\e0\(cq\&) character, not a NL, CR, or CR+LF. -This affects \fB\-\-exclude\-from\fP, \fB\-\-include\-from\fP, \fB\-\-files\-from\fP, and any -merged files specified in a \fB\-\-filter\fP rule. -It does not affect \fB\-\-cvs\-exclude\fP (since all names read from a .cvsignore -file are split on whitespace). -.IP -.IP "\fB\-s, \-\-protect\-args\fP" -This option sends all filenames and most options to -the remote rsync without allowing the remote shell to interpret them. This -means that spaces are not split in names, and any non\-wildcard special -characters are not translated (such as ~, $, ;, &, etc.). Wildcards are -expanded on the remote host by rsync (instead of the shell doing it). -.IP -If you use this option with \fB\-\-iconv\fP, the args related to the remote -side will also be translated -from the local to the remote character\-set. The translation happens before -wild\-cards are expanded. See also the \fB\-\-files\-from\fP option. -.IP +.IP +This would copy all the files specified in the /path/file-list file that +was located on the remote "src" host. +.IP +If the \fB\-\-iconv\fP and \fB\-\-protect-args\fP options are specified and the +\fB\-\-files-from\fP filenames are being sent from one host to another, the +filenames will be translated from the sending host's charset to the +receiving host's charset. +.IP +NOTE: sorting the list of files in the \fB\-\-files-from\fP input helps rsync to +be more efficient, as it will avoid re-visiting the path elements that are +shared between adjacent entries. If the input is not sorted, some path +elements (implied directories) may end up being scanned multiple times, and +rsync will eventually unduplicate them after they get turned into file-list +elements. +.IP "\fB\-\-from0\fP, \fB\-0\fP" +This tells rsync that the rules/filenames it reads from a file are +terminated by a null ('\\0') character, not a NL, CR, or CR+LF. This +affects \fB\-\-exclude-from\fP, \fB\-\-include-from\fP, \fB\-\-files-from\fP, and any merged +files specified in a \fB\-\-filter\fP rule. It does not affect \fB\-\-cvs-exclude\fP +(since all names read from a .cvsignore file are split on whitespace). +.IP "\fB\-\-protect-args\fP, \fB\-s\fP" +This option sends all filenames and most options to the remote rsync +without allowing the remote shell to interpret them. This means that +spaces are not split in names, and any non-wildcard special characters are +not translated (such as \fB~\fP, \fB$\fP, \fB;\fP, \fB&\fP, etc.). Wildcards are expanded +on the remote host by rsync (instead of the shell doing it). +.IP +If you use this option with \fB\-\-iconv\fP, the args related to the remote side +will also be translated from the local to the remote character-set. The +translation happens before wild-cards are expanded. See also the +\fB\-\-files-from\fP option. +.IP You may also control this option via the RSYNC_PROTECT_ARGS environment -variable. If this variable has a non\-zero value, this option will be enabled -by default, otherwise it will be disabled by default. Either state is -overridden by a manually specified positive or negative version of this option -(note that \fB\-\-no\-s\fP and \fB\-\-no\-protect\-args\fP are the negative versions). -Since this option was first introduced in 3.0.0, you\(cq\&ll need to make sure it\(cq\&s -disabled if you ever need to interact with a remote rsync that is older than -that. -.IP +variable. If this variable has a non-zero value, this option will be +enabled by default, otherwise it will be disabled by default. Either state +is overridden by a manually specified positive or negative version of this +option (note that \fB\-\-no-s\fP and \fB\-\-no-protect-args\fP are the negative +versions). Since this option was first introduced in 3.0.0, you'll need to +make sure it's disabled if you ever need to interact with a remote rsync +that is older than that. +.IP Rsync can also be configured (at build time) to have this option enabled by -default (with is overridden by both the environment and the command\-line). +default (with is overridden by both the environment and the command-line). This option will eventually become a new default setting at some -as\-yet\-undetermined point in the future. -.IP -.IP "\fB\-T, \-\-temp\-dir=DIR\fP" -This option instructs rsync to use DIR as a -scratch directory when creating temporary copies of the files transferred -on the receiving side. The default behavior is to create each temporary -file in the same directory as the associated destination file. -Beginning with rsync 3.1.1, the temp\-file names inside the specified DIR will -not be prefixed with an extra dot (though they will still have a random suffix -added). -.IP +as-yet-undetermined point in the future. +.IP "\fB\-\-copy-as=USER[:GROUP]\fP" +This option instructs rsync to use the USER and (if specified after a +colon) the GROUP for the copy operations. This only works if the user that +is running rsync has the ability to change users. If the group is not +specified then the user's default groups are used. +.IP +This option can help to reduce the risk of an rsync being run as root into +or out of a directory that might have live changes happening to it and you +want to make sure that root-level read or write actions of system files are +not possible. While you could alternatively run all of rsync as the +specified user, sometimes you need the root-level host-access credentials +to be used, so this allows rsync to drop root for the copying part of the +operation after the remote-shell or daemon connection is established. +.IP +The option only affects one side of the transfer unless the transfer is +local, in which case it affects both sides. Use the \fB\-\-remote-option\fP to +affect the remote side, such as \fB\-M\-\-copy-as=joe\fP. For a local transfer, +the lsh (or lsh.sh) support file provides a local-shell helper script that +can be used to allow a "localhost:" or "lh:" host-spec to be specified +without needing to setup any remote shells, allowing you to specify remote +options that affect the side of the transfer that is using the host-spec +(and using hostname "lh" avoids the overriding of the remote directory to +the user's home dir). +.IP +For example, the following rsync writes the local files as user "joe": +.RS 4 +.IP +.nf +sudo rsync -aiv --copy-as=joe host1:backups/joe/ /home/joe/ +.fi +.RE +.IP +This makes all files owned by user "joe", limits the groups to those that +are available to that user, and makes it impossible for the joe user to do +a timed exploit of the path to induce a change to a file that the joe user +has no permissions to change. +.IP +The following command does a local copy into the "dest/" dir as user "joe" +(assumimg you've installed support/lsh into a dir on your $PATH): +.RS 4 +.IP +.nf +sudo rsync -aive lsh -M--copy-as=joe src/ lh:dest/ +.fi +.RE +.IP "\fB\-\-temp-dir=DIR\fP, \fB\-T\fP" +This option instructs rsync to use DIR as a scratch directory when creating +temporary copies of the files transferred on the receiving side. The +default behavior is to create each temporary file in the same directory as +the associated destination file. Beginning with rsync 3.1.1, the temp-file +names inside the specified DIR will not be prefixed with an extra dot +(though they will still have a random suffix added). +.IP This option is most often used when the receiving disk partition does not have enough free space to hold a copy of the largest file in the transfer. In this case (i.e. when the scratch directory is on a different disk @@ -2088,1831 +2171,1784 @@ temporary file in the destination directory, and then renamed into place) it would be possible for the old file to continue taking up disk space (if someone had it open), and thus there might not be enough room to fit the new version on the disk at the same time. -.IP +.IP If you are using this option for reasons other than a shortage of disk -space, you may wish to combine it with the \fB\-\-delay\-updates\fP option, -which will ensure that all copied files get put into subdirectories in the -destination hierarchy, awaiting the end of the transfer. If you don\(cq\&t -have enough room to duplicate all the arriving files on the destination -partition, another way to tell rsync that you aren\(cq\&t overly concerned -about disk space is to use the \fB\-\-partial\-dir\fP option with a relative -path; because this tells rsync that it is OK to stash off a copy of a -single file in a subdir in the destination hierarchy, rsync will use the -partial\-dir as a staging area to bring over the copied file, and then -rename it into place from there. (Specifying a \fB\-\-partial\-dir\fP with -an absolute path does not have this side\-effect.) -.IP -.IP "\fB\-y, \-\-fuzzy\fP" -This option tells rsync that it should look for a -basis file for any destination file that is missing. The current algorithm -looks in the same directory as the destination file for either a file that -has an identical size and modified\-time, or a similarly\-named file. If -found, rsync uses the fuzzy basis file to try to speed up the transfer. -.IP +space, you may wish to combine it with the \fB\-\-delay-updates\fP option, which +will ensure that all copied files get put into subdirectories in the +destination hierarchy, awaiting the end of the transfer. If you don't have +enough room to duplicate all the arriving files on the destination +partition, another way to tell rsync that you aren't overly concerned about +disk space is to use the \fB\-\-partial-dir\fP option with a relative path; +because this tells rsync that it is OK to stash off a copy of a single file +in a subdir in the destination hierarchy, rsync will use the partial-dir as +a staging area to bring over the copied file, and then rename it into place +from there. (Specifying a \fB\-\-partial-dir\fP with an absolute path does not +have this side-effect.) +.IP "\fB\-\-fuzzy\fP, \fB\-y\fP" +This option tells rsync that it should look for a basis file for any +destination file that is missing. The current algorithm looks in the same +directory as the destination file for either a file that has an identical +size and modified-time, or a similarly-named file. If found, rsync uses +the fuzzy basis file to try to speed up the transfer. +.IP If the option is repeated, the fuzzy scan will also be done in any matching -alternate destination directories that are specified via \fB\-\-compare\-dest\fP, -\fB\-\-copy\-dest\fP, or \fB\-\-link\-dest\fP. -.IP +alternate destination directories that are specified via \fB\-\-compare-dest\fP, +\fB\-\-copy-dest\fP, or \fB\-\-link-dest\fP. +.IP Note that the use of the \fB\-\-delete\fP option might get rid of any potential -fuzzy\-match files, so either use \fB\-\-delete\-after\fP or specify some -filename exclusions if you need to prevent this. -.IP -.IP "\fB\-\-compare\-dest=DIR\fP" -This option instructs rsync to use \fIDIR\fP on -the destination machine as an additional hierarchy to compare destination -files against doing transfers (if the files are missing in the destination -directory). If a file is found in \fIDIR\fP that is identical to the -sender\(cq\&s file, the file will NOT be transferred to the destination -directory. This is useful for creating a sparse backup of just files that -have changed from an earlier backup. -This option is typically used to copy into an empty (or newly created) +fuzzy-match files, so either use \fB\-\-delete-after\fP or specify some filename +exclusions if you need to prevent this. +.IP "\fB\-\-compare-dest=DIR\fP" +This option instructs rsync to use \fIDIR\fP on the destination machine as an +additional hierarchy to compare destination files against doing transfers +(if the files are missing in the destination directory). If a file is +found in \fIDIR\fP that is identical to the sender's file, the file will NOT be +transferred to the destination directory. This is useful for creating a +sparse backup of just files that have changed from an earlier backup. This +option is typically used to copy into an empty (or newly created) directory. -.IP -Beginning in version 2.6.4, multiple \fB\-\-compare\-dest\fP directories may be +.IP +Beginning in version 2.6.4, multiple \fB\-\-compare-dest\fP directories may be provided, which will cause rsync to search the list in the order specified -for an exact match. -If a match is found that differs only in attributes, a local copy is made -and the attributes updated. -If a match is not found, a basis file from one of the \fIDIR\fPs will be -selected to try to speed up the transfer. -.IP +for an exact match. If a match is found that differs only in attributes, a +local copy is made and the attributes updated. If a match is not found, a +basis file from one of the \fIDIRs\fP will be selected to try to speed up the +transfer. +.IP If \fIDIR\fP is a relative path, it is relative to the destination directory. -See also \fB\-\-copy\-dest\fP and \fB\-\-link\-dest\fP. -.IP -NOTE: beginning with version 3.1.0, rsync will remove a file from a non\-empty -destination hierarchy if an exact match is found in one of the compare\-dest -hierarchies (making the end result more closely match a fresh copy). -.IP -.IP "\fB\-\-copy\-dest=DIR\fP" -This option behaves like \fB\-\-compare\-dest\fP, but -rsync will also copy unchanged files found in \fIDIR\fP to the destination -directory using a local copy. -This is useful for doing transfers to a new destination while leaving -existing files intact, and then doing a flash\-cutover when all files have -been successfully transferred. -.IP -Multiple \fB\-\-copy\-dest\fP directories may be provided, which will cause -rsync to search the list in the order specified for an unchanged file. -If a match is not found, a basis file from one of the \fIDIR\fPs will be -selected to try to speed up the transfer. -.IP +See also \fB\-\-copy-dest\fP and \fB\-\-link-dest\fP. +.IP +NOTE: beginning with version 3.1.0, rsync will remove a file from a +non-empty destination hierarchy if an exact match is found in one of the +compare-dest hierarchies (making the end result more closely match a fresh +copy). +.IP "\fB\-\-copy-dest=DIR\fP" +This option behaves like \fB\-\-compare-dest\fP, but rsync will also copy +unchanged files found in \fIDIR\fP to the destination directory using a local +copy. This is useful for doing transfers to a new destination while +leaving existing files intact, and then doing a flash-cutover when all +files have been successfully transferred. +.IP +Multiple \fB\-\-copy-dest\fP directories may be provided, which will cause rsync +to search the list in the order specified for an unchanged file. If a +match is not found, a basis file from one of the \fIDIRs\fP will be selected to +try to speed up the transfer. +.IP If \fIDIR\fP is a relative path, it is relative to the destination directory. -See also \fB\-\-compare\-dest\fP and \fB\-\-link\-dest\fP. -.IP -.IP "\fB\-\-link\-dest=DIR\fP" -This option behaves like \fB\-\-copy\-dest\fP, but -unchanged files are hard linked from \fIDIR\fP to the destination directory. -The files must be identical in all preserved attributes (e.g. permissions, -possibly ownership) in order for the files to be linked together. -An example: -.IP -.RS -\f(CW rsync \-av \-\-link\-dest=$PWD/prior_dir host:src_dir/ new_dir/\fP +See also \fB\-\-compare-dest\fP and \fB\-\-link-dest\fP. +.IP "\fB\-\-link-dest=DIR\fP" +This option behaves like \fB\-\-copy-dest\fP, but unchanged files are hard linked +from \fIDIR\fP to the destination directory. The files must be identical in +all preserved attributes (e.g. permissions, possibly ownership) in order +for the files to be linked together. An example: +.RS 4 +.IP +.nf +rsync -av --link-dest=$PWD/prior_dir host:src_dir/ new_dir/ +.fi .RE - -.IP -If file\(cq\&s aren\(cq\&t linking, double\-check their attributes. Also check if some -attributes are getting forced outside of rsync\(cq\&s control, such a mount option -that squishes root to a single user, or mounts a removable drive with generic -ownership (such as OS X\(cq\&s \(dq\&Ignore ownership on this volume\(dq\& option). -.IP -Beginning in version 2.6.4, multiple \fB\-\-link\-dest\fP directories may be +.IP +If file's aren't linking, double-check their attributes. Also check if +some attributes are getting forced outside of rsync's control, such a mount +option that squishes root to a single user, or mounts a removable drive +with generic ownership (such as OS X's "Ignore ownership on this volume" +option). +.IP +Beginning in version 2.6.4, multiple \fB\-\-link-dest\fP directories may be provided, which will cause rsync to search the list in the order specified -for an exact match (there is a limit of 20 such directories). -If a match is found that differs only in attributes, a local copy is made -and the attributes updated. -If a match is not found, a basis file from one of the \fIDIR\fPs will be -selected to try to speed up the transfer. -.IP +for an exact match (there is a limit of 20 such directories). If a match +is found that differs only in attributes, a local copy is made and the +attributes updated. If a match is not found, a basis file from one of the +\fIDIRs\fP will be selected to try to speed up the transfer. +.IP This option works best when copying into an empty destination hierarchy, as -existing files may get their attributes tweaked, and that can affect alternate -destination files via hard\-links. Also, itemizing of changes can get a bit -muddled. Note that prior to version 3.1.0, an alternate\-directory exact match -would never be found (nor linked into the destination) when a destination file -already exists. -.IP -Note that if you combine this option with \fB\-\-ignore\-times\fP, rsync will not +existing files may get their attributes tweaked, and that can affect +alternate destination files via hard-links. Also, itemizing of changes can +get a bit muddled. Note that prior to version 3.1.0, an +alternate-directory exact match would never be found (nor linked into the +destination) when a destination file already exists. +.IP +Note that if you combine this option with \fB\-\-ignore-times\fP, rsync will not link any files together because it only links identical files together as a -substitute for transferring the file, never as an additional check after the -file is updated. -.IP +substitute for transferring the file, never as an additional check after +the file is updated. +.IP If \fIDIR\fP is a relative path, it is relative to the destination directory. -See also \fB\-\-compare\-dest\fP and \fB\-\-copy\-dest\fP. -.IP +See also \fB\-\-compare-dest\fP and \fB\-\-copy-dest\fP. +.IP Note that rsync versions prior to 2.6.1 had a bug that could prevent -\fB\-\-link\-dest\fP from working properly for a non\-super\-user when \fB\-o\fP was -specified (or implied by \fB\-a\fP). You can work\-around this bug by avoiding +\fB\-\-link-dest\fP from working properly for a non-super-user when \fB\-o\fP was +specified (or implied by \fB\-a\fP). You can work-around this bug by avoiding the \fB\-o\fP option when sending to an old rsync. -.IP -.IP "\fB\-z, \-\-compress\fP" -With this option, rsync compresses the file data -as it is sent to the destination machine, which reduces the amount of data -being transmitted \-\- something that is useful over a slow connection. -.IP -Note that this option typically achieves better compression ratios than can -be achieved by using a compressing remote shell or a compressing transport -because it takes advantage of the implicit information in the matching data -blocks that are not explicitly sent over the connection. This matching\-data -compression comes at a cost of CPU, though, and can be disabled by repeating -the \fB\-z\fP option, but only if both sides are at least version 3.1.1. -.IP -Note that if your version of rsync was compiled with an external zlib (instead -of the zlib that comes packaged with rsync) then it will not support the -old\-style compression, only the new\-style (repeated\-option) compression. In -the future this new\-style compression will likely become the default. -.IP -The client rsync requests new\-style compression on the server via the -\fB\-\-new\-compress\fP option, so if you see that option rejected it means that -the server is not new enough to support \fB\-zz\fP. Rsync also accepts the -\fB\-\-old\-compress\fP option for a future time when new\-style compression -becomes the default. -.IP -See the \fB\-\-skip\-compress\fP option for the default list of file suffixes -that will not be compressed. -.IP -.IP "\fB\-\-compress\-level=NUM\fP" -Explicitly set the compression level to use -(see \fB\-\-compress\fP) instead of letting it default. If NUM is non\-zero, -the \fB\-\-compress\fP option is implied. -.IP -.IP "\fB\-\-skip\-compress=LIST\fP" -Override the list of file suffixes that will -not be compressed. The \fBLIST\fP should be one or more file suffixes -(without the dot) separated by slashes (/). -.IP -You may specify an empty string to indicate that no file should be skipped. -.IP -Simple character\-class matching is supported: each must consist of a list +.IP "\fB\-\-compress\fP, \fB\-z\fP" +With this option, rsync compresses the file data as it is sent to the +destination machine, which reduces the amount of data being transmitted\ \-\- +something that is useful over a slow connection. +.IP +The "zlib" compression method typically achieves better compression ratios +than can be achieved by using a compressing remote shell or a compressing +transport because it takes advantage of the implicit information in the +matching data blocks that are not explicitly sent over the connection. +This matching-data compression comes at a cost of CPU, though, and can be +disabled by using the "zlibx" compresson method instead. This can be +selected by repeating the \fB\-z\fP option or specifying +\fB\-\-compress-choice=zlibx\fP, but it only works if both sides of the transfer +are at least version 3.1.1. +.IP +Note that if you see an error about an option named \fB\-\-old-compress\fP or +\fB\-\-new-compress\fP, this is rsync trying to send the \fB\-\-compress-choice=zlib\fP +or \fB\-\-compress-choice=zlibx\fP option in a backward-compatible manner that +more rsync versions understand. This error indicates that the older rsync +version will not allow you to force the compression type. +.IP +See the \fB\-\-skip-compress\fP option for the default list of file suffixes that +will not be compressed. +.IP "\fB\-\-compress-choice=STR\fP, \fB\-\-zc=STR\fP" +This option can be used to override the automatic selection of the +compression algorithm that is the default when \fB\-\-compress\fP is used. +.IP +The compression options that you may be able to use are: +.IP +.RS +.IP o +\fBzstd\fP +.IP o +\fBlz4\fP +.IP o +\fBzlibx\fP +.IP o +\fBzlib\fP +.IP o +\fBnone\fP +.RE +.IP +Run \fBrsync\ \-V\fP to see the compress list compiled into your version. +.IP +The "zlibx" algorithm is given preference over "zlib" if both sides of the +transfer are at least version 3.2.0, otherwise it will choose "zlib" unless +you override it via something like \fB\-zz\fP. These 2 algorithms are the stame +except that "zlibx" does not try to include matched data that was not +transferred in the compression computations. +.IP +If "none" is specified, that is equivalent to using \fB\-\-no-compress\fP. +.IP +This option implies \fB\-\-compress\fP unless "none" was specified. +.IP +You can also override the compression negotiation using the +RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated +list of compression names that you consider acceptable. If no common +compress choice is found, the client exits with an error. It ignores +"auto" and all unknown compression names. If the remote rsync is not new +enough to handle a compression negotiation list, the list is silently +ignored unless it contains the string "FAIL". +.IP "\fB\-\-compress-level=NUM\fP" +Explicitly set the compression level to use (see \fB\-\-compress\fP) instead of +letting it default. If NUM is non-zero, the \fB\-\-compress\fP option is +implied. +.IP "\fB\-\-skip-compress=LIST\fP" +Override the list of file suffixes that will be compressed as little as +possible. Rsync sets the compression level on a per-file basis based on +the file's suffix. If the compression algorithm has an "off" level (such +as zlib/zlibx) then no compression occurs for those files. Other +algorithms have the level minimized to reduces the CPU usage as much as +possible. +.IP +The \fBLIST\fP should be one or more file suffixes (without the dot) separated +by slashes (\fB/\fP). You may specify an empty string to indicate that no files +should be skipped. +.IP +Simple character-class matching is supported: each must consist of a list of letters inside the square brackets (e.g. no special classes, such as -\(dq\&[:alpha:]\(dq\&, are supported, and \(cq\&\-\(cq\& has no special meaning). -.IP -The characters asterisk (*) and question\-mark (?) have no special meaning. -.IP -Here\(cq\&s an example that specifies 6 suffixes to skip (since 1 of the 5 rules +"[:alpha:]", are supported, and '\-' has no special meaning). +.IP +The characters asterisk (\fB*\fP) and question-mark (\fB?\fP) have no special meaning. +.IP +Here's an example that specifies 6 suffixes to skip (since 1 of the 5 rules matches 2 suffixes): -.IP -.nf - \-\-skip\-compress=gz/jpg/mp[34]/7z/bz2 -.fi - -.IP -The default list of suffixes that will not be compressed is this (in this -version of rsync): -.IP -\fB7z\fP -\fBace\fP -\fBavi\fP -\fBbz2\fP -\fBdeb\fP -\fBgpg\fP -\fBgz\fP -\fBiso\fP -\fBjpeg\fP -\fBjpg\fP -\fBlz\fP -\fBlzma\fP -\fBlzo\fP -\fBmov\fP -\fBmp3\fP -\fBmp4\fP -\fBogg\fP -\fBpng\fP -\fBrar\fP -\fBrpm\fP -\fBrzip\fP -\fBtbz\fP -\fBtgz\fP -\fBtlz\fP -\fBtxz\fP -\fBxz\fP -\fBz\fP -\fBzip\fP -.IP -This list will be replaced by your \fB\-\-skip\-compress\fP list in all but one -situation: a copy from a daemon rsync will add your skipped suffixes to -its list of non\-compressing files (and its list may be configured to a +.RS 4 +.IP +.nf +--skip-compress=gz/jpg/mp[34]/7z/bz2 +.fi +.RE +.IP +The default file suffixes in the skip-compress list in this version of +rsync are: +.RS 4 +.IP +7z +ace +apk +avi +bz2 +deb +flac +gpg +gz +iso +jar +jpeg +jpg +lz +lz4 +lzma +lzo +mkv +mov +mp3 +mp4 +odb +odf +odg +odi +odm +odp +ods +odt +ogg +ogv +opus +otg +oth +otp +ots +ott +oxt +png +rar +rpm +rz +rzip +squashfs +sxc +sxd +sxg +sxm +sxw +tbz +tgz +tlz +txz +tzo +webm +webp +xz +z +zip +zst +.RE +.IP +This list will be replaced by your \fB\-\-skip-compress\fP list in all but one +situation: a copy from a daemon rsync will add your skipped suffixes to its +list of non-compressing files (and its list may be configured to a different default). -.IP -.IP "\fB\-\-numeric\-ids\fP" -With this option rsync will transfer numeric group -and user IDs rather than using user and group names and mapping them -at both ends. -.IP -By default rsync will use the username and groupname to determine -what ownership to give files. The special uid 0 and the special group -0 are never mapped via user/group names even if the \fB\-\-numeric\-ids\fP -option is not specified. -.IP -If a user or group has no name on the source system or it has no match -on the destination system, then the numeric ID -from the source system is used instead. See also the comments on the -\(dq\&use chroot\(dq\& setting in the rsyncd.conf manpage for information on how -the chroot setting affects rsync\(cq\&s ability to look up the names of the -users and groups and what you can do about it. -.IP -.IP "\fB\-\-usermap=STRING, \-\-groupmap=STRING\fP" -These options allow you to -specify users and groups that should be mapped to other values by the -receiving side. The \fBSTRING\fP is one or more \fBFROM\fP:\fBTO\fP pairs of -values separated by commas. Any matching \fBFROM\fP value from the sender is -replaced with a \fBTO\fP value from the receiver. You may specify usernames -or user IDs for the \fBFROM\fP and \fBTO\fP values, and the \fBFROM\fP value may -also be a wild\-card string, which will be matched against the sender\(cq\&s -names (wild\-cards do NOT match against ID numbers, though see below for -why a \(cq\&*\(cq\& matches everything). You may instead specify a range of ID -numbers via an inclusive range: LOW\-HIGH. For example: -.IP -.nf - \-\-usermap=0\-99:nobody,wayne:admin,*:normal \-\-groupmap=usr:1,1:usr -.fi - -.IP +.IP "\fB\-\-numeric-ids\fP" +With this option rsync will transfer numeric group and user IDs rather than +using user and group names and mapping them at both ends. +.IP +By default rsync will use the username and groupname to determine what +ownership to give files. The special uid 0 and the special group 0 are +never mapped via user/group names even if the \fB\-\-numeric-ids\fP option is not +specified. +.IP +If a user or group has no name on the source system or it has no match on +the destination system, then the numeric ID from the source system is used +instead. See also the comments on the "\fBuse\ chroot\fP" setting in the +rsyncd.conf manpage for information on how the chroot setting affects +rsync's ability to look up the names of the users and groups and what you +can do about it. +.IP "\fB\-\-usermap=STRING\fP, \fB\-\-groupmap=STRING\fP" +These options allow you to specify users and groups that should be mapped +to other values by the receiving side. The \fBSTRING\fP is one or more +\fBFROM\fP:\fBTO\fP pairs of values separated by commas. Any matching \fBFROM\fP +value from the sender is replaced with a \fBTO\fP value from the receiver. +You may specify usernames or user IDs for the \fBFROM\fP and \fBTO\fP values, +and the \fBFROM\fP value may also be a wild-card string, which will be +matched against the sender's names (wild-cards do NOT match against ID +numbers, though see below for why a '\fB*\fP' matches everything). You may +instead specify a range of ID numbers via an inclusive range: LOW-HIGH. +For example: +.RS 4 +.IP +.nf +--usermap=0-99:nobody,wayne:admin,*:normal --groupmap=usr:1,1:usr +.fi +.RE +.IP The first match in the list is the one that is used. You should specify -all your user mappings using a single \fB\-\-usermap\fP option, and/or all -your group mappings using a single \fB\-\-groupmap\fP option. -.IP -Note that the sender\(cq\&s name for the 0 user and group are not transmitted -to the receiver, so you should either match these values using a 0, or use -the names in effect on the receiving side (typically \(dq\&root\(dq\&). All other +all your user mappings using a single \fB\-\-usermap\fP option, and/or all your +group mappings using a single \fB\-\-groupmap\fP option. +.IP +Note that the sender's name for the 0 user and group are not transmitted to +the receiver, so you should either match these values using a 0, or use the +names in effect on the receiving side (typically "root"). All other \fBFROM\fP names match those in use on the sending side. All \fBTO\fP names match those in use on the receiving side. -.IP -Any IDs that do not have a name on the sending side are treated as having an -empty name for the purpose of matching. This allows them to be matched via -a \(dq\&*\(dq\& or using an empty name. For instance: -.IP -.nf - \-\-usermap=:nobody \-\-groupmap=*:nobody -.fi - -.IP -When the \fB\-\-numeric\-ids\fP option is used, the sender does not send any +.IP +Any IDs that do not have a name on the sending side are treated as having +an empty name for the purpose of matching. This allows them to be matched +via a "\fB*\fP" or using an empty name. For instance: +.RS 4 +.IP +.nf +--usermap=:nobody --groupmap=*:nobody +.fi +.RE +.IP +When the \fB\-\-numeric-ids\fP option is used, the sender does not send any names, so all the IDs are treated as having an empty name. This means that you will need to specify numeric \fBFROM\fP values if you want to map these nameless IDs to different values. -.IP -For the \fB\-\-usermap\fP option to have any effect, the \fB\-o\fP (\fB\-\-owner\fP) -option must be used (or implied), and the receiver will need to be running -as a super\-user (see also the \fB\-\-fake\-super\fP option). For the \fB\-\-groupmap\fP -option to have any effect, the \fB\-g\fP (\fB\-\-groups\fP) option must be used -(or implied), and the receiver will need to have permissions to set that -group. -.IP +.IP +For the \fB\-\-usermap\fP option to have any effect, the \fB\-o\fP (\fB\-\-owner\fP) option +must be used (or implied), and the receiver will need to be running as a +super-user (see also the \fB\-\-fake-super\fP option). For the \fB\-\-groupmap\fP +option to have any effect, the \fB\-g\fP (\fB\-\-groups\fP) option must be used (or +implied), and the receiver will need to have permissions to set that group. .IP "\fB\-\-chown=USER:GROUP\fP" -This option forces all files to be owned by USER -with group GROUP. This is a simpler interface than using \fB\-\-usermap\fP and -\fB\-\-groupmap\fP directly, but it is implemented using those options internally, -so you cannot mix them. If either the USER or GROUP is empty, no mapping for -the omitted user/group will occur. If GROUP is empty, the trailing colon may -be omitted, but if USER is empty, a leading colon must be supplied. -.IP -If you specify \(dq\&\-\-chown=foo:bar, this is exactly the same as specifying -\(dq\&\-\-usermap=*:foo \-\-groupmap=*:bar\(dq\&, only easier. -.IP +This option forces all files to be owned by USER with group GROUP. This is +a simpler interface than using \fB\-\-usermap\fP and \fB\-\-groupmap\fP directly, but +it is implemented using those options internally, so you cannot mix them. +If either the USER or GROUP is empty, no mapping for the omitted user/group +will occur. If GROUP is empty, the trailing colon may be omitted, but if +USER is empty, a leading colon must be supplied. +.IP +If you specify "\fB\-\-chown=foo:bar\fP", this is exactly the same as specifying +"\fB\-\-usermap=*:foo\ \-\-groupmap=*:bar\fP", only easier. .IP "\fB\-\-timeout=TIMEOUT\fP" -This option allows you to set a maximum I/O -timeout in seconds. If no data is transferred for the specified time -then rsync will exit. The default is 0, which means no timeout. -.IP +This option allows you to set a maximum I/O timeout in seconds. If no data +is transferred for the specified time then rsync will exit. The default is +0, which means no timeout. .IP "\fB\-\-contimeout\fP" -This option allows you to set the amount of time -that rsync will wait for its connection to an rsync daemon to succeed. -If the timeout is reached, rsync exits with an error. -.IP +This option allows you to set the amount of time that rsync will wait for +its connection to an rsync daemon to succeed. If the timeout is reached, +rsync exits with an error. .IP "\fB\-\-address\fP" -By default rsync will bind to the wildcard address when -connecting to an rsync daemon. The \fB\-\-address\fP option allows you to -specify a specific IP address (or hostname) to bind to. See also this -option in the \fB\-\-daemon\fP mode section. -.IP +By default rsync will bind to the wildcard address when connecting to an +rsync daemon. The \fB\-\-address\fP option allows you to specify a specific IP +address (or hostname) to bind to. See also this option in the \fB\-\-daemon\fP +mode section. .IP "\fB\-\-port=PORT\fP" -This specifies an alternate TCP port number to use -rather than the default of 873. This is only needed if you are using the -double\-colon (::) syntax to connect with an rsync daemon (since the URL -syntax has a way to specify the port as a part of the URL). See also this -option in the \fB\-\-daemon\fP mode section. -.IP +This specifies an alternate TCP port number to use rather than the default +of 873. This is only needed if you are using the double-colon (::) syntax +to connect with an rsync daemon (since the URL syntax has a way to specify +the port as a part of the URL). See also this option in the \fB\-\-daemon\fP +mode section. .IP "\fB\-\-sockopts\fP" -This option can provide endless fun for people -who like to tune their systems to the utmost degree. You can set all -sorts of socket options which may make transfers faster (or -slower!). Read the man page for the -\f(CWsetsockopt()\fP -system call for -details on some of the options you may be able to set. By default no -special socket options are set. This only affects direct socket -connections to a remote rsync daemon. This option also exists in the -\fB\-\-daemon\fP mode section. -.IP -.IP "\fB\-\-blocking\-io\fP" -This tells rsync to use blocking I/O when launching -a remote shell transport. If the remote shell is either rsh or remsh, -rsync defaults to using -blocking I/O, otherwise it defaults to using non\-blocking I/O. (Note that -ssh prefers non\-blocking I/O.) -.IP +This option can provide endless fun for people who like to tune their +systems to the utmost degree. You can set all sorts of socket options +which may make transfers faster (or slower!). Read the man page for the +\fBsetsockopt()\fP system call for details on some of the options you may be +able to set. By default no special socket options are set. This only +affects direct socket connections to a remote rsync daemon. +.IP +This option also exists in the \fB\-\-daemon\fP mode section. +.IP "\fB\-\-blocking-io\fP" +This tells rsync to use blocking I/O when launching a remote shell +transport. If the remote shell is either rsh or remsh, rsync defaults to +using blocking I/O, otherwise it defaults to using non-blocking I/O. (Note +that ssh prefers non-blocking I/O.) .IP "\fB\-\-outbuf=MODE\fP" -This sets the output buffering mode. The mode can be -None (aka Unbuffered), Line, or Block (aka Full). You may specify as little -as a single letter for the mode, and use upper or lower case. -.IP +This sets the output buffering mode. The mode can be None (aka +Unbuffered), Line, or Block (aka Full). You may specify as little as a +single letter for the mode, and use upper or lower case. +.IP The main use of this option is to change Full buffering to Line buffering -when rsync\(cq\&s output is going to a file or pipe. -.IP -.IP "\fB\-i, \-\-itemize\-changes\fP" -Requests a simple itemized list of the -changes that are being made to each file, including attribute changes. -This is exactly the same as specifying \fB\-\-out\-format='\&%i %n%L'\&\fP. -If you repeat the option, unchanged files will also be output, but only -if the receiving rsync is at least version 2.6.7 (you can use \fB\-vv\fP -with older versions of rsync, but that also turns on the output of other -verbose messages). -.IP -The \(dq\&%i\(dq\& escape has a cryptic output that is 11 letters long. The general -format is like the string \fBYXcstpoguax\fP, where \fBY\fP is replaced by the -type of update being done, \fBX\fP is replaced by the file\-type, and the -other letters represent attributes that may be output if they are being -modified. -.IP +when rsync's output is going to a file or pipe. +.IP "\fB\-\-itemize-changes\fP, \fB\-i\fP" +Requests a simple itemized list of the changes that are being made to each +file, including attribute changes. This is exactly the same as specifying +\fB\-\-out-format='%i\ %n%L'\fP. If you repeat the option, unchanged files will +also be output, but only if the receiving rsync is at least version 2.6.7 +(you can use \fB\-vv\fP with older versions of rsync, but that also turns on the +output of other verbose messages). +.IP +The "%i" escape has a cryptic output that is 11 letters long. The general +format is like the string \fBYXcstpoguax\fP, where \fBY\fP is replaced by the type +of update being done, \fBX\fP is replaced by the file-type, and the other +letters represent attributes that may be output if they are being modified. +.IP The update types that replace the \fBY\fP are as follows: -.IP -.RS -.IP o -A \fB<\fP means that a file is being transferred to the remote host -(sent). -.IP o +.IP +.RS +.IP o +A \fB<\fP means that a file is being transferred to the remote host (sent). +.IP o A \fB>\fP means that a file is being transferred to the local host (received). -.IP o -A \fBc\fP means that a local change/creation is occurring for the item -(such as the creation of a directory or the changing of a symlink, etc.). -.IP o +.IP o +A \fBc\fP means that a local change/creation is occurring for the item (such +as the creation of a directory or the changing of a symlink, etc.). +.IP o A \fBh\fP means that the item is a hard link to another item (requires -\fB\-\-hard\-links\fP). -.IP o -A \fB.\fP means that the item is not being updated (though it might -have attributes that are being modified). -.IP o -A \fB*\fP means that the rest of the itemized\-output area contains -a message (e.g. \(dq\&deleting\(dq\&). +\fB\-\-hard-links\fP). +.IP o +A \fB.\fP means that the item is not being updated (though it might have +attributes that are being modified). +.IP o +A \fB*\fP means that the rest of the itemized-output area contains a message +(e.g. "deleting"). .RE - -.IP -The file\-types that replace the \fBX\fP are: \fBf\fP for a file, a \fBd\fP for a +.IP +The file-types that replace the \fBX\fP are: \fBf\fP for a file, a \fBd\fP for a directory, an \fBL\fP for a symlink, a \fBD\fP for a device, and a \fBS\fP for a special file (e.g. named sockets and fifos). -.IP -The other letters in the string above are the actual letters that -will be output if the associated attribute for the item is being updated or -a \(dq\&.\(dq\& for no change. Three exceptions to this are: (1) a newly created -item replaces each letter with a \(dq\&+\(dq\&, (2) an identical item replaces the -dots with spaces, and (3) an unknown attribute replaces each letter with -a \(dq\&?\(dq\& (this can happen when talking to an older rsync). -.IP +.IP +The other letters in the string above are the actual letters that will be +output if the associated attribute for the item is being updated or a "." +for no change. Three exceptions to this are: (1) a newly created item +replaces each letter with a "+", (2) an identical item replaces the dots +with spaces, and (3) an unknown attribute replaces each letter with a "?" +(this can happen when talking to an older rsync). +.IP The attribute that is associated with each letter is as follows: -.IP -.RS -.IP o -A \fBc\fP means either that a regular file has a different checksum -(requires \fB\-\-checksum\fP) or that a symlink, device, or special file has -a changed value. -Note that if you are sending files to an rsync prior to 3.0.1, this -change flag will be present only for checksum\-differing regular files. -.IP o +.IP +.RS +.IP o +A \fBc\fP means either that a regular file has a different checksum (requires +\fB\-\-checksum\fP) or that a symlink, device, or special file has a changed +value. Note that if you are sending files to an rsync prior to 3.0.1, +this change flag will be present only for checksum-differing regular +files. +.IP o A \fBs\fP means the size of a regular file is different and will be updated by the file transfer. -.IP o -A \fBt\fP means the modification time is different and is being updated -to the sender\(cq\&s value (requires \fB\-\-times\fP). An alternate value of \fBT\fP -means that the modification time will be set to the transfer time, which happens -when a file/symlink/device is updated without \fB\-\-times\fP and when a -symlink is changed and the receiver can\(cq\&t set its time. -(Note: when using an rsync 3.0.0 client, you might see the \fBs\fP flag combined -with \fBt\fP instead of the proper \fBT\fP flag for this time\-setting failure.) -.IP o -A \fBp\fP means the permissions are different and are being updated to -the sender\(cq\&s value (requires \fB\-\-perms\fP). -.IP o -An \fBo\fP means the owner is different and is being updated to the -sender\(cq\&s value (requires \fB\-\-owner\fP and super\-user privileges). -.IP o -A \fBg\fP means the group is different and is being updated to the -sender\(cq\&s value (requires \fB\-\-group\fP and the authority to set the group). -.IP o -The \fBu\fP slot is reserved for future use. -.IP o +.IP o +A \fBt\fP means the modification time is different and is being updated to +the sender's value (requires \fB\-\-times\fP). An alternate value of \fBT\fP means +that the modification time will be set to the transfer time, which +happens when a file/symlink/device is updated without \fB\-\-times\fP and when +a symlink is changed and the receiver can't set its time. (Note: when +using an rsync 3.0.0 client, you might see the \fBs\fP flag combined with \fBt\fP +instead of the proper \fBT\fP flag for this time-setting failure.) +.IP o +A \fBp\fP means the permissions are different and are being updated to the +sender's value (requires \fB\-\-perms\fP). +.IP o +An \fBo\fP means the owner is different and is being updated to the sender's +value (requires \fB\-\-owner\fP and super-user privileges). +.IP o +A \fBg\fP means the group is different and is being updated to the sender's +value (requires \fB\-\-group\fP and the authority to set the group). +.IP o +A \fBu\fP means the access (use) time is different and is being updated to +the sender's value (requires \fB\-\-atimes\fP). An alternate value of \fBU\fP +means that the access time will be set to the transfer time, which +happens when a symlink or directory is updated. +.IP o The \fBa\fP means that the ACL information changed. -.IP o +.IP o The \fBx\fP means that the extended attribute information changed. .RE - -.IP -One other output is possible: when deleting files, the \(dq\&%i\(dq\& will output -the string \(dq\&*deleting\(dq\& for each item that is being removed (assuming that -you are talking to a recent enough rsync that it logs deletions instead of +.IP +One other output is possible: when deleting files, the "%i" will output the +string "\fB*deleting\fP" for each item that is being removed (assuming that you +are talking to a recent enough rsync that it logs deletions instead of outputting them as a verbose message). -.IP -.IP "\fB\-\-out\-format=FORMAT\fP" -This allows you to specify exactly what the -rsync client outputs to the user on a per\-update basis. The format is a -text string containing embedded single\-character escape sequences prefixed -with a percent (%) character. A default format of \(dq\&%n%L\(dq\& is assumed if -either \fB\-\-info=name\fP or \fB\-v\fP is specified (this tells you just the name -of the file and, if the item is a link, where it points). For a full list -of the possible escape characters, see the \(dq\&log format\(dq\& setting in the -rsyncd.conf manpage. -.IP -Specifying the \fB\-\-out\-format\fP option implies the \fB\-\-info=name\fP option, +.IP "\fB\-\-out-format=FORMAT\fP" +This allows you to specify exactly what the rsync client outputs to the +user on a per-update basis. The format is a text string containing +embedded single-character escape sequences prefixed with a percent (%) +character. A default format of "%n%L" is assumed if either \fB\-\-info=name\fP +or \fB\-v\fP is specified (this tells you just the name of the file and, if the +item is a link, where it points). For a full list of the possible escape +characters, see the "\fBlog\ format\fP" setting in the rsyncd.conf manpage. +.IP +Specifying the \fB\-\-out-format\fP option implies the \fB\-\-info=name\fP option, which will mention each file, dir, etc. that gets updated in a significant way (a transferred file, a recreated symlink/device, or a touched -directory). In addition, if the itemize\-changes escape (%i) is included in -the string (e.g. if the \fB\-\-itemize\-changes\fP option was used), the logging +directory). In addition, if the itemize-changes escape (%i) is included in +the string (e.g. if the \fB\-\-itemize-changes\fP option was used), the logging of names increases to mention any item that is changed in any way (as long -as the receiving side is at least 2.6.4). See the \fB\-\-itemize\-changes\fP -option for a description of the output of \(dq\&%i\(dq\&. -.IP -Rsync will output the out\-format string prior to a file\(cq\&s transfer unless -one of the transfer\-statistic escapes is requested, in which case the -logging is done at the end of the file\(cq\&s transfer. When this late logging -is in effect and \fB\-\-progress\fP is also specified, rsync will also output -the name of the file being transferred prior to its progress information -(followed, of course, by the out\-format output). -.IP -.IP "\fB\-\-log\-file=FILE\fP" -This option causes rsync to log what it is doing -to a file. This is similar to the logging that a daemon does, but can be -requested for the client side and/or the server side of a non\-daemon -transfer. If specified as a client option, transfer logging will be -enabled with a default format of \(dq\&%i %n%L\(dq\&. See the \fB\-\-log\-file\-format\fP -option if you wish to override this. -.IP -Here\(cq\&s a example command that requests the remote side to log what is +as the receiving side is at least 2.6.4). See the \fB\-\-itemize-changes\fP +option for a description of the output of "%i". +.IP +Rsync will output the out-format string prior to a file's transfer unless +one of the transfer-statistic escapes is requested, in which case the +logging is done at the end of the file's transfer. When this late logging +is in effect and \fB\-\-progress\fP is also specified, rsync will also output the +name of the file being transferred prior to its progress information +(followed, of course, by the out-format output). +.IP "\fB\-\-log-file=FILE\fP" +This option causes rsync to log what it is doing to a file. This is +similar to the logging that a daemon does, but can be requested for the +client side and/or the server side of a non-daemon transfer. If specified +as a client option, transfer logging will be enabled with a default format +of "%i %n%L". See the \fB\-\-log-file-format\fP option if you wish to override +this. +.IP +Here's a example command that requests the remote side to log what is happening: -.IP -.nf - rsync \-av \-\-remote\-option=\-\-log\-file=/tmp/rlog src/ dest/ -.fi - -.IP +.RS 4 +.IP +.nf +rsync -av --remote-option=--log-file=/tmp/rlog src/ dest/ +.fi +.RE +.IP This is very useful if you need to debug why a connection is closing unexpectedly. -.IP -.IP "\fB\-\-log\-file\-format=FORMAT\fP" -This allows you to specify exactly what -per\-update logging is put into the file specified by the \fB\-\-log\-file\fP option -(which must also be specified for this option to have any effect). If you -specify an empty string, updated files will not be mentioned in the log file. -For a list of the possible escape characters, see the \(dq\&log format\(dq\& setting -in the rsyncd.conf manpage. -.IP -The default FORMAT used if \fB\-\-log\-file\fP is specified and this option is not -is \(cq\&%i %n%L\(cq\&. -.IP +.IP "\fB\-\-log-file-format=FORMAT\fP" +This allows you to specify exactly what per-update logging is put into the +file specified by the \fB\-\-log-file\fP option (which must also be specified for +this option to have any effect). If you specify an empty string, updated +files will not be mentioned in the log file. For a list of the possible +escape characters, see the "\fBlog\ format\fP" setting in the rsyncd.conf manpage. +.IP +The default FORMAT used if \fB\-\-log-file\fP is specified and this option is not +is '%i %n%L'. .IP "\fB\-\-stats\fP" -This tells rsync to print a verbose set of statistics -on the file transfer, allowing you to tell how effective rsync\(cq\&s delta\-transfer -algorithm is for your data. This option is equivalent to \fB\-\-info=stats2\fP -if combined with 0 or 1 \fB\-v\fP options, or \fB\-\-info=stats3\fP if combined -with 2 or more \fB\-v\fP options. -.IP -The current statistics are as follows: -.RS -.IP o -\fBNumber of files\fP is the count of all \(dq\&files\(dq\& (in the generic -sense), which includes directories, symlinks, etc. The total count will -be followed by a list of counts by filetype (if the total is non\-zero). -For example: \(dq\&(reg: 5, dir: 3, link: 2, dev: 1, special: 1)\(dq\& lists the -totals for regular files, directories, symlinks, devices, and special -files. If any of value is 0, it is completely omitted from the list. -.IP o -\fBNumber of created files\fP is the count of how many \(dq\&files\(dq\& (generic +This tells rsync to print a verbose set of statistics on the file transfer, +allowing you to tell how effective rsync's delta-transfer algorithm is for +your data. This option is equivalent to \fB\-\-info=stats2\fP if combined with 0 +or 1 \fB\-v\fP options, or \fB\-\-info=stats3\fP if combined with 2 or more \fB\-v\fP +options. +.IP +The current statistics are as follows: +.IP +.RS +.IP o +\fBNumber\ of\ files\fP is the count of all "files" (in the generic sense), +which includes directories, symlinks, etc. The total count will be +followed by a list of counts by filetype (if the total is non-zero). For +example: "(reg: 5, dir: 3, link: 2, dev: 1, special: 1)" lists the totals +for regular files, directories, symlinks, devices, and special files. If +any of value is 0, it is completely omitted from the list. +.IP o +\fBNumber\ of\ created\ files\fP is the count of how many "files" (generic sense) were created (as opposed to updated). The total count will be -followed by a list of counts by filetype (if the total is non\-zero). -.IP o -\fBNumber of deleted files\fP is the count of how many \(dq\&files\(dq\& (generic +followed by a list of counts by filetype (if the total is non-zero). +.IP o +\fBNumber\ of\ deleted\ files\fP is the count of how many "files" (generic sense) were created (as opposed to updated). The total count will be -followed by a list of counts by filetype (if the total is non\-zero). +followed by a list of counts by filetype (if the total is non-zero). Note that this line is only output if deletions are in effect, and only if protocol 31 is being used (the default for rsync 3.1.x). -.IP o -\fBNumber of regular files transferred\fP is the count of normal files -that were updated via rsync\(cq\&s delta\-transfer algorithm, which does not -include dirs, symlinks, etc. Note that rsync 3.1.0 added the word -\(dq\®ular\(dq\& into this heading. -.IP o -\fBTotal file size\fP is the total sum of all file sizes in the transfer. +.IP o +\fBNumber\ of\ regular\ files\ transferred\fP is the count of normal files that +were updated via rsync's delta-transfer algorithm, which does not include +dirs, symlinks, etc. Note that rsync 3.1.0 added the word "regular" into +this heading. +.IP o +\fBTotal\ file\ size\fP is the total sum of all file sizes in the transfer. This does not count any size for directories or special files, but does include the size of symlinks. -.IP o -\fBTotal transferred file size\fP is the total sum of all files sizes -for just the transferred files. -.IP o -\fBLiteral data\fP is how much unmatched file\-update data we had to -send to the receiver for it to recreate the updated files. -.IP o -\fBMatched data\fP is how much data the receiver got locally when -recreating the updated files. -.IP o -\fBFile list size\fP is how big the file\-list data was when the sender -sent it to the receiver. This is smaller than the in\-memory size for the -file list due to some compressing of duplicated data when rsync sends the +.IP o +\fBTotal\ transferred\ file\ size\fP is the total sum of all files sizes for +just the transferred files. +.IP o +\fBLiteral\ data\fP is how much unmatched file-update data we had to send to +the receiver for it to recreate the updated files. +.IP o +\fBMatched\ data\fP is how much data the receiver got locally when recreating +the updated files. +.IP o +\fBFile\ list\ size\fP is how big the file-list data was when the sender sent +it to the receiver. This is smaller than the in-memory size for the file +list due to some compressing of duplicated data when rsync sends the list. -.IP o -\fBFile list generation time\fP is the number of seconds that the -sender spent creating the file list. This requires a modern rsync on the +.IP o +\fBFile\ list\ generation\ time\fP is the number of seconds that the sender +spent creating the file list. This requires a modern rsync on the sending side for this to be present. -.IP o -\fBFile list transfer time\fP is the number of seconds that the sender -spent sending the file list to the receiver. -.IP o -\fBTotal bytes sent\fP is the count of all the bytes that rsync sent -from the client side to the server side. -.IP o -\fBTotal bytes received\fP is the count of all non\-message bytes that -rsync received by the client side from the server side. \(dq\&Non\-message\(dq\& -bytes means that we don\(cq\&t count the bytes for a verbose message that the -server sent to us, which makes the stats more consistent. +.IP o +\fBFile\ list\ transfer\ time\fP is the number of seconds that the sender spent +sending the file list to the receiver. +.IP o +\fBTotal\ bytes\ sent\fP is the count of all the bytes that rsync sent from the +client side to the server side. +.IP o +\fBTotal\ bytes\ received\fP is the count of all non-message bytes that rsync +received by the client side from the server side. "Non-message" bytes +means that we don't count the bytes for a verbose message that the server +sent to us, which makes the stats more consistent. .RE - -.IP -.IP "\fB\-8, \-\-8\-bit\-output\fP" -This tells rsync to leave all high\-bit characters -unescaped in the output instead of trying to test them to see if they\(cq\&re -valid in the current locale and escaping the invalid ones. All control -characters (but never tabs) are always escaped, regardless of this option\(cq\&s -setting. -.IP -The escape idiom that started in 2.6.7 is to output a literal backslash (\e) -and a hash (#), followed by exactly 3 octal digits. For example, a newline -would output as \(dq\&\e#012\(dq\&. A literal backslash that is in a filename is not -escaped unless it is followed by a hash and 3 digits (0\-9). -.IP -.IP "\fB\-h, \-\-human\-readable\fP" -Output numbers in a more human\-readable format. -There are 3 possible levels: (1) output numbers with a separator between each -set of 3 digits (either a comma or a period, depending on if the decimal point -is represented by a period or a comma); (2) output numbers in units of 1000 -(with a character suffix for larger units \-\- see below); (3) output numbers in -units of 1024. -.IP -The default is human\-readable level 1. Each \fB\-h\fP option increases the level -by one. You can take the level down to 0 (to output numbers as pure digits) by -specifing the \fB\-\-no\-human\-readable\fP (\fB\-\-no\-h\fP) option. -.IP -The unit letters that are appended in levels 2 and 3 are: K (kilo), M (mega), -G (giga), or T (tera). For example, a 1234567\-byte file would output as 1.23M -in level\-2 (assuming that a period is your local decimal point). -.IP -Backward compatibility note: versions of rsync prior to 3.1.0 do not support -human\-readable level 1, and they default to level 0. Thus, specifying one or -two \fB\-h\fP options will behave in a comparable manner in old and new versions -as long as you didn\(cq\&t specify a \fB\-\-no\-h\fP option prior to one or more \fB\-h\fP -options. See the \fB\-\-list\-only\fP option for one difference. -.IP +.IP "\fB\-\-8-bit-output\fP, \fB\-8\fP" +This tells rsync to leave all high-bit characters unescaped in the output +instead of trying to test them to see if they're valid in the current +locale and escaping the invalid ones. All control characters (but never +tabs) are always escaped, regardless of this option's setting. +.IP +The escape idiom that started in 2.6.7 is to output a literal backslash +(\fB\\\fP) and a hash (\fB#\fP), followed by exactly 3 octal digits. For example, a +newline would output as "\fB\\#012\fP". A literal backslash that is in a +filename is not escaped unless it is followed by a hash and 3 digits (0-9). +.IP "\fB\-\-human-readable\fP, \fB\-h\fP" +Output numbers in a more human-readable format. There are 3 possible +levels: (1) output numbers with a separator between each set of 3 digits +(either a comma or a period, depending on if the decimal point is +represented by a period or a comma); (2) output numbers in units of 1000 +(with a character suffix for larger units\ \-\- see below); (3) output +numbers in units of 1024. +.IP +The default is human-readable level 1. Each \fB\-h\fP option increases the +level by one. You can take the level down to 0 (to output numbers as pure +digits) by specifying the \fB\-\-no-human-readable\fP (\fB\-\-no-h\fP) option. +.IP +The unit letters that are appended in levels 2 and 3 are: K (kilo), M +(mega), G (giga), or T (tera). For example, a 1234567-byte file would +output as 1.23M in level-2 (assuming that a period is your local decimal +point). +.IP +Backward compatibility note: versions of rsync prior to 3.1.0 do not +support human-readable level 1, and they default to level 0. Thus, +specifying one or two \fB\-h\fP options will behave in a comparable manner in +old and new versions as long as you didn't specify a \fB\-\-no-h\fP option prior +to one or more \fB\-h\fP options. See the \fB\-\-list-only\fP option for one +difference. .IP "\fB\-\-partial\fP" -By default, rsync will delete any partially -transferred file if the transfer is interrupted. In some circumstances -it is more desirable to keep partially transferred files. Using the -\fB\-\-partial\fP option tells rsync to keep the partial file which should -make a subsequent transfer of the rest of the file much faster. -.IP -.IP "\fB\-\-partial\-dir=DIR\fP" -A better way to keep partial files than the -\fB\-\-partial\fP option is to specify a \fIDIR\fP that will be used to hold the -partial data (instead of writing it out to the destination file). -On the next transfer, rsync will use a file found in this -dir as data to speed up the resumption of the transfer and then delete it -after it has served its purpose. -.IP -Note that if \fB\-\-whole\-file\fP is specified (or implied), any partial\-dir -file that is found for a file that is being updated will simply be removed -(since -rsync is sending files without using rsync\(cq\&s delta\-transfer algorithm). -.IP -Rsync will create the \fIDIR\fP if it is missing (just the last dir \-\- not -the whole path). This makes it easy to use a relative path (such as -\(dq\&\fB\-\-partial\-dir=.rsync\-partial\fP\(dq\&) to have rsync create the -partial\-directory in the destination file\(cq\&s directory when needed, and then -remove it again when the partial file is deleted. -.IP -If the partial\-dir value is not an absolute path, rsync will add an exclude +By default, rsync will delete any partially transferred file if the +transfer is interrupted. In some circumstances it is more desirable to +keep partially transferred files. Using the \fB\-\-partial\fP option tells rsync +to keep the partial file which should make a subsequent transfer of the +rest of the file much faster. +.IP "\fB\-\-partial-dir=DIR\fP" +A better way to keep partial files than the \fB\-\-partial\fP option is to +specify a \fIDIR\fP that will be used to hold the partial data (instead of +writing it out to the destination file). On the next transfer, rsync will +use a file found in this dir as data to speed up the resumption of the +transfer and then delete it after it has served its purpose. +.IP +Note that if \fB\-\-whole-file\fP is specified (or implied), any partial-dir file +that is found for a file that is being updated will simply be removed +(since rsync is sending files without using rsync's delta-transfer +algorithm). +.IP +Rsync will create the \fIDIR\fP if it is missing (just the last dir\ \-\- not the +whole path). This makes it easy to use a relative path (such as +"\fB\-\-partial-dir=.rsync-partial\fP") to have rsync create the +partial-directory in the destination file's directory when needed, and then +remove it again when the partial file is deleted. Note that the directory +is only removed if it is a relative pathname, as it is expected that an +absolute path is to a directory that is reserved for partial-dir work. +.IP +If the partial-dir value is not an absolute path, rsync will add an exclude rule at the end of all your existing excludes. This will prevent the -sending of any partial\-dir files that may exist on the sending side, and -will also prevent the untimely deletion of partial\-dir items on the -receiving side. An example: the above \fB\-\-partial\-dir\fP option would add -the equivalent of \(dq\&\fB\-f '\&\-p .rsync\-partial/'\&\fP\(dq\& at the end of any other -filter rules. -.IP +sending of any partial-dir files that may exist on the sending side, and +will also prevent the untimely deletion of partial-dir items on the +receiving side. An example: the above \fB\-\-partial-dir\fP option would add the +equivalent of "\fB\-f\ '\-p\ .rsync-partial/'\fP" at the end of any other filter +rules. +.IP If you are supplying your own exclude rules, you may need to add your own -exclude/hide/protect rule for the partial\-dir because (1) the auto\-added +exclude/hide/protect rule for the partial-dir because (1) the auto-added rule may be ineffective at the end of your other rules, or (2) you may wish -to override rsync\(cq\&s exclude choice. For instance, if you want to make -rsync clean\-up any left\-over partial\-dirs that may be lying around, you -should specify \fB\-\-delete\-after\fP and add a \(dq\&risk\(dq\& filter rule, e.g. -\fB\-f '\&R .rsync\-partial/'\&\fP. (Avoid using \fB\-\-delete\-before\fP or -\fB\-\-delete\-during\fP unless you don\(cq\&t need rsync to use any of the -left\-over partial\-dir data during the current run.) -.IP -IMPORTANT: the \fB\-\-partial\-dir\fP should not be writable by other users or it -is a security risk. E.g. AVOID \(dq\&/tmp\(dq\&. -.IP -You can also set the partial\-dir value the RSYNC_PARTIAL_DIR environment +to override rsync's exclude choice. For instance, if you want to make +rsync clean-up any left-over partial-dirs that may be lying around, you +should specify \fB\-\-delete-after\fP and add a "risk" filter rule, e.g. +\fB\-f\ 'R\ .rsync-partial/'\fP. (Avoid using \fB\-\-delete-before\fP or +\fB\-\-delete-during\fP unless you don't need rsync to use any of the left-over +partial-dir data during the current run.) +.IP +IMPORTANT: the \fB\-\-partial-dir\fP should not be writable by other users or it +is a security risk. E.g. AVOID "/tmp". +.IP +You can also set the partial-dir value the RSYNC_PARTIAL_DIR environment variable. Setting this in the environment does not force \fB\-\-partial\fP to be enabled, but rather it affects where partial files go when \fB\-\-partial\fP is -specified. For instance, instead of using \fB\-\-partial\-dir=.rsync\-tmp\fP -along with \fB\-\-progress\fP, you could set RSYNC_PARTIAL_DIR=.rsync\-tmp in your +specified. For instance, instead of using \fB\-\-partial-dir=.rsync-tmp\fP along +with \fB\-\-progress\fP, you could set RSYNC_PARTIAL_DIR=.rsync-tmp in your environment and then just use the \fB\-P\fP option to turn on the use of the -\&.rsync\-tmp dir for partial transfers. The only times that the \fB\-\-partial\fP -option does not look for this environment value are (1) when \fB\-\-inplace\fP was -specified (since \fB\-\-inplace\fP conflicts with \fB\-\-partial\-dir\fP), and (2) when -\fB\-\-delay\-updates\fP was specified (see below). -.IP -For the purposes of the daemon\-config\(cq\&s \(dq\&refuse options\(dq\& setting, -\fB\-\-partial\-dir\fP does \fInot\fP imply \fB\-\-partial\fP. This is so that a -refusal of the \fB\-\-partial\fP option can be used to disallow the overwriting -of destination files with a partial transfer, while still allowing the -safer idiom provided by \fB\-\-partial\-dir\fP. -.IP -.IP "\fB\-\-delay\-updates\fP" -This option puts the temporary file from each -updated file into a holding directory until the end of the -transfer, at which time all the files are renamed into place in rapid -succession. This attempts to make the updating of the files a little more -atomic. By default the files are placed into a directory named \(dq\&.~tmp~\(dq\& in -each file\(cq\&s destination directory, but if you\(cq\&ve specified the -\fB\-\-partial\-dir\fP option, that directory will be used instead. See the -comments in the \fB\-\-partial\-dir\fP section for a discussion of how this -\(dq\&.~tmp~\(dq\& dir will be excluded from the transfer, and what you can do if -you want rsync to cleanup old \(dq\&.~tmp~\(dq\& dirs that might be lying around. -Conflicts with \fB\-\-inplace\fP and \fB\-\-append\fP. -.IP +\&.rsync-tmp dir for partial transfers. The only times that the \fB\-\-partial\fP +option does not look for this environment value are (1) when \fB\-\-inplace\fP +was specified (since \fB\-\-inplace\fP conflicts with \fB\-\-partial-dir\fP), and (2) +when \fB\-\-delay-updates\fP was specified (see below). +.IP +When a modern rsync resumes the transfer of a file in the partial-dir, that +partial file is now updated in-place instead of creating yet another +tmp-file copy (so it maxes out at dest + tmp instead of dest + partial + +tmp). This requires both ends of the transfer to be at least version +3.2.0. +.IP +For the purposes of the daemon-config's "\fBrefuse\ options\fP" setting, +\fB\-\-partial-dir\fP does \fInot\fP imply \fB\-\-partial\fP. This is so that a refusal of +the \fB\-\-partial\fP option can be used to disallow the overwriting of +destination files with a partial transfer, while still allowing the safer +idiom provided by \fB\-\-partial-dir\fP. +.IP "\fB\-\-delay-updates\fP" +This option puts the temporary file from each updated file into a holding +directory until the end of the transfer, at which time all the files are +renamed into place in rapid succession. This attempts to make the updating +of the files a little more atomic. By default the files are placed into a +directory named \fB.~tmp~\fP in each file's destination directory, but if +you've specified the \fB\-\-partial-dir\fP option, that directory will be used +instead. See the comments in the \fB\-\-partial-dir\fP section for a discussion +of how this \fB.~tmp~\fP dir will be excluded from the transfer, and what you +can do if you want rsync to cleanup old \fB.~tmp~\fP dirs that might be lying +around. Conflicts with \fB\-\-inplace\fP and \fB\-\-append\fP. +.IP This option uses more memory on the receiving side (one bit per file -transferred) and also requires enough free disk space on the receiving -side to hold an additional copy of all the updated files. Note also that -you should not use an absolute path to \fB\-\-partial\-dir\fP unless (1) -there is no +transferred) and also requires enough free disk space on the receiving side +to hold an additional copy of all the updated files. Note also that you +should not use an absolute path to \fB\-\-partial-dir\fP unless (1) there is no chance of any of the files in the transfer having the same name (since all the updated files will be put into a single directory if the path is -absolute) -and (2) there are no mount points in the hierarchy (since the -delayed updates will fail if they can\(cq\&t be renamed into place). -.IP -See also the \(dq\&atomic\-rsync\(dq\& perl script in the \(dq\&support\(dq\& subdir for an -update algorithm that is even more atomic (it uses \fB\-\-link\-dest\fP and a +absolute) and (2) there are no mount points in the hierarchy (since the +delayed updates will fail if they can't be renamed into place). +.IP +See also the "atomic-rsync" perl script in the "support" subdir for an +update algorithm that is even more atomic (it uses \fB\-\-link-dest\fP and a parallel hierarchy of files). -.IP -.IP "\fB\-m, \-\-prune\-empty\-dirs\fP" -This option tells the receiving rsync to get -rid of empty directories from the file\-list, including nested directories -that have no non\-directory children. This is useful for avoiding the -creation of a bunch of useless directories when the sending rsync is -recursively scanning a hierarchy of files using include/exclude/filter -rules. -.IP -Note that the use of transfer rules, such as the \fB\-\-min\-size\fP option, does -not affect what goes into the file list, and thus does not leave directories -empty, even if none of the files in a directory match the transfer rule. -.IP -Because the file\-list is actually being pruned, this option also affects +.IP "\fB\-\-prune-empty-dirs\fP, \fB\-m\fP" +This option tells the receiving rsync to get rid of empty directories from +the file-list, including nested directories that have no non-directory +children. This is useful for avoiding the creation of a bunch of useless +directories when the sending rsync is recursively scanning a hierarchy of +files using include/exclude/filter rules. +.IP +Note that the use of transfer rules, such as the \fB\-\-min-size\fP option, does +not affect what goes into the file list, and thus does not leave +directories empty, even if none of the files in a directory match the +transfer rule. +.IP +Because the file-list is actually being pruned, this option also affects what directories get deleted when a delete is active. However, keep in mind that excluded files and directories can prevent existing items from being deleted due to an exclude both hiding source files and protecting -destination files. See the perishable filter\-rule option for how to avoid +destination files. See the perishable filter-rule option for how to avoid this. -.IP -You can prevent the pruning of certain empty directories from the file\-list -by using a global \(dq\&protect\(dq\& filter. For instance, this option would ensure -that the directory \(dq\&emptydir\(dq\& was kept in the file\-list: -.IP -.RS -\-\-filter \(cq\&protect emptydir/\(cq\& +.IP +You can prevent the pruning of certain empty directories from the file-list +by using a global "protect" filter. For instance, this option would ensure +that the directory "emptydir" was kept in the file-list: +.RS 4 +.IP +.nf +--filter 'protect emptydir/' +.fi .RE - -.IP -Here\(cq\&s an example that copies all .pdf files in a hierarchy, only creating +.IP +Here's an example that copies all .pdf files in a hierarchy, only creating the necessary destination directories to hold the .pdf files, and ensures that any superfluous files and directories in the destination are removed -(note the hide filter of non\-directories being used instead of an exclude): -.IP -.RS -rsync \-avm \-\-del \-\-include=\(cq\&*.pdf\(cq\& \-f \(cq\&hide,! */\(cq\& src/ dest +(note the hide filter of non-directories being used instead of an exclude): +.RS 4 +.IP +.nf +rsync -avm --del --include='*.pdf' -f 'hide,! */' src/ dest +.fi .RE - -.IP -If you didn\(cq\&t want to remove superfluous destination files, the more -time\-honored options of \(dq\&\fB\-\-include='\&*/'\& \-\-exclude='\&*'\&\fP\(dq\& would work fine -in place of the hide\-filter (if that is more natural to you). -.IP +.IP +If you didn't want to remove superfluous destination files, the more +time-honored options of \fB\-\-include='*/'\ \-\-exclude='*'\fP would work +fine in place of the hide-filter (if that is more natural to you). .IP "\fB\-\-progress\fP" -This option tells rsync to print information -showing the progress of the transfer. This gives a bored user -something to watch. -With a modern rsync this is the same as specifying -\fB\-\-info=flist2,name,progress\fP, but any user\-supplied settings for those -info flags takes precedence (e.g. \(dq\&\-\-info=flist0 \-\-progress\(dq\&). -.IP +This option tells rsync to print information showing the progress of the +transfer. This gives a bored user something to watch. With a modern rsync +this is the same as specifying \fB\-\-info=flist2,name,progress\fP, but any +user-supplied settings for those info flags takes precedence (e.g. +"\fB\-\-info=flist0\ \-\-progress\fP"). +.IP While rsync is transferring a regular file, it updates a progress line that looks like this: -.IP -.nf - 782448 63% 110.64kB/s 0:00:04 -.fi - -.IP +.RS 4 +.IP +.nf +782448 63% 110.64kB/s 0:00:04 +.fi +.RE +.IP In this example, the receiver has reconstructed 782448 bytes or 63% of the -sender\(cq\&s file, which is being reconstructed at a rate of 110.64 kilobytes +sender's file, which is being reconstructed at a rate of 110.64 kilobytes per second, and the transfer will finish in 4 seconds if the current rate is maintained until the end. -.IP -These statistics can be misleading if rsync\(cq\&s delta\-transfer algorithm is -in use. For example, if the sender\(cq\&s file consists of the basis file +.IP +These statistics can be misleading if rsync's delta-transfer algorithm is +in use. For example, if the sender's file consists of the basis file followed by additional data, the reported rate will probably drop dramatically when the receiver gets to the literal data, and the transfer will probably take much longer to finish than the receiver estimated as it was finishing the matched part of the file. -.IP +.IP When the file transfer finishes, rsync replaces the progress line with a summary line that looks like this: -.IP -.nf - 1,238,099 100% 146.38kB/s 0:00:08 (xfr#5, to\-chk=169/396) -.fi - -.IP -In this example, the file was 1,238,099 bytes long in total, the average rate -of transfer for the whole file was 146.38 kilobytes per second over the 8 -seconds that it took to complete, it was the 5th transfer of a regular file -during the current rsync session, and there are 169 more files for the -receiver to check (to see if they are up\-to\-date or not) remaining out of -the 396 total files in the file\-list. -.IP -In an incremental recursion scan, rsync won\(cq\&t know the total number of files -in the file\-list until it reaches the ends of the scan, but since it starts to -transfer files during the scan, it will display a line with the text \(dq\&ir\-chk\(dq\& -(for incremental recursion check) instead of \(dq\&to\-chk\(dq\& until the point that it -knows the full size of the list, at which point it will switch to using -\(dq\&to\-chk\(dq\&. Thus, seeing \(dq\&ir\-chk\(dq\& lets you know that the total count of files -in the file list is still going to increase (and each time it does, the count -of files left to check will increase by the number of the files added to the -list). -.IP +.RS 4 +.IP +.nf +1,238,099 100% 146.38kB/s 0:00:08 (xfr#5, to-chk=169/396) +.fi +.RE +.IP +In this example, the file was 1,238,099 bytes long in total, the average +rate of transfer for the whole file was 146.38 kilobytes per second over +the 8 seconds that it took to complete, it was the 5th transfer of a +regular file during the current rsync session, and there are 169 more files +for the receiver to check (to see if they are up-to-date or not) remaining +out of the 396 total files in the file-list. +.IP +In an incremental recursion scan, rsync won't know the total number of +files in the file-list until it reaches the ends of the scan, but since it +starts to transfer files during the scan, it will display a line with the +text "ir-chk" (for incremental recursion check) instead of "to-chk" until +the point that it knows the full size of the list, at which point it will +switch to using "to-chk". Thus, seeing "ir-chk" lets you know that the +total count of files in the file list is still going to increase (and each +time it does, the count of files left to check will increase by the number +of the files added to the list). .IP "\fB\-P\fP" -The \fB\-P\fP option is equivalent to \fB\-\-partial\fP \fB\-\-progress\fP. Its -purpose is to make it much easier to specify these two options for a long -transfer that may be interrupted. -.IP -There is also a \fB\-\-info=progress2\fP option that outputs statistics based -on the whole transfer, rather than individual files. Use this flag without +The \fB\-P\fP option is equivalent to \fB\-\-partial\ \-\-progress\fP. Its purpose is +to make it much easier to specify these two options for a long transfer +that may be interrupted. +.IP +There is also a \fB\-\-info=progress2\fP option that outputs statistics based on +the whole transfer, rather than individual files. Use this flag without outputting a filename (e.g. avoid \fB\-v\fP or specify \fB\-\-info=name0\fP) if you want to see how the transfer is doing without scrolling the screen with a -lot of names. (You don\(cq\&t need to specify the \fB\-\-progress\fP option in -order to use \fB\-\-info=progress2\fP.) -.IP -.IP "\fB\-\-password\-file=FILE\fP" -This option allows you to provide a password for -accessing an rsync daemon via a file or via standard input if \fBFILE\fP is -\fB\-\fP. The file should contain just the password on the first line (all other -lines are ignored). Rsync will exit with an error if \fBFILE\fP is world -readable or if a root\-run rsync command finds a non\-root\-owned file. -.IP +lot of names. (You don't need to specify the \fB\-\-progress\fP option in order +to use \fB\-\-info=progress2\fP.) +.IP +Finally, you can get an instant progress report by sending rsync a signal +of either SIGINFO or SIGVTALRM. On BSD systems, a SIGINFO is generated by +typing a Ctrl+T (Linux doesn't currently support a SIGINFO signal). When +the client-side process receives one of those signals, it sets a flag to +output a single progress report which is output when the current file +transfer finishes (so it may take a little time if a big file is being +handled when the signal arrives). A filename is output (if needed) +followed by the \fB\-\-info=progress2\fP format of progress info. If you don't +know which of the 3 rsync processes is the client process, it's OK to +signal all of them (since the non-client processes ignore the signal). +.IP +CAUTION: sending SIGVTALRM to an older rsync (pre-3.2.0) will kill it. +.IP "\fB\-\-password-file=FILE\fP" +This option allows you to provide a password for accessing an rsync daemon +via a file or via standard input if \fBFILE\fP is \fB\-\fP. The file should +contain just the password on the first line (all other lines are ignored). +Rsync will exit with an error if \fBFILE\fP is world readable or if a +root-run rsync command finds a non-root-owned file. +.IP This option does not supply a password to a remote shell transport such as -ssh; to learn how to do that, consult the remote shell\(cq\&s documentation. +ssh; to learn how to do that, consult the remote shell's documentation. When accessing an rsync daemon using a remote shell as the transport, this option only comes into effect after the remote shell finishes its -authentication (i.e. if you have also specified a password in the daemon\(cq\&s +authentication (i.e. if you have also specified a password in the daemon's config file). -.IP -.IP "\fB\-\-list\-only\fP" -This option will cause the source files to be listed -instead of transferred. This option is inferred if there is a single source -arg and no destination specified, so its main uses are: (1) to turn a copy -command that includes a -destination arg into a file\-listing command, or (2) to be able to specify -more than one source arg (note: be sure to include the destination). -Caution: keep in mind that a source arg with a wild\-card is expanded by the -shell into multiple args, so it is never safe to try to list such an arg -without using this option. For example: -.IP -.nf - rsync \-av \-\-list\-only foo* dest/ -.fi - -.IP -Starting with rsync 3.1.0, the sizes output by \fB\-\-list\-only\fP are affected -by the \fB\-\-human\-readable\fP option. By default they will contain digit +.IP "\fB\-\-list-only\fP" +This option will cause the source files to be listed instead of +transferred. This option is inferred if there is a single source arg and +no destination specified, so its main uses are: (1) to turn a copy command +that includes a destination arg into a file-listing command, or (2) to be +able to specify more than one source arg (note: be sure to include the +destination). Caution: keep in mind that a source arg with a wild-card is +expanded by the shell into multiple args, so it is never safe to try to +list such an arg without using this option. For example: +.RS 4 +.IP +.nf +rsync -av --list-only foo* dest/ +.fi +.RE +.IP +Starting with rsync 3.1.0, the sizes output by \fB\-\-list-only\fP are affected +by the \fB\-\-human-readable\fP option. By default they will contain digit separators, but higher levels of readability will output the sizes with unit suffixes. Note also that the column width for the size output has -increased from 11 to 14 characters for all human\-readable levels. Use -\fB\-\-no\-h\fP if you want just digits in the sizes, and the old column width -of 11 characters. -.IP -Compatibility note: when requesting a remote listing of files from an rsync +increased from 11 to 14 characters for all human-readable levels. Use +\fB\-\-no-h\fP if you want just digits in the sizes, and the old column width of +11 characters. +.IP +Compatibility note: when requesting a remote listing of files from an rsync that is version 2.6.3 or older, you may encounter an error if you ask for a -non\-recursive listing. This is because a file listing implies the \fB\-\-dirs\fP -option w/o \fB\-\-recursive\fP, and older rsyncs don\(cq\&t have that option. To -avoid this problem, either specify the \fB\-\-no\-dirs\fP option (if you don\(cq\&t -need to expand a directory\(cq\&s content), or turn on recursion and exclude -the content of subdirectories: \fB\-r \-\-exclude='\&/*/*'\&\fP. -.IP +non-recursive listing. This is because a file listing implies the \fB\-\-dirs\fP +option w/o \fB\-\-recursive\fP, and older rsyncs don't have that option. To +avoid this problem, either specify the \fB\-\-no-dirs\fP option (if you don't +need to expand a directory's content), or turn on recursion and exclude the +content of subdirectories: \fB\-r\ \-\-exclude='/*/*'\fP. .IP "\fB\-\-bwlimit=RATE\fP" -This option allows you to specify the maximum transfer -rate for the data sent over the socket, specified in units per second. The -RATE value can be suffixed with a string to indicate a size multiplier, and may -be a fractional value (e.g. \(dq\&\fB\-\-bwlimit=1.5m\fP\(dq\&). If no suffix is specified, -the value will be assumed to be in units of 1024 bytes (as if \(dq\&K\(dq\& or \(dq\&KiB\(dq\& had -been appended). See the \fB\-\-max\-size\fP option for a description of all the -available suffixes. A value of zero specifies no limit. -.IP -For backward\-compatibility reasons, the rate limit will be rounded to the -nearest KiB unit, so no rate smaller than 1024 bytes per second is possible. -.IP -Rsync writes data over the socket in blocks, and this option both limits the -size of the blocks that rsync writes, and tries to keep the average transfer -rate at the requested limit. Some \(dq\&burstiness\(dq\& may be seen where rsync writes -out a block of data and then sleeps to bring the average rate into compliance. -.IP -Due to the internal buffering of data, the \fB\-\-progress\fP option may not be an -accurate reflection on how fast the data is being sent. This is because some -files can show up as being rapidly sent when the data is quickly buffered, -while other can show up as very slow when the flushing of the output buffer -occurs. This may be fixed in a future version. -.IP -.IP "\fB\-\-write\-batch=FILE\fP" -Record a file that can later be applied to -another identical destination with \fB\-\-read\-batch\fP. See the \(dq\&BATCH MODE\(dq\& -section for details, and also the \fB\-\-only\-write\-batch\fP option. -.IP -.IP "\fB\-\-only\-write\-batch=FILE\fP" -Works like \fB\-\-write\-batch\fP, except that -no updates are made on the destination system when creating the batch. -This lets you transport the changes to the destination system via some -other means and then apply the changes via \fB\-\-read\-batch\fP. -.IP +This option allows you to specify the maximum transfer rate for the data +sent over the socket, specified in units per second. The RATE value can be +suffixed with a string to indicate a size multiplier, and may be a +fractional value (e.g. "\fB\-\-bwlimit=1.5m\fP"). If no suffix is specified, the +value will be assumed to be in units of 1024 bytes (as if "K" or "KiB" had +been appended). See the \fB\-\-max-size\fP option for a description of all the +available suffixes. A value of zero specifies no limit. +.IP +For backward-compatibility reasons, the rate limit will be rounded to the +nearest KiB unit, so no rate smaller than 1024 bytes per second is +possible. +.IP +Rsync writes data over the socket in blocks, and this option both limits +the size of the blocks that rsync writes, and tries to keep the average +transfer rate at the requested limit. Some burstiness may be seen where +rsync writes out a block of data and then sleeps to bring the average rate +into compliance. +.IP +Due to the internal buffering of data, the \fB\-\-progress\fP option may not be +an accurate reflection on how fast the data is being sent. This is because +some files can show up as being rapidly sent when the data is quickly +buffered, while other can show up as very slow when the flushing of the +output buffer occurs. This may be fixed in a future version. +.IP "\fB\-\-write-batch=FILE\fP" +Record a file that can later be applied to another identical destination +with \fB\-\-read-batch\fP. See the "BATCH MODE" section for details, and also +the \fB\-\-only-write-batch\fP option. +.IP "\fB\-\-only-write-batch=FILE\fP" +Works like \fB\-\-write-batch\fP, except that no updates are made on the +destination system when creating the batch. This lets you transport the +changes to the destination system via some other means and then apply the +changes via \fB\-\-read-batch\fP. +.IP Note that you can feel free to write the batch directly to some portable media: if this media fills to capacity before the end of the transfer, you can just apply that partial transfer to the destination and repeat the -whole process to get the rest of the changes (as long as you don\(cq\&t mind a -partially updated destination system while the multi\-update cycle is +whole process to get the rest of the changes (as long as you don't mind a +partially updated destination system while the multi-update cycle is happening). -.IP +.IP Also note that you only save bandwidth when pushing changes to a remote system because this allows the batched data to be diverted from the sender into the batch file without having to flow over the wire to the receiver -(when pulling, the sender is remote, and thus can\(cq\&t write the batch). -.IP -.IP "\fB\-\-read\-batch=FILE\fP" -Apply all of the changes stored in FILE, a -file previously generated by \fB\-\-write\-batch\fP. -If \fIFILE\fP is \fB\-\fP, the batch data will be read from standard input. -See the \(dq\&BATCH MODE\(dq\& section for details. -.IP +(when pulling, the sender is remote, and thus can't write the batch). +.IP "\fB\-\-read-batch=FILE\fP" +Apply all of the changes stored in FILE, a file previously generated by +\fB\-\-write-batch\fP. If \fIFILE\fP is \fB\-\fP, the batch data will be read from +standard input. See the "BATCH MODE" section for details. .IP "\fB\-\-protocol=NUM\fP" -Force an older protocol version to be used. This -is useful for creating a batch file that is compatible with an older -version of rsync. For instance, if rsync 2.6.4 is being used with the -\fB\-\-write\-batch\fP option, but rsync 2.6.3 is what will be used to run the -\fB\-\-read\-batch\fP option, you should use \(dq\&\-\-protocol=28\(dq\& when creating the -batch file to force the older protocol version to be used in the batch -file (assuming you can\(cq\&t upgrade the rsync on the reading system). -.IP +Force an older protocol version to be used. This is useful for creating a +batch file that is compatible with an older version of rsync. For +instance, if rsync 2.6.4 is being used with the \fB\-\-write-batch\fP option, but +rsync 2.6.3 is what will be used to run the \fB\-\-read-batch\fP option, you +should use "\-\-protocol=28" when creating the batch file to force the older +protocol version to be used in the batch file (assuming you can't upgrade +the rsync on the reading system). .IP "\fB\-\-iconv=CONVERT_SPEC\fP" -Rsync can convert filenames between character -sets using this option. Using a CONVERT_SPEC of \(dq\&.\(dq\& tells rsync to look up -the default character\-set via the locale setting. Alternately, you can -fully specify what conversion to do by giving a local and a remote charset -separated by a comma in the order \fB\-\-iconv=LOCAL,REMOTE\fP, e.g. -\fB\-\-iconv=utf8,iso88591\fP. This order ensures that the option -will stay the same whether you\(cq\&re pushing or pulling files. -Finally, you can specify either \fB\-\-no\-iconv\fP or a CONVERT_SPEC of \(dq\&\-\(dq\& -to turn off any conversion. -The default setting of this option is site\-specific, and can also be -affected via the RSYNC_ICONV environment variable. -.IP +Rsync can convert filenames between character sets using this option. +Using a CONVERT_SPEC of "." tells rsync to look up the default +character-set via the locale setting. Alternately, you can fully specify +what conversion to do by giving a local and a remote charset separated by a +comma in the order \fB\-\-iconv=LOCAL,REMOTE\fP, e.g. \fB\-\-iconv=utf8,iso88591\fP. +This order ensures that the option will stay the same whether you're +pushing or pulling files. Finally, you can specify either \fB\-\-no-iconv\fP or +a CONVERT_SPEC of "\-" to turn off any conversion. The default setting of +this option is site-specific, and can also be affected via the RSYNC_ICONV +environment variable. +.IP For a list of what charset names your local iconv library supports, you can -run \(dq\&iconv \-\-list\(dq\&. -.IP -If you specify the \fB\-\-protect\-args\fP option (\fB\-s\fP), rsync will translate -the filenames you specify on the command\-line that are being sent to the -remote host. See also the \fB\-\-files\-from\fP option. -.IP +run "\fBiconv\ \-\-list\fP". +.IP +If you specify the \fB\-\-protect-args\fP option (\fB\-s\fP), rsync will translate the +filenames you specify on the command-line that are being sent to the remote +host. See also the \fB\-\-files-from\fP option. +.IP Note that rsync does not do any conversion of names in filter files -(including include/exclude files). It is up to you to ensure that you\(cq\&re +(including include/exclude files). It is up to you to ensure that you're specifying matching rules that can match on both sides of the transfer. For instance, you can specify extra include/exclude rules if there are filename differences on the two sides that need to be accounted for. -.IP +.IP When you pass an \fB\-\-iconv\fP option to an rsync daemon that allows it, the -daemon uses the charset specified in its \(dq\&charset\(dq\& configuration parameter -regardless of the remote charset you actually pass. Thus, you may feel free to -specify just the local charset for a daemon transfer (e.g. \fB\-\-iconv=utf8\fP). -.IP -.IP "\fB\-4, \-\-ipv4\fP or \fB\-6, \-\-ipv6\fP" -Tells rsync to prefer IPv4/IPv6 -when creating sockets. This only affects sockets that rsync has direct -control over, such as the outgoing socket when directly contacting an -rsync daemon. See also these options in the \fB\-\-daemon\fP mode section. -.IP -If rsync was complied without support for IPv6, the \fB\-\-ipv6\fP option -will have no effect. The \fB\-\-version\fP output will tell you if this -is the case. -.IP -.IP "\fB\-\-checksum\-seed=NUM\fP" -Set the checksum seed to the integer NUM. This 4 -byte checksum seed is included in each block and MD4 file checksum calculation -(the more modern MD5 file checksums don\(cq\&t use a seed). By default the checksum -seed is generated by the server and defaults to the current -\f(CWtime()\fP -\&. This +daemon uses the charset specified in its "charset" configuration parameter +regardless of the remote charset you actually pass. Thus, you may feel +free to specify just the local charset for a daemon transfer (e.g. +\fB\-\-iconv=utf8\fP). +.IP "\fB\-\-ipv4\fP, \fB\-4\fP or \fB\-\-ipv6\fP, \fB\-6\fP" +Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh. This +affects sockets that rsync has direct control over, such as the outgoing +socket when directly contacting an rsync daemon, as well as the forwarding +of the \fB\-4\fP or \fB\-6\fP option to ssh when rsync can deduce that ssh is being +used as the remote shell. For other remote shells you'll need to specify +the "\fB\-\-rsh\ SHELL\ \-4\fP" option directly (or whatever ipv4/ipv6 hint options +it uses). +.IP +These options also exist in the \fB\-\-daemon\fP mode section. +.IP +If rsync was complied without support for IPv6, the \fB\-\-ipv6\fP option will +have no effect. The \fBrsync\ \-V\fP output will contain "\fBno\ IPv6\fP" if is the +case. +.IP "\fB\-\-checksum-seed=NUM\fP" +Set the checksum seed to the integer NUM. This 4 byte checksum seed is +included in each block and MD4 file checksum calculation (the more modern +MD5 file checksums don't use a seed). By default the checksum seed is +generated by the server and defaults to the current \fBtime\fP(). This option is used to set a specific checksum seed, which is useful for applications that want repeatable block checksums, or in the case where the -user wants a more random checksum seed. Setting NUM to 0 causes rsync to use -the default of -\f(CWtime()\fP -for checksum seed. -.IP +user wants a more random checksum seed. Setting NUM to 0 causes rsync to +use the default of \fBtime\fP() for checksum seed. +.P .SH "DAEMON OPTIONS" - -.PP +.P The options allowed when starting an rsync daemon are as follows: -.PP +.P .IP "\fB\-\-daemon\fP" -This tells rsync that it is to run as a daemon. The -daemon you start running may be accessed using an rsync client using -the \fBhost::module\fP or \fBrsync://host/module/\fP syntax. -.IP -If standard input is a socket then rsync will assume that it is being -run via inetd, otherwise it will detach from the current terminal and -become a background daemon. The daemon will read the config file -(rsyncd.conf) on each connect made by a client and respond to -requests accordingly. See the \fBrsyncd.conf\fP(5) man page for more -details. -.IP +This tells rsync that it is to run as a daemon. The daemon you start +running may be accessed using an rsync client using the \fBhost::module\fP or +\fBrsync://host/module/\fP syntax. +.IP +If standard input is a socket then rsync will assume that it is being run +via inetd, otherwise it will detach from the current terminal and become a +background daemon. The daemon will read the config file (rsyncd.conf) on +each connect made by a client and respond to requests accordingly. See the +\fBrsyncd.conf\fP(5) man page for more details. .IP "\fB\-\-address\fP" -By default rsync will bind to the wildcard address when -run as a daemon with the \fB\-\-daemon\fP option. The \fB\-\-address\fP option -allows you to specify a specific IP address (or hostname) to bind to. This -makes virtual hosting possible in conjunction with the \fB\-\-config\fP option. -See also the \(dq\&address\(dq\& global option in the rsyncd.conf manpage. -.IP +By default rsync will bind to the wildcard address when run as a daemon +with the \fB\-\-daemon\fP option. The \fB\-\-address\fP option allows you to specify a +specific IP address (or hostname) to bind to. This makes virtual hosting +possible in conjunction with the \fB\-\-config\fP option. See also the "address" +global option in the rsyncd.conf manpage. .IP "\fB\-\-bwlimit=RATE\fP" -This option allows you to specify the maximum transfer -rate for the data the daemon sends over the socket. The client can still -specify a smaller \fB\-\-bwlimit\fP value, but no larger value will be allowed. -See the client version of this option (above) for some extra details. -.IP +This option allows you to specify the maximum transfer rate for the data +the daemon sends over the socket. The client can still specify a smaller +\fB\-\-bwlimit\fP value, but no larger value will be allowed. See the client +version of this option (above) for some extra details. .IP "\fB\-\-config=FILE\fP" -This specifies an alternate config file than -the default. This is only relevant when \fB\-\-daemon\fP is specified. -The default is /etc/rsyncd.conf unless the daemon is running over -a remote shell program and the remote user is not the super\-user; in that case -the default is rsyncd.conf in the current directory (typically $HOME). -.IP -.IP "\fB\-M, \-\-dparam=OVERRIDE\fP" -This option can be used to set a daemon\-config -parameter when starting up rsync in daemon mode. It is equivalent to adding -the parameter at the end of the global settings prior to the first module\(cq\&s -definition. The parameter names can be specified without spaces, if you so -desire. For instance: -.IP -.nf - rsync \-\-daemon \-M pidfile=/path/rsync.pid -.fi - -.IP -.IP "\fB\-\-no\-detach\fP" -When running as a daemon, this option instructs -rsync to not detach itself and become a background process. This -option is required when running as a service on Cygwin, and may also -be useful when rsync is supervised by a program such as -\fBdaemontools\fP or AIX\(cq\&s \fBSystem Resource Controller\fP. -\fB\-\-no\-detach\fP is also recommended when rsync is run under a -debugger. This option has no effect if rsync is run from inetd or -sshd. -.IP +This specifies an alternate config file than the default. This is only +relevant when \fB\-\-daemon\fP is specified. The default is /etc/rsyncd.conf +unless the daemon is running over a remote shell program and the remote +user is not the super-user; in that case the default is rsyncd.conf in the +current directory (typically $HOME). +.IP "\fB\-\-dparam=OVERRIDE\fP, \fB\-M\fP" +This option can be used to set a daemon-config parameter when starting up +rsync in daemon mode. It is equivalent to adding the parameter at the end +of the global settings prior to the first module's definition. The +parameter names can be specified without spaces, if you so desire. For +instance: +.RS 4 +.IP +.nf +rsync --daemon -M pidfile=/path/rsync.pid +.fi +.RE +.IP "\fB\-\-no-detach\fP" +When running as a daemon, this option instructs rsync to not detach itself +and become a background process. This option is required when running as a +service on Cygwin, and may also be useful when rsync is supervised by a +program such as \fBdaemontools\fP or AIX's \fBSystem\ Resource\ Controller\fP. +\fB\-\-no-detach\fP is also recommended when rsync is run under a debugger. This +option has no effect if rsync is run from inetd or sshd. .IP "\fB\-\-port=PORT\fP" -This specifies an alternate TCP port number for the -daemon to listen on rather than the default of 873. See also the \(dq\&port\(dq\& -global option in the rsyncd.conf manpage. -.IP -.IP "\fB\-\-log\-file=FILE\fP" -This option tells the rsync daemon to use the -given log\-file name instead of using the \(dq\&log file\(dq\& setting in the config -file. -.IP -.IP "\fB\-\-log\-file\-format=FORMAT\fP" -This option tells the rsync daemon to use the -given FORMAT string instead of using the \(dq\&log format\(dq\& setting in the config -file. It also enables \(dq\&transfer logging\(dq\& unless the string is empty, in which -case transfer logging is turned off. -.IP +This specifies an alternate TCP port number for the daemon to listen on +rather than the default of 873. See also the "port" global option in the +rsyncd.conf manpage. +.IP "\fB\-\-log-file=FILE\fP" +This option tells the rsync daemon to use the given log-file name instead +of using the "\fBlog\ file\fP" setting in the config file. +.IP "\fB\-\-log-file-format=FORMAT\fP" +This option tells the rsync daemon to use the given FORMAT string instead +of using the "\fBlog\ format\fP" setting in the config file. It also enables +"\fBtransfer\ logging\fP" unless the string is empty, in which case transfer +logging is turned off. .IP "\fB\-\-sockopts\fP" -This overrides the \fBsocket options\fP setting in the -rsyncd.conf file and has the same syntax. -.IP -.IP "\fB\-v, \-\-verbose\fP" -This option increases the amount of information the -daemon logs during its startup phase. After the client connects, the -daemon\(cq\&s verbosity level will be controlled by the options that the client -used and the \(dq\&max verbosity\(dq\& setting in the module\(cq\&s config section. -.IP -.IP "\fB\-4, \-\-ipv4\fP or \fB\-6, \-\-ipv6\fP" -Tells rsync to prefer IPv4/IPv6 -when creating the incoming sockets that the rsync daemon will use to -listen for connections. One of these options may be required in older -versions of Linux to work around an IPv6 bug in the kernel (if you see -an \(dq\&address already in use\(dq\& error when nothing else is using the port, -try specifying \fB\-\-ipv6\fP or \fB\-\-ipv4\fP when starting the daemon). -.IP -If rsync was complied without support for IPv6, the \fB\-\-ipv6\fP option -will have no effect. The \fB\-\-version\fP output will tell you if this -is the case. -.IP -.IP "\fB\-h, \-\-help\fP" -When specified after \fB\-\-daemon\fP, print a short help -page describing the options available for starting an rsync daemon. -.IP +This overrides the \fBsocket\ options\fP setting in the rsyncd.conf file and has +the same syntax. +.IP "\fB\-\-verbose\fP, \fB\-v\fP" +This option increases the amount of information the daemon logs during its +startup phase. After the client connects, the daemon's verbosity level +will be controlled by the options that the client used and the +"\fBmax\ verbosity\fP" setting in the module's config section. +.IP "\fB\-\-ipv4\fP, \fB\-4\fP or \fB\-\-ipv6\fP, \fB\-6\fP" +Tells rsync to prefer IPv4/IPv6 when creating the incoming sockets that the +rsync daemon will use to listen for connections. One of these options may +be required in older versions of Linux to work around an IPv6 bug in the +kernel (if you see an "address already in use" error when nothing else is +using the port, try specifying \fB\-\-ipv6\fP or \fB\-\-ipv4\fP when starting the +daemon). +.IP +These options also exist in the regular rsync options section. +.IP +If rsync was complied without support for IPv6, the \fB\-\-ipv6\fP option will +have no effect. The \fBrsync\ \-V\fP output will contain "\fBno\ IPv6\fP" if is the +case. +.IP "\fB\-\-help\fP, \fB\-h\fP" +When specified after \fB\-\-daemon\fP, print a short help page describing the +options available for starting an rsync daemon. +.P .SH "FILTER RULES" - -.PP +.P The filter rules allow for flexible selection of which files to transfer -(include) and which files to skip (exclude). The rules either directly -specify include/exclude patterns or they specify a way to acquire more -include/exclude patterns (e.g. to read them from a file). -.PP -As the list of files/directories to transfer is built, rsync checks each -name to be transferred against the list of include/exclude patterns in -turn, and the first matching pattern is acted on: if it is an exclude -pattern, then that file is skipped; if it is an include pattern then that -filename is not skipped; if no matching pattern is found, then the -filename is not skipped. -.PP -Rsync builds an ordered list of filter rules as specified on the -command\-line. Filter rules have the following syntax: -.PP -.RS -\f(CWRULE [PATTERN_OR_FILENAME]\fP -.br -\f(CWRULE,MODIFIERS [PATTERN_OR_FILENAME]\fP -.br +(include) and which files to skip (exclude). The rules either directly specify +include/exclude patterns or they specify a way to acquire more include/exclude +patterns (e.g. to read them from a file). +.P +As the list of files/directories to transfer is built, rsync checks each name +to be transferred against the list of include/exclude patterns in turn, and the +first matching pattern is acted on: if it is an exclude pattern, then that file +is skipped; if it is an include pattern then that filename is not skipped; if +no matching pattern is found, then the filename is not skipped. +.P +Rsync builds an ordered list of filter rules as specified on the command-line. +Filter rules have the following syntax: +.RS 4 +.P +.nf +RULE [PATTERN_OR_FILENAME] +RULE,MODIFIERS [PATTERN_OR_FILENAME] +.fi .RE - -.PP +.P You have your choice of using either short or long RULE names, as described -below. If you use a short\-named rule, the \(cq\&,\(cq\& separating the RULE from the +below. If you use a short-named rule, the ',' separating the RULE from the MODIFIERS is optional. The PATTERN or FILENAME that follows (when present) -must come after either a single space or an underscore (_). -Here are the available rule prefixes: -.PP -.RS -\fBexclude, \-\fP specifies an exclude pattern. -.br -\fBinclude, +\fP specifies an include pattern. -.br -\fBmerge, .\fP specifies a merge\-file to read for more rules. -.br -\fBdir\-merge, :\fP specifies a per\-directory merge\-file. -.br -\fBhide, H\fP specifies a pattern for hiding files from the transfer. -.br -\fBshow, S\fP files that match the pattern are not hidden. -.br -\fBprotect, P\fP specifies a pattern for protecting files from deletion. -.br -\fBrisk, R\fP files that match the pattern are not protected. -.br -\fBclear, !\fP clears the current include/exclude list (takes no arg) -.br -.RE - -.PP -When rules are being read from a file, empty lines are ignored, as are -comment lines that start with a \(dq\&#\(dq\&. -.PP -Note that the \fB\-\-include\fP/\fB\-\-exclude\fP command\-line options do not allow the -full range of rule parsing as described above \-\- they only allow the -specification of include/exclude patterns plus a \(dq\&!\(dq\& token to clear the -list (and the normal comment parsing when rules are read from a file). -If a pattern -does not begin with \(dq\&\- \(dq\& (dash, space) or \(dq\&+ \(dq\& (plus, space), then the -rule will be interpreted as if \(dq\&+ \(dq\& (for an include option) or \(dq\&\- \(dq\& (for -an exclude option) were prefixed to the string. A \fB\-\-filter\fP option, on +must come after either a single space or an underscore (_). Here are the +available rule prefixes: +.P +.IP "\fBexclude,\ '\-'\fP" +specifies an exclude pattern. +.IP "\fBinclude,\ '+'\fP" +specifies an include pattern. +.IP "\fBmerge,\ '.'\fP" +specifies a merge-file to read for more rules. +.IP "\fBdir-merge,\ ':'\fP" +specifies a per-directory merge-file. +.IP "\fBhide,\ 'H'\fP" +specifies a pattern for hiding files from the transfer. +.IP "\fBshow,\ 'S'\fP" +files that match the pattern are not hidden. +.IP "\fBprotect,\ 'P'\fP" +specifies a pattern for protecting files from deletion. +.IP "\fBrisk,\ 'R'\fP" +files that match the pattern are not protected. +.IP "\fBclear,\ '!'\fP" +clears the current include/exclude list (takes no arg) +.P +When rules are being read from a file, empty lines are ignored, as are comment +lines that start with a "#". +.P +Note that the \fB\-\-include\fP & \fB\-\-exclude\fP command-line options do not allow the +full range of rule parsing as described above\ \-\- they only allow the +specification of include / exclude patterns plus a "\fB!\fP" token to clear the +list (and the normal comment parsing when rules are read from a file). If a +pattern does not begin with "\fB\-\ \fP" (dash, space) or "\fB+\ \fP" (plus, space), then +the rule will be interpreted as if "\fB+\ \fP" (for an include option) or "\fB\-\ \fP" +(for an exclude option) were prefixed to the string. A \fB\-\-filter\fP option, on the other hand, must always contain either a short or long rule name at the start of the rule. -.PP +.P Note also that the \fB\-\-filter\fP, \fB\-\-include\fP, and \fB\-\-exclude\fP options take one -rule/pattern each. To add multiple ones, you can repeat the options on -the command\-line, use the merge\-file syntax of the \fB\-\-filter\fP option, or -the \fB\-\-include\-from\fP/\fB\-\-exclude\-from\fP options. -.PP +rule/pattern each. To add multiple ones, you can repeat the options on the +command-line, use the merge-file syntax of the \fB\-\-filter\fP option, or the +\fB\-\-include-from\fP / \fB\-\-exclude-from\fP options. +.P .SH "INCLUDE/EXCLUDE PATTERN RULES" - -.PP -You can include and exclude files by specifying patterns using the \(dq\&+\(dq\&, -\(dq\&\-\(dq\&, etc. filter rules (as introduced in the FILTER RULES section above). -The include/exclude rules each specify a pattern that is matched against -the names of the files that are going to be transferred. These patterns -can take several forms: -.PP -.IP o -if the pattern starts with a / then it is anchored to a -particular spot in the hierarchy of files, otherwise it is matched -against the end of the pathname. This is similar to a leading ^ in -regular expressions. -Thus \(dq\&/foo\(dq\& would match a name of \(dq\&foo\(dq\& at either the \(dq\&root of the -transfer\(dq\& (for a global rule) or in the merge\-file\(cq\&s directory (for a -per\-directory rule). -An unqualified \(dq\&foo\(dq\& would match a name of \(dq\&foo\(dq\& anywhere in the -tree because the algorithm is applied recursively from the -top down; it behaves as if each path component gets a turn at being the -end of the filename. Even the unanchored \(dq\&sub/foo\(dq\& would match at -any point in the hierarchy where a \(dq\&foo\(dq\& was found within a directory -named \(dq\&sub\(dq\&. See the section on ANCHORING INCLUDE/EXCLUDE PATTERNS for -a full discussion of how to specify a pattern that matches at the root -of the transfer. -.IP o -if the pattern ends with a / then it will only match a -directory, not a regular file, symlink, or device. -.IP o -rsync chooses between doing a simple string match and wildcard -matching by checking if the pattern contains one of these three wildcard -characters: \(cq\&*\(cq\&, \(cq\&?\(cq\&, and \(cq\&[\(cq\& . -.IP o -a \(cq\&*\(cq\& matches any path component, but it stops at slashes. -.IP o -use \(cq\&**\(cq\& to match anything, including slashes. -.IP o -a \(cq\&?\(cq\& matches any character except a slash (/). -.IP o -a \(cq\&[\(cq\& introduces a character class, such as [a\-z] or [[:alpha:]]. -.IP o +.P +You can include and exclude files by specifying patterns using the "+", "\-", +etc. filter rules (as introduced in the FILTER RULES section above). The +include/exclude rules each specify a pattern that is matched against the names +of the files that are going to be transferred. These patterns can take several +forms: +.P +.IP o +if the pattern starts with a \fB/\fP then it is anchored to a particular spot in +the hierarchy of files, otherwise it is matched against the end of the +pathname. This is similar to a leading \fB^\fP in regular expressions. Thus +\fB/foo\fP would match a name of "foo" at either the "root of the transfer" (for +a global rule) or in the merge-file's directory (for a per-directory rule). +An unqualified \fBfoo\fP would match a name of "foo" anywhere in the tree because +the algorithm is applied recursively from the top down; it behaves as if each +path component gets a turn at being the end of the filename. Even the +unanchored "sub/foo" would match at any point in the hierarchy where a "foo" +was found within a directory named "sub". See the section on ANCHORING +INCLUDE/EXCLUDE PATTERNS for a full discussion of how to specify a pattern +that matches at the root of the transfer. +.IP o +if the pattern ends with a \fB/\fP then it will only match a directory, not a +regular file, symlink, or device. +.IP o +rsync chooses between doing a simple string match and wildcard matching by +checking if the pattern contains one of these three wildcard characters: +\&'\fB*\fP', '\fB?\fP', and '\fB[\fP' . +.IP o +a '\fB*\fP' matches any path component, but it stops at slashes. +.IP o +use '\fB**\fP' to match anything, including slashes. +.IP o +a '\fB?\fP' matches any character except a slash (\fB/\fP). +.IP o +a '\fB[\fP' introduces a character class, such as \fB[a-z]\fP or \fB[[:alpha:]]\fP. +.IP o in a wildcard pattern, a backslash can be used to escape a wildcard -character, but it is matched literally when no wildcards are present. -This means that there is an extra level of backslash removal when a -pattern contains wildcard characters compared to a pattern that has none. -e.g. if you add a wildcard to \(dq\&foo\ebar\(dq\& (which matches the backslash) you -would need to use \(dq\&foo\e\ebar*\(dq\& to avoid the \(dq\&\eb\(dq\& becoming just \(dq\&b\(dq\&. -.IP o -if the pattern contains a / (not counting a trailing /) or a \(dq\&**\(dq\&, -then it is matched against the full pathname, including any leading -directories. If the pattern doesn\(cq\&t contain a / or a \(dq\&**\(dq\&, then it is -matched only against the final component of the filename. -(Remember that the algorithm is applied recursively so \(dq\&full filename\(dq\& -can actually be any portion of a path from the starting directory on -down.) -.IP o -a trailing \(dq\&dir_name/***\(dq\& will match both the directory (as if -\(dq\&dir_name/\(dq\& had been specified) and everything in the directory -(as if \(dq\&dir_name/**\(dq\& had been specified). This behavior was added in -version 2.6.7. - -.PP +character, but it is matched literally when no wildcards are present. This +means that there is an extra level of backslash removal when a pattern +contains wildcard characters compared to a pattern that has none. e.g. if +you add a wildcard to "\fBfoo\\bar\fP" (which matches the backslash) you would +need to use "\fBfoo\\\\bar*\fP" to avoid the "\fB\\b\fP" becoming just "b". +.IP o +if the pattern contains a \fB/\fP (not counting a trailing /) or a "\fB**\fP", then it +is matched against the full pathname, including any leading directories. If +the pattern doesn't contain a \fB/\fP or a "\fB**\fP", then it is matched only against +the final component of the filename. (Remember that the algorithm is applied +recursively so "full filename" can actually be any portion of a path from the +starting directory on down.) +.IP o +a trailing "\fBdir_name/***\fP" will match both the directory (as if "dir_name/" +had been specified) and everything in the directory (as if "\fBdir_name/**\fP" +had been specified). This behavior was added in version 2.6.7. +.P Note that, when using the \fB\-\-recursive\fP (\fB\-r\fP) option (which is implied by -\fB\-a\fP), every subdir component of every path is visited left to right, with -each directory having a chance for exclusion before its content. In this way +\fB\-a\fP), every subdir component of every path is visited left to right, with each +directory having a chance for exclusion before its content. In this way include/exclude patterns are applied recursively to the pathname of each node -in the filesystem\(cq\&s tree (those inside the transfer). The exclude patterns -short\-circuit the directory traversal stage as rsync finds the files to send. -.PP -For instance, to include \(dq\&/foo/bar/baz\(dq\&, the directories \(dq\&/foo\(dq\& and \(dq\&/foo/bar\(dq\& +in the filesystem's tree (those inside the transfer). The exclude patterns +short-circuit the directory traversal stage as rsync finds the files to send. +.P +For instance, to include "\fB/foo/bar/baz\fP", the directories "\fB/foo\fP" and "\fB/foo/bar\fP" must not be excluded. Excluding one of those parent directories prevents the -examination of its content, cutting off rsync\(cq\&s recursion into those paths and -rendering the include for \(dq\&/foo/bar/baz\(dq\& ineffectual (since rsync can\(cq\&t match -something it never sees in the cut\-off section of the directory hierarchy). -.PP -The concept path exclusion is particularly important when using a trailing \(cq\&*\(cq\& -rule. For instance, this won\(cq\&t work: -.PP -.RS -\f(CW+ /some/path/this\-file\-will\-not\-be\-found\fP -.br -\f(CW+ /file\-is\-included\fP -.br -\f(CW\- *\fP -.br +examination of its content, cutting off rsync's recursion into those paths and +rendering the include for "\fB/foo/bar/baz\fP" ineffectual (since rsync can't match +something it never sees in the cut-off section of the directory hierarchy). +.P +The concept path exclusion is particularly important when using a trailing '\fB*\fP' +rule. For instance, this won't work: +.RS 4 +.P +.nf ++ /some/path/this-file-will-not-be-found ++ /file-is-included +- * +.fi .RE - -.PP -This fails because the parent directory \(dq\&some\(dq\& is excluded by the \(cq\&*\(cq\& -rule, so rsync never visits any of the files in the \(dq\&some\(dq\& or \(dq\&some/path\(dq\& -directories. One solution is to ask for all directories in the hierarchy -to be included by using a single rule: \(dq\&+ */\(dq\& (put it somewhere before the -\(dq\&\- *\(dq\& rule), and perhaps use the \fB\-\-prune\-empty\-dirs\fP option. Another -solution is to add specific include rules for all -the parent dirs that need to be visited. For instance, this set of rules -works fine: -.PP -.RS -\f(CW+ /some/\fP -.br -\f(CW+ /some/path/\fP -.br -\f(CW+ /some/path/this\-file\-is\-found\fP -.br -\f(CW+ /file\-also\-included\fP -.br -\f(CW\- *\fP -.br +.P +This fails because the parent directory "some" is excluded by the '\fB*\fP' rule, so +rsync never visits any of the files in the "some" or "some/path" directories. +One solution is to ask for all directories in the hierarchy to be included by +using a single rule: "\fB+\ */\fP" (put it somewhere before the "\fB\-\ *\fP" rule), and +perhaps use the \fB\-\-prune-empty-dirs\fP option. Another solution is to add +specific include rules for all the parent dirs that need to be visited. For +instance, this set of rules works fine: +.RS 4 +.P +.nf ++ /some/ ++ /some/path/ ++ /some/path/this-file-is-found ++ /file-also-included +- * +.fi .RE - -.PP +.P Here are some examples of exclude/include matching: -.PP -.IP o -\(dq\&\- *.o\(dq\& would exclude all names matching *.o -.IP o -\(dq\&\- /foo\(dq\& would exclude a file (or directory) named foo in the -transfer\-root directory -.IP o -\(dq\&\- foo/\(dq\& would exclude any directory named foo -.IP o -\(dq\&\- /foo/*/bar\(dq\& would exclude any file named bar which is at two -levels below a directory named foo in the transfer\-root directory -.IP o -\(dq\&\- /foo/**/bar\(dq\& would exclude any file named bar two -or more levels below a directory named foo in the transfer\-root directory -.IP o -The combination of \(dq\&+ */\(dq\&, \(dq\&+ *.c\(dq\&, and \(dq\&\- *\(dq\& would include all +.P +.IP o +"\fB\-\ *.o\fP" would exclude all names matching \fB*.o\fP +.IP o +"\fB\-\ /foo\fP" would exclude a file (or directory) named foo in the transfer-root +directory +.IP o +"\fB\-\ foo/\fP" would exclude any directory named foo +.IP o +"\fB\-\ /foo/*/bar\fP" would exclude any file named bar which is at two levels +below a directory named foo in the transfer-root directory +.IP o +"\fB\-\ /foo/**/bar\fP" would exclude any file named bar two or more levels below a +directory named foo in the transfer-root directory +.IP o +The combination of "\fB+\ */\fP", "\fB+\ *.c\fP", and "\fB\-\ *\fP" would include all directories and C source files but nothing else (see also the -\fB\-\-prune\-empty\-dirs\fP option) -.IP o -The combination of \(dq\&+ foo/\(dq\&, \(dq\&+ foo/bar.c\(dq\&, and \(dq\&\- *\(dq\& would include -only the foo directory and foo/bar.c (the foo directory must be -explicitly included or it would be excluded by the \(dq\&*\(dq\&) - -.PP -The following modifiers are accepted after a \(dq\&+\(dq\& or \(dq\&\-\(dq\&: -.PP -.IP o -A \fB/\fP specifies that the include/exclude rule should be matched -against the absolute pathname of the current item. For example, -\(dq\&\-/ /etc/passwd\(dq\& would exclude the passwd file any time the transfer -was sending files from the \(dq\&/etc\(dq\& directory, and \(dq\&\-/ subdir/foo\(dq\& -would always exclude \(dq\&foo\(dq\& when it is in a dir named \(dq\&subdir\(dq\&, even -if \(dq\&foo\(dq\& is at the root of the current transfer. -.IP o -A \fB!\fP specifies that the include/exclude should take effect if -the pattern fails to match. For instance, \(dq\&\-! */\(dq\& would exclude all -non\-directories. -.IP o -A \fBC\fP is used to indicate that all the global CVS\-exclude rules -should be inserted as excludes in place of the \(dq\&\-C\(dq\&. No arg should -follow. -.IP o -An \fBs\fP is used to indicate that the rule applies to the sending -side. When a rule affects the sending side, it prevents files from -being transferred. The default is for a rule to affect both sides -unless \fB\-\-delete\-excluded\fP was specified, in which case default rules -become sender\-side only. See also the hide (H) and show (S) rules, -which are an alternate way to specify sending\-side includes/excludes. -.IP o -An \fBr\fP is used to indicate that the rule applies to the receiving -side. When a rule affects the receiving side, it prevents files from -being deleted. See the \fBs\fP modifier for more info. See also the -protect (P) and risk (R) rules, which are an alternate way to -specify receiver\-side includes/excludes. -.IP o -A \fBp\fP indicates that a rule is perishable, meaning that it is -ignored in directories that are being deleted. For instance, the \fB\-C\fP -option\(cq\&s default rules that exclude things like \(dq\&CVS\(dq\& and \(dq\&*.o\(dq\& are -marked as perishable, and will not prevent a directory that was removed -on the source from being deleted on the destination. -.IP o +\fB\-\-prune-empty-dirs\fP option) +.IP o +The combination of "\fB+\ foo/\fP", "\fB+\ foo/bar.c\fP", and "\fB\-\ *\fP" would include +only the foo directory and foo/bar.c (the foo directory must be explicitly +included or it would be excluded by the "\fB*\fP") +.P +The following modifiers are accepted after a "\fB+\fP" or "\fB\-\fP": +.P +.IP o +A \fB/\fP specifies that the include/exclude rule should be matched against the +absolute pathname of the current item. For example, "\fB\-/\ /etc/passwd\fP" would +exclude the passwd file any time the transfer was sending files from the +"/etc" directory, and "\-/ subdir/foo" would always exclude "foo" when it is +in a dir named "subdir", even if "foo" is at the root of the current +transfer. +.IP o +A \fB!\fP specifies that the include/exclude should take effect if the pattern +fails to match. For instance, "\fB\-!\ */\fP" would exclude all non-directories. +.IP o +A \fBC\fP is used to indicate that all the global CVS-exclude rules should be +inserted as excludes in place of the "\-C". No arg should follow. +.IP o +An \fBs\fP is used to indicate that the rule applies to the sending side. When a +rule affects the sending side, it prevents files from being transferred. The +default is for a rule to affect both sides unless \fB\-\-delete-excluded\fP was +specified, in which case default rules become sender-side only. See also the +hide (H) and show (S) rules, which are an alternate way to specify +sending-side includes/excludes. +.IP o +An \fBr\fP is used to indicate that the rule applies to the receiving side. When +a rule affects the receiving side, it prevents files from being deleted. See +the \fBs\fP modifier for more info. See also the protect (P) and risk (R) rules, +which are an alternate way to specify receiver-side includes/excludes. +.IP o +A \fBp\fP indicates that a rule is perishable, meaning that it is ignored in +directories that are being deleted. For instance, the \fB\-C\fP option's default +rules that exclude things like "CVS" and "\fB*.o\fP" are marked as perishable, +and will not prevent a directory that was removed on the source from being +deleted on the destination. +.IP o An \fBx\fP indicates that a rule affects xattr names in xattr copy/delete -operations (and is thus ignored when matching file/dir names). If no -xattr\-matching rules are specified, a default xattr filtering rule is -used (see the \fB\-\-xattrs\fP option). - -.PP -.SH "MERGE\-FILE FILTER RULES" - -.PP -You can merge whole files into your filter rules by specifying either a -merge (.) or a dir\-merge (:) filter rule (as introduced in the FILTER RULES -section above). -.PP -There are two kinds of merged files \-\- single\-instance (\(cq\&.\(cq\&) and -per\-directory (\(cq\&:\(cq\&). A single\-instance merge file is read one time, and -its rules are incorporated into the filter list in the place of the \(dq\&.\(dq\& -rule. For per\-directory merge files, rsync will scan every directory that -it traverses for the named file, merging its contents when the file exists -into the current list of inherited rules. These per\-directory rule files -must be created on the sending side because it is the sending side that is -being scanned for the available files to transfer. These rule files may -also need to be transferred to the receiving side if you want them to -affect what files don\(cq\&t get deleted (see PER\-DIRECTORY RULES AND DELETE -below). -.PP +operations (and is thus ignored when matching file/dir names). If no +xattr-matching rules are specified, a default xattr filtering rule is used +(see the \fB\-\-xattrs\fP option). +.P +.SH "MERGE-FILE FILTER RULES" +.P +You can merge whole files into your filter rules by specifying either a merge +(.) or a dir-merge (:) filter rule (as introduced in the FILTER RULES section +above). +.P +There are two kinds of merged files\ \-\- single-instance ('.') and per-directory +(':'). A single-instance merge file is read one time, and its rules are +incorporated into the filter list in the place of the "." rule. For +per-directory merge files, rsync will scan every directory that it traverses +for the named file, merging its contents when the file exists into the current +list of inherited rules. These per-directory rule files must be created on the +sending side because it is the sending side that is being scanned for the +available files to transfer. These rule files may also need to be transferred +to the receiving side if you want them to affect what files don't get deleted +(see PER-DIRECTORY RULES AND DELETE below). +.P Some examples: -.PP -.RS -\f(CWmerge /etc/rsync/default.rules\fP -.br -\f(CW. /etc/rsync/default.rules\fP -.br -\f(CWdir\-merge .per\-dir\-filter\fP -.br -\f(CWdir\-merge,n\- .non\-inherited\-per\-dir\-excludes\fP -.br -\f(CW:n\- .non\-inherited\-per\-dir\-excludes\fP -.br +.RS 4 +.P +.nf +merge /etc/rsync/default.rules +\&. /etc/rsync/default.rules +dir-merge .per-dir-filter +dir-merge,n- .non-inherited-per-dir-excludes +:n- .non-inherited-per-dir-excludes +.fi .RE - -.PP -The following modifiers are accepted after a merge or dir\-merge rule: -.PP -.IP o -A \fB\-\fP specifies that the file should consist of only exclude -patterns, with no other rule\-parsing except for in\-file comments. -.IP o -A \fB+\fP specifies that the file should consist of only include -patterns, with no other rule\-parsing except for in\-file comments. -.IP o -A \fBC\fP is a way to specify that the file should be read in a -CVS\-compatible manner. This turns on \(cq\&n\(cq\&, \(cq\&w\(cq\&, and \(cq\&\-\(cq\&, but also -allows the list\-clearing token (!) to be specified. If no filename is -provided, \(dq\&.cvsignore\(dq\& is assumed. -.IP o -A \fBe\fP will exclude the merge\-file name from the transfer; e.g. -\(dq\&dir\-merge,e .rules\(dq\& is like \(dq\&dir\-merge .rules\(dq\& and \(dq\&\- .rules\(dq\&. -.IP o +.P +The following modifiers are accepted after a merge or dir-merge rule: +.P +.IP o +A \fB\-\fP specifies that the file should consist of only exclude patterns, with +no other rule-parsing except for in-file comments. +.IP o +A \fB+\fP specifies that the file should consist of only include patterns, with +no other rule-parsing except for in-file comments. +.IP o +A \fBC\fP is a way to specify that the file should be read in a CVS-compatible +manner. This turns on 'n', 'w', and '\-', but also allows the list-clearing +token (!) to be specified. If no filename is provided, ".cvsignore" is +assumed. +.IP o +A \fBe\fP will exclude the merge-file name from the transfer; e.g. "dir-merge,e +\&.rules" is like "dir-merge .rules" and "\- .rules". +.IP o An \fBn\fP specifies that the rules are not inherited by subdirectories. -.IP o -A \fBw\fP specifies that the rules are word\-split on whitespace instead -of the normal line\-splitting. This also turns off comments. Note: the -space that separates the prefix from the rule is treated specially, so -\(dq\&\- foo + bar\(dq\& is parsed as two rules (assuming that prefix\-parsing wasn\(cq\&t -also disabled). -.IP o -You may also specify any of the modifiers for the \(dq\&+\(dq\& or \(dq\&\-\(dq\& rules -(above) in order to have the rules that are read in from the file -default to having that modifier set (except for the \fB!\fP modifier, which -would not be useful). For instance, \(dq\&merge,\-/ .excl\(dq\& would -treat the contents of .excl as absolute\-path excludes, -while \(dq\&dir\-merge,s .filt\(dq\& and \(dq\&:sC\(dq\& would each make all their -per\-directory rules apply only on the sending side. If the merge rule -specifies sides to affect (via the \fBs\fP or \fBr\fP modifier or both), -then the rules in the file must not specify sides (via a modifier or -a rule prefix such as \fBhide\fP). - -.PP -Per\-directory rules are inherited in all subdirectories of the directory -where the merge\-file was found unless the \(cq\&n\(cq\& modifier was used. Each -subdirectory\(cq\&s rules are prefixed to the inherited per\-directory rules -from its parents, which gives the newest rules a higher priority than the -inherited rules. The entire set of dir\-merge rules are grouped together in -the spot where the merge\-file was specified, so it is possible to override -dir\-merge rules via a rule that got specified earlier in the list of global -rules. When the list\-clearing rule (\(dq\&!\(dq\&) is read from a per\-directory -file, it only clears the inherited rules for the current merge file. -.PP -Another way to prevent a single rule from a dir\-merge file from being inherited is to -anchor it with a leading slash. Anchored rules in a per\-directory -merge\-file are relative to the merge\-file\(cq\&s directory, so a pattern \(dq\&/foo\(dq\& -would only match the file \(dq\&foo\(dq\& in the directory where the dir\-merge filter +.IP o +A \fBw\fP specifies that the rules are word-split on whitespace instead of the +normal line-splitting. This also turns off comments. Note: the space that +separates the prefix from the rule is treated specially, so "\- foo + bar" is +parsed as two rules (assuming that prefix-parsing wasn't also disabled). +.IP o +You may also specify any of the modifiers for the "+" or "\-" rules (above) in +order to have the rules that are read in from the file default to having that +modifier set (except for the \fB!\fP modifier, which would not be useful). For +instance, "merge,\-/ .excl" would treat the contents of .excl as absolute-path +excludes, while "dir-merge,s .filt" and ":sC" would each make all their +per-directory rules apply only on the sending side. If the merge rule +specifies sides to affect (via the \fBs\fP or \fBr\fP modifier or both), then the +rules in the file must not specify sides (via a modifier or a rule prefix +such as \fBhide\fP). +.P +Per-directory rules are inherited in all subdirectories of the directory where +the merge-file was found unless the 'n' modifier was used. Each subdirectory's +rules are prefixed to the inherited per-directory rules from its parents, which +gives the newest rules a higher priority than the inherited rules. The entire +set of dir-merge rules are grouped together in the spot where the merge-file +was specified, so it is possible to override dir-merge rules via a rule that +got specified earlier in the list of global rules. When the list-clearing rule +("!") is read from a per-directory file, it only clears the inherited rules for +the current merge file. +.P +Another way to prevent a single rule from a dir-merge file from being inherited +is to anchor it with a leading slash. Anchored rules in a per-directory +merge-file are relative to the merge-file's directory, so a pattern "/foo" +would only match the file "foo" in the directory where the dir-merge filter file was found. -.PP -Here\(cq\&s an example filter file which you\(cq\&d specify via \fB\-\-filter=\(dq\&. file\(dq\&:\fP -.PP -.RS -\f(CWmerge /home/user/.global\-filter\fP -.br -\f(CW\- *.gz\fP -.br -\f(CWdir\-merge .rules\fP -.br -\f(CW+ *.[ch]\fP -.br -\f(CW\- *.o\fP -.br +.P +Here's an example filter file which you'd specify via \fB\-\-filter=".\ file":\fP +.RS 4 +.P +.nf +merge /home/user/.global-filter +- *.gz +dir-merge .rules ++ *.[ch] +- *.o +- foo* +.fi .RE - -.PP -This will merge the contents of the /home/user/.global\-filter file at the -start of the list and also turns the \(dq\&.rules\(dq\& filename into a per\-directory -filter file. All rules read in prior to the start of the directory scan -follow the global anchoring rules (i.e. a leading slash matches at the root -of the transfer). -.PP -If a per\-directory merge\-file is specified with a path that is a parent -directory of the first transfer directory, rsync will scan all the parent -dirs from that starting point to the transfer directory for the indicated -per\-directory file. For instance, here is a common filter (see \fB\-F\fP): -.PP -.RS -\f(CW\-\-filter='\&: /.rsync\-filter'\&\fP +.P +This will merge the contents of the /home/user/.global-filter file at the start +of the list and also turns the ".rules" filename into a per-directory filter +file. All rules read in prior to the start of the directory scan follow the +global anchoring rules (i.e. a leading slash matches at the root of the +transfer). +.P +If a per-directory merge-file is specified with a path that is a parent +directory of the first transfer directory, rsync will scan all the parent dirs +from that starting point to the transfer directory for the indicated +per-directory file. For instance, here is a common filter (see \fB\-F\fP): +.RS 4 +.P +.nf +--filter=': /.rsync-filter' +.fi .RE - -.PP -That rule tells rsync to scan for the file .rsync\-filter in all -directories from the root down through the parent directory of the -transfer prior to the start of the normal directory scan of the file in -the directories that are sent as a part of the transfer. (Note: for an -rsync daemon, the root is always the same as the module\(cq\&s \(dq\&path\(dq\&.) -.PP -Some examples of this pre\-scanning for per\-directory files: -.PP -.RS -\f(CWrsync \-avF /src/path/ /dest/dir\fP -.br -\f(CWrsync \-av \-\-filter='\&: ../../.rsync\-filter'\& /src/path/ /dest/dir\fP -.br -\f(CWrsync \-av \-\-filter='\&: .rsync\-filter'\& /src/path/ /dest/dir\fP -.br +.P +That rule tells rsync to scan for the file .rsync-filter in all directories +from the root down through the parent directory of the transfer prior to the +start of the normal directory scan of the file in the directories that are sent +as a part of the transfer. (Note: for an rsync daemon, the root is always the +same as the module's "path".) +.P +Some examples of this pre-scanning for per-directory files: +.RS 4 +.P +.nf +rsync -avF /src/path/ /dest/dir +rsync -av --filter=': ../../.rsync-filter' /src/path/ /dest/dir +rsync -av --filter=': .rsync-filter' /src/path/ /dest/dir +.fi .RE - -.PP -The first two commands above will look for \(dq\&.rsync\-filter\(dq\& in \(dq\&/\(dq\& and -\(dq\&/src\(dq\& before the normal scan begins looking for the file in \(dq\&/src/path\(dq\& -and its subdirectories. The last command avoids the parent\-dir scan -and only looks for the \(dq\&.rsync\-filter\(dq\& files in each directory that is -a part of the transfer. -.PP -If you want to include the contents of a \(dq\&.cvsignore\(dq\& in your patterns, -you should use the rule \(dq\&:C\(dq\&, which creates a dir\-merge of the .cvsignore -file, but parsed in a CVS\-compatible manner. You can -use this to affect where the \fB\-\-cvs\-exclude\fP (\fB\-C\fP) option\(cq\&s inclusion of the -per\-directory .cvsignore file gets placed into your rules by putting the -\(dq\&:C\(dq\& wherever you like in your filter rules. Without this, rsync would -add the dir\-merge rule for the .cvsignore file at the end of all your other -rules (giving it a lower priority than your command\-line rules). For -example: -.PP -.RS -\f(CWcat <<EOT | rsync \-avC \-\-filter='\&. \-'\& a/ b\fP -.br -\f(CW+ foo.o\fP -.br -\f(CW:C\fP -.br -\f(CW\- *.old\fP -.br -\f(CWEOT\fP -.br -\f(CWrsync \-avC \-\-include=foo.o \-f :C \-\-exclude='\&*.old'\& a/ b\fP -.br +.P +The first two commands above will look for ".rsync-filter" in "/" and "/src" +before the normal scan begins looking for the file in "/src/path" and its +subdirectories. The last command avoids the parent-dir scan and only looks for +the ".rsync-filter" files in each directory that is a part of the transfer. +.P +If you want to include the contents of a ".cvsignore" in your patterns, you +should use the rule ":C", which creates a dir-merge of the .cvsignore file, but +parsed in a CVS-compatible manner. You can use this to affect where the +\fB\-\-cvs-exclude\fP (\fB\-C\fP) option's inclusion of the per-directory .cvsignore file +gets placed into your rules by putting the ":C" wherever you like in your +filter rules. Without this, rsync would add the dir-merge rule for the +\&.cvsignore file at the end of all your other rules (giving it a lower priority +than your command-line rules). For example: +.RS 4 +.P +.nf +cat <<EOT | rsync -avC --filter='. -' a/ b ++ foo.o +:C +- *.old +EOT +rsync -avC --include=foo.o -f :C --exclude='*.old' a/ b +.fi .RE - -.PP -Both of the above rsync commands are identical. Each one will merge all -the per\-directory .cvsignore rules in the middle of the list rather than -at the end. This allows their dir\-specific rules to supersede the rules -that follow the :C instead of being subservient to all your rules. To -affect the other CVS exclude rules (i.e. the default list of exclusions, -the contents of $HOME/.cvsignore, and the value of $CVSIGNORE) you should -omit the \fB\-C\fP command\-line option and instead insert a \(dq\&\-C\(dq\& rule into -your filter rules; e.g. \(dq\&\fB\-\-filter=\-C\fP\(dq\&. -.PP -.SH "LIST\-CLEARING FILTER RULE" - -.PP -You can clear the current include/exclude list by using the \(dq\&!\(dq\& filter -rule (as introduced in the FILTER RULES section above). The \(dq\¤t\(dq\& -list is either the global list of rules (if the rule is encountered while -parsing the filter options) or a set of per\-directory rules (which are -inherited in their own sub\-list, so a subdirectory can use this to clear -out the parent\(cq\&s rules). -.PP +.P +Both of the above rsync commands are identical. Each one will merge all the +per-directory .cvsignore rules in the middle of the list rather than at the +end. This allows their dir-specific rules to supersede the rules that follow +the :C instead of being subservient to all your rules. To affect the other CVS +exclude rules (i.e. the default list of exclusions, the contents of +$HOME/.cvsignore, and the value of $CVSIGNORE) you should omit the \fB\-C\fP +command-line option and instead insert a "\-C" rule into your filter rules; e.g. +"\fB\-\-filter=\-C\fP". +.P +.SH "LIST-CLEARING FILTER RULE" +.P +You can clear the current include/exclude list by using the "!" filter rule (as +introduced in the FILTER RULES section above). The "current" list is either +the global list of rules (if the rule is encountered while parsing the filter +options) or a set of per-directory rules (which are inherited in their own +sub-list, so a subdirectory can use this to clear out the parent's rules). +.P .SH "ANCHORING INCLUDE/EXCLUDE PATTERNS" - -.PP -As mentioned earlier, global include/exclude patterns are anchored at the -\(dq\&root of the transfer\(dq\& (as opposed to per\-directory patterns, which are -anchored at the merge\-file\(cq\&s directory). If you think of the transfer as -a subtree of names that are being sent from sender to receiver, the -transfer\-root is where the tree starts to be duplicated in the destination -directory. This root governs where patterns that start with a / match. -.PP -Because the matching is relative to the transfer\-root, changing the -trailing slash on a source path or changing your use of the \fB\-\-relative\fP -option affects the path you need to use in your matching (in addition to -changing how much of the file tree is duplicated on the destination -host). The following examples demonstrate this. -.PP -Let\(cq\&s say that we want to match two source files, one with an absolute -path of \(dq\&/home/me/foo/bar\(dq\&, and one with a path of \(dq\&/home/you/bar/baz\(dq\&. -Here is how the various command choices differ for a 2\-source transfer: -.PP -.RS -Example cmd: rsync \-a /home/me /home/you /dest -.br -+/\- pattern: /me/foo/bar -.br -+/\- pattern: /you/bar/baz -.br -Target file: /dest/me/foo/bar -.br -Target file: /dest/you/bar/baz -.br +.P +As mentioned earlier, global include/exclude patterns are anchored at the "root +of the transfer" (as opposed to per-directory patterns, which are anchored at +the merge-file's directory). If you think of the transfer as a subtree of +names that are being sent from sender to receiver, the transfer-root is where +the tree starts to be duplicated in the destination directory. This root +governs where patterns that start with a / match. +.P +Because the matching is relative to the transfer-root, changing the trailing +slash on a source path or changing your use of the \fB\-\-relative\fP option affects +the path you need to use in your matching (in addition to changing how much of +the file tree is duplicated on the destination host). The following examples +demonstrate this. +.P +Let's say that we want to match two source files, one with an absolute +path of "/home/me/foo/bar", and one with a path of "/home/you/bar/baz". +Here is how the various command choices differ for a 2-source transfer: +.RS 4 +.P +.nf +Example cmd: rsync -a /home/me /home/you /dest ++/- pattern: /me/foo/bar ++/- pattern: /you/bar/baz +Target file: /dest/me/foo/bar +Target file: /dest/you/bar/baz +.fi .RE - -.PP -.RS -Example cmd: rsync \-a /home/me/ /home/you/ /dest -.br -+/\- pattern: /foo/bar (note missing \(dq\&me\(dq\&) -.br -+/\- pattern: /bar/baz (note missing \(dq\&you\(dq\&) -.br -Target file: /dest/foo/bar -.br -Target file: /dest/bar/baz -.br +.RS 4 +.P +.nf +Example cmd: rsync -a /home/me/ /home/you/ /dest ++/- pattern: /foo/bar (note missing "me") ++/- pattern: /bar/baz (note missing "you") +Target file: /dest/foo/bar +Target file: /dest/bar/baz +.fi .RE - -.PP -.RS -Example cmd: rsync \-a \-\-relative /home/me/ /home/you /dest -.br -+/\- pattern: /home/me/foo/bar (note full path) -.br -+/\- pattern: /home/you/bar/baz (ditto) -.br -Target file: /dest/home/me/foo/bar -.br -Target file: /dest/home/you/bar/baz -.br +.RS 4 +.P +.nf +Example cmd: rsync -a --relative /home/me/ /home/you /dest ++/- pattern: /home/me/foo/bar (note full path) ++/- pattern: /home/you/bar/baz (ditto) +Target file: /dest/home/me/foo/bar +Target file: /dest/home/you/bar/baz +.fi .RE - -.PP -.RS -Example cmd: cd /home; rsync \-a \-\-relative me/foo you/ /dest -.br -+/\- pattern: /me/foo/bar (starts at specified path) -.br -+/\- pattern: /you/bar/baz (ditto) -.br -Target file: /dest/me/foo/bar -.br -Target file: /dest/you/bar/baz -.br +.RS 4 +.P +.nf +Example cmd: cd /home; rsync -a --relative me/foo you/ /dest ++/- pattern: /me/foo/bar (starts at specified path) ++/- pattern: /you/bar/baz (ditto) +Target file: /dest/me/foo/bar +Target file: /dest/you/bar/baz +.fi .RE - -.PP +.P The easiest way to see what name you should filter is to just look at the output when using \fB\-\-verbose\fP and put a / in front of the name -(use the \fB\-\-dry\-run\fP option if you\(cq\&re not yet ready to copy any files). -.PP -.SH "PER\-DIRECTORY RULES AND DELETE" - -.PP -Without a delete option, per\-directory rules are only relevant on the -sending side, so you can feel free to exclude the merge files themselves -without affecting the transfer. To make this easy, the \(cq\&e\(cq\& modifier adds -this exclude for you, as seen in these two equivalent commands: -.PP -.RS -\f(CWrsync \-av \-\-filter='\&: .excl'\& \-\-exclude=.excl host:src/dir /dest\fP -.br -\f(CWrsync \-av \-\-filter='\&:e .excl'\& host:src/dir /dest\fP -.br +(use the \fB\-\-dry-run\fP option if you're not yet ready to copy any files). +.P +.SH "PER-DIRECTORY RULES AND DELETE" +.P +Without a delete option, per-directory rules are only relevant on the sending +side, so you can feel free to exclude the merge files themselves without +affecting the transfer. To make this easy, the 'e' modifier adds this exclude +for you, as seen in these two equivalent commands: +.RS 4 +.P +.nf +rsync -av --filter=': .excl' --exclude=.excl host:src/dir /dest +rsync -av --filter=':e .excl' host:src/dir /dest +.fi .RE - -.PP +.P However, if you want to do a delete on the receiving side AND you want some -files to be excluded from being deleted, you\(cq\&ll need to be sure that the -receiving side knows what files to exclude. The easiest way is to include -the per\-directory merge files in the transfer and use \fB\-\-delete\-after\fP, -because this ensures that the receiving side gets all the same exclude -rules as the sending side before it tries to delete anything: -.PP -.RS -\f(CWrsync \-avF \-\-delete\-after host:src/dir /dest\fP +files to be excluded from being deleted, you'll need to be sure that the +receiving side knows what files to exclude. The easiest way is to include the +per-directory merge files in the transfer and use \fB\-\-delete-after\fP, because +this ensures that the receiving side gets all the same exclude rules as the +sending side before it tries to delete anything: +.RS 4 +.P +.nf +rsync -avF --delete-after host:src/dir /dest +.fi .RE - -.PP -However, if the merge files are not a part of the transfer, you\(cq\&ll need to -either specify some global exclude rules (i.e. specified on the command -line), or you\(cq\&ll need to maintain your own per\-directory merge files on -the receiving side. An example of the first is this (assume that the -remote .rules files exclude themselves): -.PP -.nf -rsync \-av \-\-filter=\(cq\&: .rules\(cq\& \-\-filter=\(cq\&. /my/extra.rules\(cq\& - \-\-delete host:src/dir /dest -.fi - -.PP +.P +However, if the merge files are not a part of the transfer, you'll need to +either specify some global exclude rules (i.e. specified on the command line), +or you'll need to maintain your own per-directory merge files on the receiving +side. An example of the first is this (assume that the remote .rules files +exclude themselves): +.RS 4 +.P +.nf +rsync -av --filter=': .rules' --filter='. /my/extra.rules' + --delete host:src/dir /dest +.fi +.RE +.P In the above example the extra.rules file can affect both sides of the transfer, but (on the sending side) the rules are subservient to the rules merged from the .rules files because they were specified after the -per\-directory merge rule. -.PP -In one final example, the remote side is excluding the .rsync\-filter -files from the transfer, but we want to use our own .rsync\-filter files -to control what gets deleted on the receiving side. To do this we must -specifically exclude the per\-directory merge files (so that they don\(cq\&t get -deleted) and then put rules into the local files to control what else -should not get deleted. Like one of these commands: -.PP -.nf - rsync \-av \-\-filter='\&:e /.rsync\-filter'\& \-\-delete \e - host:src/dir /dest - rsync \-avFF \-\-delete host:src/dir /dest -.fi - -.PP +per-directory merge rule. +.P +In one final example, the remote side is excluding the .rsync-filter files from +the transfer, but we want to use our own .rsync-filter files to control what +gets deleted on the receiving side. To do this we must specifically exclude +the per-directory merge files (so that they don't get deleted) and then put +rules into the local files to control what else should not get deleted. Like +one of these commands: +.RS 4 +.P +.nf +rsync -av --filter=':e /.rsync-filter' --delete \\ + host:src/dir /dest +rsync -avFF --delete host:src/dir /dest +.fi +.RE +.P .SH "BATCH MODE" - -.PP -Batch mode can be used to apply the same set of updates to many -identical systems. Suppose one has a tree which is replicated on a -number of hosts. Now suppose some changes have been made to this -source tree and those changes need to be propagated to the other -hosts. In order to do this using batch mode, rsync is run with the -write\-batch option to apply the changes made to the source tree to one -of the destination trees. The write\-batch option causes the rsync -client to store in a \(dq\&batch file\(dq\& all the information needed to repeat +.P +Batch mode can be used to apply the same set of updates to many identical +systems. Suppose one has a tree which is replicated on a number of hosts. Now +suppose some changes have been made to this source tree and those changes need +to be propagated to the other hosts. In order to do this using batch mode, +rsync is run with the write-batch option to apply the changes made to the +source tree to one of the destination trees. The write-batch option causes the +rsync client to store in a "batch file" all the information needed to repeat this operation against other, identical destination trees. -.PP -Generating the batch file once saves having to perform the file -status, checksum, and data block generation more than once when -updating multiple destination trees. Multicast transport protocols can -be used to transfer the batch update files in parallel to many hosts -at once, instead of sending the same data to every host individually. -.PP -To apply the recorded changes to another destination tree, run rsync -with the read\-batch option, specifying the name of the same batch -file, and the destination tree. Rsync updates the destination tree -using the information stored in the batch file. -.PP -For your convenience, a script file is also created when the write\-batch -option is used: it will be named the same as the batch file with \(dq\&.sh\(dq\& -appended. This script file contains a command\-line suitable for updating a -destination tree using the associated batch file. It can be executed using -a Bourne (or Bourne\-like) shell, optionally passing in an alternate -destination tree pathname which is then used instead of the original -destination path. This is useful when the destination tree path on the -current host differs from the one used to create the batch file. -.PP +.P +Generating the batch file once saves having to perform the file status, +checksum, and data block generation more than once when updating multiple +destination trees. Multicast transport protocols can be used to transfer the +batch update files in parallel to many hosts at once, instead of sending the +same data to every host individually. +.P +To apply the recorded changes to another destination tree, run rsync with the +read-batch option, specifying the name of the same batch file, and the +destination tree. Rsync updates the destination tree using the information +stored in the batch file. +.P +For your convenience, a script file is also created when the write-batch option +is used: it will be named the same as the batch file with ".sh" appended. This +script file contains a command-line suitable for updating a destination tree +using the associated batch file. It can be executed using a Bourne (or +Bourne-like) shell, optionally passing in an alternate destination tree +pathname which is then used instead of the original destination path. This is +useful when the destination tree path on the current host differs from the one +used to create the batch file. +.P Examples: -.PP -.RS -\f(CW$ rsync \-\-write\-batch=foo \-a host:/source/dir/ /adest/dir/\fP -.br -\f(CW$ scp foo* remote:\fP -.br -\f(CW$ ssh remote ./foo.sh /bdest/dir/\fP -.br +.RS 4 +.P +.nf +$ rsync --write-batch=foo -a host:/source/dir/ /adest/dir/ +$ scp foo* remote: +$ ssh remote ./foo.sh /bdest/dir/ +.fi .RE - -.PP -.RS -\f(CW$ rsync \-\-write\-batch=foo \-a /source/dir/ /adest/dir/\fP -.br -\f(CW$ ssh remote rsync \-\-read\-batch=\- \-a /bdest/dir/ <foo\fP -.br +.RS 4 +.P +.nf +$ rsync --write-batch=foo -a /source/dir/ /adest/dir/ +$ ssh remote rsync --read-batch=- -a /bdest/dir/ <foo +.fi .RE - -.PP -In these examples, rsync is used to update /adest/dir/ from /source/dir/ -and the information to repeat this operation is stored in \(dq\&foo\(dq\& and -\(dq\&foo.sh\(dq\&. The host \(dq\&remote\(dq\& is then updated with the batched data going -into the directory /bdest/dir. The differences between the two examples -reveals some of the flexibility you have in how you deal with batches: -.PP -.IP o -The first example shows that the initial copy doesn\(cq\&t have to be -local \-\- you can push or pull data to/from a remote host using either the -remote\-shell syntax or rsync daemon syntax, as desired. -.IP o -The first example uses the created \(dq\&foo.sh\(dq\& file to get the right -rsync options when running the read\-batch command on the remote host. -.IP o -The second example reads the batch data via standard input so that -the batch file doesn\(cq\&t need to be copied to the remote machine first. -This example avoids the foo.sh script because it needed to use a modified -\fB\-\-read\-batch\fP option, but you could edit the script file if you wished to -make use of it (just be sure that no other option is trying to use -standard input, such as the \(dq\&\fB\-\-exclude\-from=\-\fP\(dq\& option). - -.PP +.P +In these examples, rsync is used to update /adest/dir/ from /source/dir/ and +the information to repeat this operation is stored in "foo" and "foo.sh". The +host "remote" is then updated with the batched data going into the directory +/bdest/dir. The differences between the two examples reveals some of the +flexibility you have in how you deal with batches: +.P +.IP o +The first example shows that the initial copy doesn't have to be local\ \-\- you +can push or pull data to/from a remote host using either the remote-shell +syntax or rsync daemon syntax, as desired. +.IP o +The first example uses the created "foo.sh" file to get the right rsync +options when running the read-batch command on the remote host. +.IP o +The second example reads the batch data via standard input so that the batch +file doesn't need to be copied to the remote machine first. This example +avoids the foo.sh script because it needed to use a modified \fB\-\-read-batch\fP +option, but you could edit the script file if you wished to make use of it +(just be sure that no other option is trying to use standard input, such as +the "\fB\-\-exclude-from=\-\fP" option). +.P Caveats: -.PP -The read\-batch option expects the destination tree that it is updating -to be identical to the destination tree that was used to create the -batch update fileset. When a difference between the destination trees -is encountered the update might be discarded with a warning (if the file -appears to be up\-to\-date already) or the file\-update may be attempted -and then, if the file fails to verify, the update discarded with an -error. This means that it should be safe to re\-run a read\-batch operation -if the command got interrupted. If you wish to force the batched\-update to -always be attempted regardless of the file\(cq\&s size and date, use the \fB\-I\fP -option (when reading the batch). -If an error occurs, the destination tree will probably be in a -partially updated state. In that case, rsync can -be used in its regular (non\-batch) mode of operation to fix up the -destination tree. -.PP -The rsync version used on all destinations must be at least as new as the -one used to generate the batch file. Rsync will die with an error if the -protocol version in the batch file is too new for the batch\-reading rsync -to handle. See also the \fB\-\-protocol\fP option for a way to have the -creating rsync generate a batch file that an older rsync can understand. -(Note that batch files changed format in version 2.6.3, so mixing versions -older than that with newer versions will not work.) -.PP -When reading a batch file, rsync will force the value of certain options -to match the data in the batch file if you didn\(cq\&t set them to the same -as the batch\-writing command. Other options can (and should) be changed. -For instance \fB\-\-write\-batch\fP changes to \fB\-\-read\-batch\fP, -\fB\-\-files\-from\fP is dropped, and the -\fB\-\-filter\fP/\fB\-\-include\fP/\fB\-\-exclude\fP options are not needed unless +.P +The read-batch option expects the destination tree that it is updating to be +identical to the destination tree that was used to create the batch update +fileset. When a difference between the destination trees is encountered the +update might be discarded with a warning (if the file appears to be up-to-date +already) or the file-update may be attempted and then, if the file fails to +verify, the update discarded with an error. This means that it should be safe +to re-run a read-batch operation if the command got interrupted. If you wish +to force the batched-update to always be attempted regardless of the file's +size and date, use the \fB\-I\fP option (when reading the batch). If an error +occurs, the destination tree will probably be in a partially updated state. In +that case, rsync can be used in its regular (non-batch) mode of operation to +fix up the destination tree. +.P +The rsync version used on all destinations must be at least as new as the one +used to generate the batch file. Rsync will die with an error if the protocol +version in the batch file is too new for the batch-reading rsync to handle. +See also the \fB\-\-protocol\fP option for a way to have the creating rsync generate +a batch file that an older rsync can understand. (Note that batch files +changed format in version 2.6.3, so mixing versions older than that with newer +versions will not work.) +.P +When reading a batch file, rsync will force the value of certain options to +match the data in the batch file if you didn't set them to the same as the +batch-writing command. Other options can (and should) be changed. For +instance \fB\-\-write-batch\fP changes to \fB\-\-read-batch\fP, \fB\-\-files-from\fP is dropped, +and the \fB\-\-filter\fP / \fB\-\-include\fP / \fB\-\-exclude\fP options are not needed unless one of the \fB\-\-delete\fP options is specified. -.PP +.P The code that creates the BATCH.sh file transforms any filter/include/exclude -options into a single list that is appended as a \(dq\&here\(dq\& document to the -shell script file. An advanced user can use this to modify the exclude -list if a change in what gets deleted by \fB\-\-delete\fP is desired. A normal -user can ignore this detail and just use the shell script as an easy way -to run the appropriate \fB\-\-read\-batch\fP command for the batched data. -.PP -The original batch mode in rsync was based on \(dq\&rsync+\(dq\&, but the latest +options into a single list that is appended as a "here" document to the shell +script file. An advanced user can use this to modify the exclude list if a +change in what gets deleted by \fB\-\-delete\fP is desired. A normal user can ignore +this detail and just use the shell script as an easy way to run the appropriate +\fB\-\-read-batch\fP command for the batched data. +.P +The original batch mode in rsync was based on "rsync+", but the latest version uses a new implementation. -.PP +.P .SH "SYMBOLIC LINKS" - -.PP +.P Three basic behaviors are possible when rsync encounters a symbolic link in the source directory. -.PP -By default, symbolic links are not transferred at all. A message -\(dq\&skipping non\-regular\(dq\& file is emitted for any symlinks that exist. -.PP -If \fB\-\-links\fP is specified, then symlinks are recreated with the same -target on the destination. Note that \fB\-\-archive\fP implies -\fB\-\-links\fP. -.PP -If \fB\-\-copy\-links\fP is specified, then symlinks are \(dq\&collapsed\(dq\& by +.P +By default, symbolic links are not transferred at all. A message "skipping +non-regular" file is emitted for any symlinks that exist. +.P +If \fB\-\-links\fP is specified, then symlinks are recreated with the same target on +the destination. Note that \fB\-\-archive\fP implies \fB\-\-links\fP. +.P +If \fB\-\-copy-links\fP is specified, then symlinks are "collapsed" by copying their referent, rather than the symlink. -.PP -Rsync can also distinguish \(dq\&safe\(dq\& and \(dq\&unsafe\(dq\& symbolic links. An -example where this might be used is a web site mirror that wishes to -ensure that the rsync module that is copied does not include symbolic links to -\fB/etc/passwd\fP in the public section of the site. Using -\fB\-\-copy\-unsafe\-links\fP will cause any links to be copied as the file -they point to on the destination. Using \fB\-\-safe\-links\fP will cause -unsafe links to be omitted altogether. (Note that you must specify -\fB\-\-links\fP for \fB\-\-safe\-links\fP to have any effect.) -.PP +.P +Rsync can also distinguish "safe" and "unsafe" symbolic links. An example +where this might be used is a web site mirror that wishes to ensure that the +rsync module that is copied does not include symbolic links to \fB/etc/passwd\fP in +the public section of the site. Using \fB\-\-copy-unsafe-links\fP will cause any +links to be copied as the file they point to on the destination. Using +\fB\-\-safe-links\fP will cause unsafe links to be omitted altogether. (Note that you +must specify \fB\-\-links\fP for \fB\-\-safe-links\fP to have any effect.) +.P Symbolic links are considered unsafe if they are absolute symlinks -(start with \fB/\fP), empty, or if they contain enough \(dq\&..\(dq\& +(start with \fB/\fP), empty, or if they contain enough ".." components to ascend from the directory being copied. -.PP -Here\(cq\&s a summary of how the symlink options are interpreted. The list is -in order of precedence, so if your combination of options isn\(cq\&t mentioned, -use the first line that is a complete subset of your options: -.PP -.IP "\fB\-\-copy\-links\fP" -Turn all symlinks into normal files (leaving no -symlinks for any other options to affect). -.PP -.IP "\fB\-\-links \-\-copy\-unsafe\-links\fP" -Turn all unsafe symlinks into files -and duplicate all safe symlinks. -.PP -.IP "\fB\-\-copy\-unsafe\-links\fP" -Turn all unsafe symlinks into files, noisily -skip all safe symlinks. -.PP -.IP "\fB\-\-links \-\-safe\-links\fP" -Duplicate safe symlinks and skip unsafe -ones. -.PP +.P +Here's a summary of how the symlink options are interpreted. The list is in +order of precedence, so if your combination of options isn't mentioned, use the +first line that is a complete subset of your options: +.P +.IP "\fB\-\-copy-links\fP" +Turn all symlinks into normal files (leaving no symlinks for +any other options to affect). +.IP "\fB\-\-links\ \-\-copy-unsafe-links\fP" +Turn all unsafe symlinks into files and +duplicate all safe symlinks. +.IP "\fB\-\-copy-unsafe-links\fP" +Turn all unsafe symlinks into files, noisily skip all +safe symlinks. +.IP "\fB\-\-links\ \-\-safe-links\fP" +Duplicate safe symlinks and skip unsafe ones. .IP "\fB\-\-links\fP" Duplicate all symlinks. -.PP +.P .SH "DIAGNOSTICS" - -.PP -rsync occasionally produces error messages that may seem a little -cryptic. The one that seems to cause the most confusion is \(dq\&protocol -version mismatch \-\- is your shell clean?\(dq\&. -.PP -This message is usually caused by your startup scripts or remote shell -facility producing unwanted garbage on the stream that rsync is using -for its transport. The way to diagnose this problem is to run your -remote shell like this: -.PP -.RS -\f(CWssh remotehost /bin/true > out.dat\fP +.P +rsync occasionally produces error messages that may seem a little cryptic. The +one that seems to cause the most confusion is "protocol version mismatch\ \-\- is +your shell clean?". +.P +This message is usually caused by your startup scripts or remote shell facility +producing unwanted garbage on the stream that rsync is using for its transport. +The way to diagnose this problem is to run your remote shell like this: +.RS 4 +.P +.nf +ssh remotehost /bin/true > out.dat +.fi .RE - -.PP -then look at out.dat. If everything is working correctly then out.dat -should be a zero length file. If you are getting the above error from -rsync then you will probably find that out.dat contains some text or -data. Look at the contents and try to work out what is producing -it. The most common cause is incorrectly configured shell startup -scripts (such as .cshrc or .profile) that contain output statements -for non\-interactive logins. -.PP -If you are having trouble debugging filter patterns, then -try specifying the \fB\-vv\fP option. At this level of verbosity rsync will -show why each individual file is included or excluded. -.PP +.P +then look at out.dat. If everything is working correctly then out.dat should +be a zero length file. If you are getting the above error from rsync then you +will probably find that out.dat contains some text or data. Look at the +contents and try to work out what is producing it. The most common cause is +incorrectly configured shell startup scripts (such as .cshrc or .profile) that +contain output statements for non-interactive logins. +.P +If you are having trouble debugging filter patterns, then try specifying the +\fB\-vv\fP option. At this level of verbosity rsync will show why each individual +file is included or excluded. +.P .SH "EXIT VALUES" - -.PP +.P .IP "\fB0\fP" Success .IP "\fB1\fP" @@ -3922,14 +3958,13 @@ Protocol incompatibility .IP "\fB3\fP" Errors selecting input/output files, dirs .IP "\fB4\fP" -Requested action not supported: an attempt -was made to manipulate 64\-bit files on a platform that cannot support -them; or an option was specified that is supported by the client and -not by the server. +Requested action not supported: an attempt was made to manipulate +64-bit files on a platform that cannot support them; or an option was +specified that is supported by the client and not by the server. .IP "\fB5\fP" -Error starting client\-server protocol +Error starting client-server protocol .IP "\fB6\fP" -Daemon unable to append to log\-file +Daemon unable to append to log-file .IP "\fB10\fP" Error in socket I/O .IP "\fB11\fP" @@ -3943,8 +3978,7 @@ Error in IPC code .IP "\fB20\fP" Received SIGUSR1 or SIGINT .IP "\fB21\fP" -Some error returned by -\f(CWwaitpid()\fP +Some error returned by \fBwaitpid()\fP .IP "\fB22\fP" Error allocating core memory buffers .IP "\fB23\fP" @@ -3952,129 +3986,112 @@ Partial transfer due to error .IP "\fB24\fP" Partial transfer due to vanished source files .IP "\fB25\fP" -The \-\-max\-delete limit stopped deletions +The \-\-max-delete limit stopped deletions .IP "\fB30\fP" Timeout in data send/receive .IP "\fB35\fP" Timeout waiting for daemon connection - -.PP +.P .SH "ENVIRONMENT VARIABLES" - -.PP +.P .IP "\fBCVSIGNORE\fP" -The CVSIGNORE environment variable supplements any -ignore patterns in .cvsignore files. See the \fB\-\-cvs\-exclude\fP option for -more details. +The CVSIGNORE environment variable supplements any ignore patterns in +\&.cvsignore files. See the \fB\-\-cvs-exclude\fP option for more details. .IP "\fBRSYNC_ICONV\fP" -Specify a default \fB\-\-iconv\fP setting using this -environment variable. (First supported in 3.0.0.) +Specify a default \fB\-\-iconv\fP setting using this environment variable. (First +supported in 3.0.0.) .IP "\fBRSYNC_PROTECT_ARGS\fP" -Specify a non\-zero numeric value if you want the -\fB\-\-protect\-args\fP option to be enabled by default, or a zero value to make -sure that it is disabled by default. (First supported in 3.1.0.) +Specify a non-zero numeric value if you want the \fB\-\-protect-args\fP option to +be enabled by default, or a zero value to make sure that it is disabled by +default. (First supported in 3.1.0.) .IP "\fBRSYNC_RSH\fP" -The RSYNC_RSH environment variable allows you to -override the default shell used as the transport for rsync. Command line -options are permitted after the command name, just as in the \fB\-e\fP option. +The RSYNC_RSH environment variable allows you to override the default shell +used as the transport for rsync. Command line options are permitted after +the command name, just as in the \fB\-e\fP option. .IP "\fBRSYNC_PROXY\fP" -The RSYNC_PROXY environment variable allows you to -redirect your rsync client to use a web proxy when connecting to a -rsync daemon. You should set RSYNC_PROXY to a hostname:port pair. +The RSYNC_PROXY environment variable allows you to redirect your rsync +client to use a web proxy when connecting to a rsync daemon. You should +set RSYNC_PROXY to a hostname:port pair. .IP "\fBRSYNC_PASSWORD\fP" -Setting RSYNC_PASSWORD to the required -password allows you to run authenticated rsync connections to an rsync -daemon without user intervention. Note that this does not supply a -password to a remote shell transport such as ssh; to learn how to do that, -consult the remote shell\(cq\&s documentation. +Setting RSYNC_PASSWORD to the required password allows you to run +authenticated rsync connections to an rsync daemon without user +intervention. Note that this does not supply a password to a remote shell +transport such as ssh; to learn how to do that, consult the remote shell's +documentation. .IP "\fBUSER\fP or \fBLOGNAME\fP" -The USER or LOGNAME environment variables -are used to determine the default username sent to an rsync daemon. -If neither is set, the username defaults to \(dq\&nobody\(dq\&. +The USER or LOGNAME environment variables are used to determine the default +username sent to an rsync daemon. If neither is set, the username defaults +to "nobody". .IP "\fBHOME\fP" -The HOME environment variable is used to find the user\(cq\&s -default .cvsignore file. - -.PP +The HOME environment variable is used to find the user's default .cvsignore +file. +.P .SH "FILES" - -.PP +.P /etc/rsyncd.conf or rsyncd.conf -.PP +.P .SH "SEE ALSO" - -.PP -\fBrsyncd.conf\fP(5) -.PP +.P +\fBrsync-ssl\fP(1), \fBrsyncd.conf\fP(5) +.P .SH "BUGS" - -.PP +.P times are transferred as *nix time_t values -.PP -When transferring to FAT filesystems rsync may re\-sync +.P +When transferring to FAT filesystems rsync may re-sync unmodified files. -See the comments on the \fB\-\-modify\-window\fP option. -.PP +See the comments on the \fB\-\-modify-window\fP option. +.P file permissions, devices, etc. are transferred as native numerical values -.PP +.P see also the comments on the \fB\-\-delete\fP option -.PP -Please report bugs! See the web site at -http://rsync.samba.org/ -.PP +.P +Please report bugs! See the web site at http://rsync.samba.org/. +.P .SH "VERSION" - -.PP -This man page is current for version 3.1.3 of rsync. -.PP +.P +This man page is current for version 3.2.0 of rsync. +.P .SH "INTERNAL OPTIONS" - -.PP -The options \fB\-\-server\fP and \fB\-\-sender\fP are used internally by rsync, -and should never be typed by a user under normal circumstances. Some -awareness of these options may be needed in certain scenarios, such as -when setting up a login that can only run an rsync command. For instance, -the support directory of the rsync distribution has an example script -named rrsync (for restricted rsync) that can be used with a restricted -ssh login. -.PP +.P +The options \fB\-\-server\fP and \fB\-\-sender\fP are used internally by rsync, and should +never be typed by a user under normal circumstances. Some awareness of these +options may be needed in certain scenarios, such as when setting up a login +that can only run an rsync command. For instance, the support directory of the +rsync distribution has an example script named rrsync (for restricted rsync) +that can be used with a restricted ssh login. +.P .SH "CREDITS" - -.PP +.P rsync is distributed under the GNU General Public License. See the file COPYING for details. -.PP -A WEB site is available at -http://rsync.samba.org/. The site -includes an FAQ\-O\-Matic which may cover questions unanswered by this -manual page. -.PP -The primary ftp site for rsync is -ftp://rsync.samba.org/pub/rsync. -.PP -We would be delighted to hear from you if you like this program. -Please contact the mailing\-list at rsync@lists.samba.org. -.PP -This program uses the excellent zlib compression library written by -Jean\-loup Gailly and Mark Adler. -.PP +.P +A web site is available at http://rsync.samba.org/. The site includes an +FAQ-O-Matic which may cover questions unanswered by this manual page. +.P +The primary ftp site for rsync is ftp://rsync.samba.org/pub/rsync +.P +We would be delighted to hear from you if you like this program. Please +contact the mailing-list at rsync@lists.samba.org. +.P +This program uses the excellent zlib compression library written by Jean-loup +Gailly and Mark Adler. +.P .SH "THANKS" - -.PP +.P Special thanks go out to: John Van Essen, Matt McCutchen, Wesley W. Terpstra, David Dykstra, Jos Backus, Sebastian Krahmer, Martin Pool, and our -gone\-but\-not\-forgotten compadre, J.W. Schultz. -.PP -Thanks also to Richard Brent, Brendan Mackay, Bill Waite, Stephen Rothwell -and David Bell. I\(cq\&ve probably missed some people, my apologies if I have. -.PP +gone-but-not-forgotten compadre, J.W. Schultz. +.P +Thanks also to Richard Brent, Brendan Mackay, Bill Waite, Stephen Rothwell and +David Bell. I've probably missed some people, my apologies if I have. +.P .SH "AUTHOR" - -.PP -rsync was originally written by Andrew Tridgell and Paul Mackerras. -Many people have later contributed to it. It is currently maintained -by Wayne Davison. -.PP +.P +rsync was originally written by Andrew Tridgell and Paul Mackerras. Many +people have later contributed to it. It is currently maintained by Wayne +Davison. +.P Mailing lists for support and development are available at -http://lists.samba.org +http://lists.samba.org/. diff --git a/rsync.1.html b/rsync.1.html new file mode 100644 index 00000000..63beda9b --- /dev/null +++ b/rsync.1.html @@ -0,0 +1,3722 @@ +<html><head> +<title>rsync(1) man page</title> +<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet"> +<style> +body { + max-width: 50em; + margin: auto; +} +body, b, strong, u { + font-family: 'Roboto', sans-serif; +} +code { + font-family: 'Roboto Mono', monospace; + font-weight: bold; + white-space: pre; +} +pre code { + display: block; + font-weight: normal; +} +blockquote pre code { + background: #f1f1f1; +} +dd p:first-of-type { + margin-block-start: 0em; +} +</style> +</head><body> +<h1>NAME</h1> +<p>rsync -⁠ a fast, versatile, remote (and local) file-copying tool</p> +<h1>SYNOPSIS</h1> +<pre><code>Local: + rsync [OPTION...] SRC... [DEST] + +Access via remote shell: + Pull: + rsync [OPTION...] [USER@]HOST:SRC... [DEST] + Push: + rsync [OPTION...] SRC... [USER@]HOST:DEST + +Access via rsync daemon: + Pull: + rsync [OPTION...] [USER@]HOST::SRC... [DEST] + rsync [OPTION...] rsync://[USER@]HOST[:PORT]/SRC... [DEST] + Push: + rsync [OPTION...] SRC... [USER@]HOST::DEST + rsync [OPTION...] SRC... rsync://[USER@]HOST[:PORT]/DEST) +</code></pre> +<p>Usages with just one SRC arg and no DEST arg will list the source files instead +of copying.</p> +<h1>DESCRIPTION</h1> +<p>Rsync is a fast and extraordinarily versatile file copying tool. It can copy +locally, to/from another host over any remote shell, or to/from a remote rsync +daemon. It offers a large number of options that control every aspect of its +behavior and permit very flexible specification of the set of files to be +copied. It is famous for its delta-transfer algorithm, which reduces the +amount of data sent over the network by sending only the differences between +the source files and the existing files in the destination. Rsync is widely +used for backups and mirroring and as an improved copy command for everyday +use.</p> +<p>Rsync finds files that need to be transferred using a "quick check" algorithm +(by default) that looks for files that have changed in size or in last-modified +time. Any changes in the other preserved attributes (as requested by options) +are made on the destination file directly when the quick check indicates that +the file's data does not need to be updated.</p> +<p>Some of the additional features of rsync are:</p> +<ul> +<li>support for copying links, devices, owners, groups, and permissions</li> +<li>exclude and exclude-from options similar to GNU tar</li> +<li>a CVS exclude mode for ignoring the same files that CVS would ignore</li> +<li>can use any transparent remote shell, including ssh or rsh</li> +<li>does not require super-user privileges</li> +<li>pipelining of file transfers to minimize latency costs</li> +<li>support for anonymous or authenticated rsync daemons (ideal for mirroring)</li> +</ul> +<h1>GENERAL</h1> +<p>Rsync copies files either to or from a remote host, or locally on the current +host (it does not support copying files between two remote hosts).</p> +<p>There are two different ways for rsync to contact a remote system: using a +remote-shell program as the transport (such as ssh or rsh) or contacting an +rsync daemon directly via TCP. The remote-shell transport is used whenever the +source or destination path contains a single colon (:) separator after a host +specification. Contacting an rsync daemon directly happens when the source or +destination path contains a double colon (::) separator after a host +specification, OR when an rsync:// URL is specified (see also the "USING +RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION" section for an exception +to this latter rule).</p> +<p>As a special case, if a single source arg is specified without a destination, +the files are listed in an output format similar to "<code>ls -l</code>".</p> +<p>As expected, if neither the source or destination path specify a remote host, +the copy occurs locally (see also the <code>--list-only</code> option).</p> +<p>Rsync refers to the local side as the client and the remote side as the server. +Don't confuse server with an rsync daemon. A daemon is always a server, but a +server can be either a daemon or a remote-shell spawned process.</p> +<h1>SETUP</h1> +<p>See the file README.md for installation instructions.</p> +<p>Once installed, you can use rsync to any machine that you can access via a +remote shell (as well as some that you can access using the rsync daemon-mode +protocol). For remote transfers, a modern rsync uses ssh for its +communications, but it may have been configured to use a different remote shell +by default, such as rsh or remsh.</p> +<p>You can also specify any remote shell you like, either by using the <code>-e</code> +command line option, or by setting the RSYNC_RSH environment variable.</p> +<p>Note that rsync must be installed on both the source and destination machines.</p> +<h1>USAGE</h1> +<p>You use rsync in the same way you use rcp. You must specify a source and a +destination, one of which may be remote.</p> +<p>Perhaps the best way to explain the syntax is with some examples:</p> +<blockquote> +<pre><code>rsync -t *.c foo:src/ +</code></pre> +</blockquote> +<p>This would transfer all files matching the pattern <code>*.c</code> from the current +directory to the directory src on the machine foo. If any of the files already +exist on the remote system then the rsync remote-update protocol is used to +update the file by sending only the differences in the data. Note that the +expansion of wildcards on the commandline (<code>*.c</code>) into a list of files is +handled by the shell before it runs rsync and not by rsync itself (exactly the +same as all other posix-style programs).</p> +<blockquote> +<pre><code>rsync -avz foo:src/bar /data/tmp +</code></pre> +</blockquote> +<p>This would recursively transfer all files from the directory src/bar on the +machine foo into the /data/tmp/bar directory on the local machine. The files +are transferred in archive mode, which ensures that symbolic links, devices, +attributes, permissions, ownerships, etc. are preserved in the transfer. +Additionally, compression will be used to reduce the size of data portions of +the transfer.</p> +<blockquote> +<pre><code>rsync -avz foo:src/bar/ /data/tmp +</code></pre> +</blockquote> +<p>A trailing slash on the source changes this behavior to avoid creating an +additional directory level at the destination. You can think of a trailing / +on a source as meaning "copy the contents of this directory" as opposed to +"copy the directory by name", but in both cases the attributes of the +containing directory are transferred to the containing directory on the +destination. In other words, each of the following commands copies the files +in the same way, including their setting of the attributes of /dest/foo:</p> +<blockquote> +<pre><code>rsync -av /src/foo /dest +rsync -av /src/foo/ /dest/foo +</code></pre> +</blockquote> +<p>Note also that host and module references don't require a trailing slash to +copy the contents of the default directory. For example, both of these copy +the remote directory's contents into "/dest":</p> +<blockquote> +<pre><code>rsync -av host: /dest +rsync -av host::module /dest +</code></pre> +</blockquote> +<p>You can also use rsync in local-only mode, where both the source and +destination don't have a ':' in the name. In this case it behaves like an +improved copy command.</p> +<p>Finally, you can list all the (listable) modules available from a particular +rsync daemon by leaving off the module name:</p> +<blockquote> +<pre><code>rsync somehost.mydomain.com:: +</code></pre> +</blockquote> +<p>See the following section for more details.</p> +<h1>ADVANCED USAGE</h1> +<p>The syntax for requesting multiple files from a remote host is done by +specifying additional remote-host args in the same style as the first, or with +the hostname omitted. For instance, all these work:</p> +<blockquote> +<pre><code>rsync -av host:file1 :file2 host:file{3,4} /dest/ +rsync -av host::modname/file{1,2} host::modname/file3 /dest/ +rsync -av host::modname/file1 ::modname/file{3,4} +</code></pre> +</blockquote> +<p>Older versions of rsync required using quoted spaces in the SRC, like these +examples:</p> +<blockquote> +<pre><code>rsync -av host:'dir1/file1 dir2/file2' /dest +rsync host::'modname/dir1/file1 modname/dir2/file2' /dest +</code></pre> +</blockquote> +<p>This word-splitting still works (by default) in the latest rsync, but is not as +easy to use as the first method.</p> +<p>If you need to transfer a filename that contains whitespace, you can either +specify the <code>--protect-args</code> (<code>-s</code>) option, or you'll need to escape the +whitespace in a way that the remote shell will understand. For instance:</p> +<blockquote> +<pre><code>rsync -av host:'file\ name\ with\ spaces' /dest +</code></pre> +</blockquote> +<h1>CONNECTING TO AN RSYNC DAEMON</h1> +<p>It is also possible to use rsync without a remote shell as the transport. In +this case you will directly connect to a remote rsync daemon, typically using +TCP port 873. (This obviously requires the daemon to be running on the remote +system, so refer to the STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS section +below for information on that.)</p> +<p>Using rsync in this way is the same as using it with a remote shell except +that:</p> +<ul> +<li>you either use a double colon :: instead of a single colon to separate the +hostname from the path, or you use an rsync:// URL.</li> +<li>the first word of the "path" is actually a module name.</li> +<li>the remote daemon may print a message of the day when you connect.</li> +<li>if you specify no path name on the remote daemon then the list of accessible +paths on the daemon will be shown.</li> +<li>if you specify no local destination then a listing of the specified files on +the remote daemon is provided.</li> +<li>you must not specify the <code>--rsh</code> (<code>-e</code>) option (since that overrides the +daemon connection to use ssh -⁠-⁠ see USING RSYNC-DAEMON FEATURES VIA A +REMOTE-SHELL CONNECTION below).</li> +</ul> +<p>An example that copies all the files in a remote module named "src":</p> +<blockquote> +<pre><code>rsync -av host::src /dest +</code></pre> +</blockquote> +<p>Some modules on the remote daemon may require authentication. If so, you will +receive a password prompt when you connect. You can avoid the password prompt +by setting the environment variable RSYNC_PASSWORD to the password you want to +use or using the <code>--password-file</code> option. This may be useful when scripting +rsync.</p> +<p>WARNING: On some systems environment variables are visible to all users. On +those systems using <code>--password-file</code> is recommended.</p> +<p>You may establish the connection via a web proxy by setting the environment +variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note +that your web proxy's configuration must support proxy connections to port 873.</p> +<p>You may also establish a daemon connection using a program as a proxy by +setting the environment variable RSYNC_CONNECT_PROG to the commands you wish to +run in place of making a direct socket connection. The string may contain the +escape "%H" to represent the hostname specified in the rsync command (so use +"%%" if you need a single "%" in your string). For example:</p> +<blockquote> +<pre><code>export RSYNC_CONNECT_PROG='ssh proxyhost nc %H 873' +rsync -av targethost1::module/src/ /dest/ +rsync -av rsync://targethost2/module/src/ /dest/ +</code></pre> +</blockquote> +<p>The command specified above uses ssh to run nc (netcat) on a proxyhost, which +forwards all data to port 873 (the rsync daemon) on the targethost (%H).</p> +<p>Note also that if the RSYNC_SHELL environment variable is set, that program +will be used to run the RSYNC_CONNECT_PROG command instead of using the default +shell of the <strong>system()</strong> call.</p> +<h1>USING RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION</h1> +<p>It is sometimes useful to use various features of an rsync daemon (such as +named modules) without actually allowing any new socket connections into a +system (other than what is already required to allow remote-shell access). +Rsync supports connecting to a host using a remote shell and then spawning a +single-use "daemon" server that expects to read its config file in the home dir +of the remote user. This can be useful if you want to encrypt a daemon-style +transfer's data, but since the daemon is started up fresh by the remote user, +you may not be able to use features such as chroot or change the uid used by +the daemon. (For another way to encrypt a daemon transfer, consider using ssh +to tunnel a local port to a remote machine and configure a normal rsync daemon +on that remote host to only allow connections from "localhost".)</p> +<p>From the user's perspective, a daemon transfer via a remote-shell connection +uses nearly the same command-line syntax as a normal rsync-daemon transfer, +with the only exception being that you must explicitly set the remote shell +program on the command-line with the <code>--rsh=COMMAND</code> option. (Setting the +RSYNC_RSH in the environment will not turn on this functionality.) For example:</p> +<blockquote> +<pre><code>rsync -av --rsh=ssh host::module /dest +</code></pre> +</blockquote> +<p>If you need to specify a different remote-shell user, keep in mind that the +user@ prefix in front of the host is specifying the rsync-user value (for a +module that requires user-based authentication). This means that you must give +the '-⁠l user' option to ssh when specifying the remote-shell, as in this +example that uses the short version of the <code>--rsh</code> option:</p> +<blockquote> +<pre><code>rsync -av -e "ssh -l ssh-user" rsync-user@host::module /dest +</code></pre> +</blockquote> +<p>The "ssh-user" will be used at the ssh level; the "rsync-user" will be used to +log-in to the "module".</p> +<h1>STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS</h1> +<p>In order to connect to an rsync daemon, the remote system needs to have a +daemon already running (or it needs to have configured something like inetd to +spawn an rsync daemon for incoming connections on a particular port). For full +information on how to start a daemon that will handling incoming socket +connections, see the <strong>rsyncd.conf</strong>(5) man page -⁠-⁠ that is the config file for +the daemon, and it contains the full details for how to run the daemon +(including stand-alone and inetd configurations).</p> +<p>If you're using one of the remote-shell transports for the transfer, there is +no need to manually start an rsync daemon.</p> +<h1>SORTED TRANSFER ORDER</h1> +<p>Rsync always sorts the specified filenames into its internal transfer list. +This handles the merging together of the contents of identically named +directories, makes it easy to remove duplicate filenames, and may confuse +someone when the files are transferred in a different order than what was given +on the command-line.</p> +<p>If you need a particular file to be transferred prior to another, either +separate the files into different rsync calls, or consider using +<code>--delay-updates</code> (which doesn't affect the sorted transfer order, but does +make the final file-updating phase happen much more rapidly).</p> +<h1>EXAMPLES</h1> +<p>Here are some examples of how I use rsync.</p> +<p>To backup my wife's home directory, which consists of large MS Word files and +mail folders, I use a cron job that runs</p> +<blockquote> +<pre><code>rsync -Cavz . arvidsjaur:backup +</code></pre> +</blockquote> +<p>each night over a PPP connection to a duplicate directory on my machine +"arvidsjaur".</p> +<p>To synchronize my samba source trees I use the following Makefile targets:</p> +<blockquote> +<pre><code>get: + rsync -avuzb --exclude '*~' samba:samba/ . +put: + rsync -Cavuzb . samba:samba/ +sync: get put +</code></pre> +</blockquote> +<p>This allows me to sync with a CVS directory at the other end of the connection. +I then do CVS operations on the remote machine, which saves a lot of time as +the remote CVS protocol isn't very efficient.</p> +<p>I mirror a directory between my "old" and "new" ftp sites with the command:</p> +<blockquote> +<pre><code>rsync -az -e ssh --delete ~ftp/pub/samba nimbus:"~ftp/pub/tridge" +</code></pre> +</blockquote> +<p>This is launched from cron every few hours.</p> +<h1>OPTION SUMMARY</h1> +<p>Here is a short summary of the options available in rsync. Please refer to the +detailed description below for a complete description.</p> +<pre><code>--verbose, -v increase verbosity +--info=FLAGS fine-grained informational verbosity +--debug=FLAGS fine-grained debug verbosity +--msgs2stderr output messages directly to stderr +--quiet, -q suppress non-error messages +--no-motd suppress daemon-mode MOTD +--checksum, -c skip based on checksum, not mod-time & size +--archive, -a archive mode; equals -rlptgoD (no -H,-A,-X) +--no-OPTION turn off an implied OPTION (e.g. --no-D) +--recursive, -r recurse into directories +--relative, -R use relative path names +--no-implied-dirs don't send implied dirs with --relative +--backup, -b make backups (see --suffix & --backup-dir) +--backup-dir=DIR make backups into hierarchy based in DIR +--suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) +--update, -u skip files that are newer on the receiver +--inplace update destination files in-place +--append append data onto shorter files +--append-verify --append w/old data in file checksum +--dirs, -d transfer directories without recursing +--links, -l copy symlinks as symlinks +--copy-links, -L transform symlink into referent file/dir +--copy-unsafe-links only "unsafe" symlinks are transformed +--safe-links ignore symlinks that point outside the tree +--munge-links munge symlinks to make them safe & unusable +--copy-dirlinks, -k transform symlink to dir into referent dir +--keep-dirlinks, -K treat symlinked dir on receiver as dir +--hard-links, -H preserve hard links +--perms, -p preserve permissions +--executability, -E preserve executability +--chmod=CHMOD affect file and/or directory permissions +--acls, -A preserve ACLs (implies --perms) +--xattrs, -X preserve extended attributes +--owner, -o preserve owner (super-user only) +--group, -g preserve group +--devices preserve device files (super-user only) +--specials preserve special files +-D same as --devices --specials +--times, -t preserve modification times +--atimes, -U preserve access (use) times +--open-noatime avoid changing the atime on opened files +--omit-dir-times, -O omit directories from --times +--omit-link-times, -J omit symlinks from --times +--super receiver attempts super-user activities +--fake-super store/recover privileged attrs using xattrs +--sparse, -S turn sequences of nulls into sparse blocks +--preallocate allocate dest files before writing them +--write-devices write to devices as files (implies --inplace) +--dry-run, -n perform a trial run with no changes made +--whole-file, -W copy files whole (w/o delta-xfer algorithm) +--checksum-choice=STR choose the checksum algorithms +--one-file-system, -x don't cross filesystem boundaries +--block-size=SIZE, -B force a fixed checksum block-size +--rsh=COMMAND, -e specify the remote shell to use +--rsync-path=PROGRAM specify the rsync to run on remote machine +--existing skip creating new files on receiver +--ignore-existing skip updating files that exist on receiver +--remove-source-files sender removes synchronized files (non-dir) +--del an alias for --delete-during +--delete delete extraneous files from dest dirs +--delete-before receiver deletes before xfer, not during +--delete-during receiver deletes during the transfer +--delete-delay find deletions during, delete after +--delete-after receiver deletes after transfer, not during +--delete-excluded also delete excluded files from dest dirs +--ignore-missing-args ignore missing source args without error +--delete-missing-args delete missing source args from destination +--ignore-errors delete even if there are I/O errors +--force force deletion of dirs even if not empty +--max-delete=NUM don't delete more than NUM files +--max-size=SIZE don't transfer any file larger than SIZE +--min-size=SIZE don't transfer any file smaller than SIZE +--partial keep partially transferred files +--partial-dir=DIR put a partially transferred file into DIR +--delay-updates put all updated files into place at end +--prune-empty-dirs, -m prune empty directory chains from file-list +--numeric-ids don't map uid/gid values by user/group name +--usermap=STRING custom username mapping +--groupmap=STRING custom groupname mapping +--chown=USER:GROUP simple username/groupname mapping +--timeout=SECONDS set I/O timeout in seconds +--contimeout=SECONDS set daemon connection timeout in seconds +--ignore-times, -I don't skip files that match size and time +--size-only skip files that match in size +--modify-window=NUM, -@ set the accuracy for mod-time comparisons +--temp-dir=DIR, -T create temporary files in directory DIR +--fuzzy, -y find similar file for basis if no dest file +--compare-dest=DIR also compare destination files relative to DIR +--copy-dest=DIR ... and include copies of unchanged files +--link-dest=DIR hardlink to files in DIR when unchanged +--compress, -z compress file data during the transfer +--compress-level=NUM explicitly set compression level +--skip-compress=LIST skip compressing files with suffix in LIST +--cvs-exclude, -C auto-ignore files in the same way CVS does +--filter=RULE, -f add a file-filtering RULE +-F same as --filter='dir-merge /.rsync-filter' + repeated: --filter='- .rsync-filter' +--exclude=PATTERN exclude files matching PATTERN +--exclude-from=FILE read exclude patterns from FILE +--include=PATTERN don't exclude files matching PATTERN +--include-from=FILE read include patterns from FILE +--files-from=FILE read list of source-file names from FILE +--from0, -0 all *-from/filter files are delimited by 0s +--protect-args, -s no space-splitting; wildcard chars only +--copy-as=USER[:GROUP] specify user & optional group for the copy +--address=ADDRESS bind address for outgoing socket to daemon +--port=PORT specify double-colon alternate port number +--sockopts=OPTIONS specify custom TCP options +--blocking-io use blocking I/O for the remote shell +--outbuf=N|L|B set out buffering to None, Line, or Block +--stats give some file-transfer stats +--8-bit-output, -8 leave high-bit chars unescaped in output +--human-readable, -h output numbers in a human-readable format +--progress show progress during transfer +-P same as --partial --progress +--itemize-changes, -i output a change-summary for all updates +--remote-option=OPT, -M send OPTION to the remote side only +--out-format=FORMAT output updates using the specified FORMAT +--log-file=FILE log what we're doing to the specified FILE +--log-file-format=FMT log updates using the specified FMT +--password-file=FILE read daemon-access password from FILE +--list-only list the files instead of copying them +--bwlimit=RATE limit socket I/O bandwidth +--write-batch=FILE write a batched update to FILE +--only-write-batch=FILE like --write-batch but w/o updating dest +--read-batch=FILE read a batched update from FILE +--protocol=NUM force an older protocol version to be used +--iconv=CONVERT_SPEC request charset conversion of filenames +--checksum-seed=NUM set block/file checksum seed (advanced) +--ipv4, -4 prefer IPv4 +--ipv6, -6 prefer IPv6 +--version, -V print the version + other info and exit +--help, -h (*) show this help (* -h is help only on its own) +</code></pre> +<p>Rsync can also be run as a daemon, in which case the following options are +accepted:</p> +<pre><code>--daemon run as an rsync daemon +--address=ADDRESS bind to the specified address +--bwlimit=RATE limit socket I/O bandwidth +--config=FILE specify alternate rsyncd.conf file +--dparam=OVERRIDE, -M override global daemon config parameter +--no-detach do not detach from the parent +--port=PORT listen on alternate port number +--log-file=FILE override the "log file" setting +--log-file-format=FMT override the "log format" setting +--sockopts=OPTIONS specify custom TCP options +--verbose, -v increase verbosity +--ipv4, -4 prefer IPv4 +--ipv6, -6 prefer IPv6 +--help, -h show this help (when used with --daemon) +</code></pre> +<h1>OPTIONS</h1> +<p>Rsync accepts both long (double-dash + word) and short (single-dash + letter) +options. The full list of the available options are described below. If an +option can be specified in more than one way, the choices are comma-separated. +Some options only have a long variant, not a short. If the option takes a +parameter, the parameter is only listed after the long variant, even though it +must also be specified for the short. When specifying a parameter, you can +either use the form <code>--option=param</code> or replace the '=' with whitespace. The +parameter may need to be quoted in some manner for it to survive the shell's +command-line parsing. Keep in mind that a leading tilde (<code>~</code>) in a filename is +substituted by your shell, so <code>--option=~/foo</code> will not change the tilde into +your home directory (remove the '=' for that).</p> +<dl> + +<dt><code>--help</code>, <code>-h</code> <code>(*)</code></dt><dd> +<p>Print a short help page describing the options available in rsync and exit. +(*) The <code>-h</code> short option will only invoke <code>--help</code> when used without other +options since it normally means <code>--human-readable</code>.</p> +</dd> + +<dt><code>--version</code>, <code>-V</code></dt><dd> +<p>Print the rsync version plus other info and exit.</p> +<p>The output includes the default list of checksum algorithms, the default +list of compression algorithms, a list of compiled-in capabilities, a link +to the rsync web site, and some license/copyright info.</p> +</dd> + +<dt><code>--verbose</code>, <code>-v</code></dt><dd> +<p>This option increases the amount of information you are given during the +transfer. By default, rsync works silently. A single <code>-v</code> will give you +information about what files are being transferred and a brief summary at +the end. Two <code>-v</code> options will give you information on what files are +being skipped and slightly more information at the end. More than two <code>-v</code> +options should only be used if you are debugging rsync.</p> +<p>In a modern rsync, the <code>-v</code> option is equivalent to the setting of groups +of <code>--info</code> and <code>--debug</code> options. You can choose to use these newer +options in addition to, or in place of using <code>--verbose</code>, as any +fine-grained settings override the implied settings of <code>-v</code>. Both <code>--info</code> +and <code>--debug</code> have a way to ask for help that tells you exactly what flags +are set for each increase in verbosity.</p> +<p>However, do keep in mind that a daemon's "<code>max verbosity</code>" setting will limit +how high of a level the various individual flags can be set on the daemon +side. For instance, if the max is 2, then any info and/or debug flag that +is set to a higher value than what would be set by <code>-vv</code> will be downgraded +to the <code>-vv</code> level in the daemon's logging.</p> +</dd> + +<dt><code>--info=FLAGS</code></dt><dd> +<p>This option lets you have fine-grained control over the information output +you want to see. An individual flag name may be followed by a level +number, with 0 meaning to silence that output, 1 being the default output +level, and higher numbers increasing the output of that flag (for those +that support higher levels). Use <code>--info=help</code> to see all the available +flag names, what they output, and what flag names are added for each +increase in the verbose level. Some examples:</p> +<blockquote> +<pre><code>rsync -a --info=progress2 src/ dest/ +rsync -avv --info=stats2,misc1,flist0 src/ dest/ +</code></pre> +</blockquote> +<p>Note that <code>--info=name</code>'s output is affected by the <code>--out-format</code> and +<code>--itemize-changes</code> (<code>-i</code>) options. See those options for more information +on what is output and when.</p> +<p>This option was added to 3.1.0, so an older rsync on the server side might +reject your attempts at fine-grained control (if one or more flags needed +to be send to the server and the server was too old to understand them). +See also the "<code>max verbosity</code>" caveat above when dealing with a daemon.</p> +</dd> + +<dt><code>--debug=FLAGS</code></dt><dd> +<p>This option lets you have fine-grained control over the debug output you +want to see. An individual flag name may be followed by a level number, +with 0 meaning to silence that output, 1 being the default output level, +and higher numbers increasing the output of that flag (for those that +support higher levels). Use <code>--debug=help</code> to see all the available flag +names, what they output, and what flag names are added for each increase in +the verbose level. Some examples:</p> +<blockquote> +<pre><code>rsync -avvv --debug=none src/ dest/ +rsync -avA --del --debug=del2,acl src/ dest/ +</code></pre> +</blockquote> +<p>Note that some debug messages will only be output when <code>--msgs2stderr</code> is +specified, especially those pertaining to I/O and buffer debugging.</p> +<p>Beginning in 3.2.0, this option is no longer auto-forwared to the server +side in order to allow you to specify different debug values for each side +of the transfer, as well as to specify a new debug option that is only +present in one of the rsync versions. If you want to duplicate the same +option on both sides, using brace expansion is an easy way to save you some +typing. This works in zsh and bash:</p> +<blockquote> +<pre><code>rsync -aiv {-M,}--debug=del2 src/ dest/ +</code></pre> +</blockquote> +</dd> + +<dt><code>--msgs2stderr</code></dt><dd> +<p>This option changes rsync to send all its output directly to stderr rather +than to send messages to the client side via the protocol. The protocol +allows rsync to output normal messages via stdout and errors via stderr, +but it can delay messages behind a slew of data.</p> +<p>One case where this is helpful is when sending really large files, since +errors that happen on a remote receiver tend to get delayed until after the +file's data is fully sent. It is also helpful for debugging, since it +helps to avoid overpopulating the protocol data with extra message data.</p> +<p>The option does not affect the remote side of a transfer without using +<code>--remote-option</code> -⁠-⁠ e.g. <code>-M--msgs2stderr</code> or <code>{-M,}--msgs2stderr</code>.</p> +<p>Also keep in mind that connecting to a normal (non-remote-shell) daemon +does not have a stderr channel to send messages back to the client side, so +a modern rsync only allows the option on a remote-shell-run daemon.</p> +<p>This option has the side-effect of making stderr output get line-buffered +so that the merging of the output of 3 programs happens in a more readable +manner.</p> +</dd> + +<dt><code>--quiet</code>, <code>-q</code></dt><dd> +<p>This option decreases the amount of information you are given during the +transfer, notably suppressing information messages from the remote server. +This option is useful when invoking rsync from cron.</p> +</dd> + +<dt><code>--no-motd</code></dt><dd> +<p>This option affects the information that is output by the client at the +start of a daemon transfer. This suppresses the message-of-the-day (MOTD) +text, but it also affects the list of modules that the daemon sends in +response to the "rsync host::" request (due to a limitation in the rsync +protocol), so omit this option if you want to request the list of modules +from the daemon.</p> +</dd> + +<dt><code>--ignore-times</code>, <code>-I</code></dt><dd> +<p>Normally rsync will skip any files that are already the same size and have +the same modification timestamp. This option turns off this "quick check" +behavior, causing all files to be updated.</p> +</dd> + +<dt><code>--size-only</code></dt><dd> +<p>This modifies rsync's "quick check" algorithm for finding files that need +to be transferred, changing it from the default of transferring files with +either a changed size or a changed last-modified time to just looking for +files that have changed in size. This is useful when starting to use rsync +after using another mirroring system which may not preserve timestamps +exactly.</p> +</dd> + +<dt><code>--modify-window</code>, <code>-@</code></dt><dd> +<p>When comparing two timestamps, rsync treats the timestamps as being equal +if they differ by no more than the modify-window value. The default is 0, +which matches just integer seconds. If you specify a negative value (and +the receiver is at least version 3.1.3) then nanoseconds will also be taken +into account. Specifying 1 is useful for copies to/from MS Windows FAT +filesystems, because FAT represents times with a 2-second resolution +(allowing times to differ from the original by up to 1 second).</p> +<p>If you want all your transfers to default to comparing nanoseconds, you can +create a <code>~/.popt</code> file and put these lines in it:</p> +<blockquote> +<pre><code>rsync alias -a -a@-1 +rsync alias -t -t@-1 +</code></pre> +</blockquote> +<p>With that as the default, you'd need to specify <code>--modify-window=0</code> (aka +<code>-@0</code>) to override it and ignore nanoseconds, e.g. if you're copying +between ext3 and ext4, or if the receiving rsync is older than 3.1.3.</p> +</dd> + +<dt><code>--checksum</code>, <code>-c</code></dt><dd> +<p>This changes the way rsync checks if the files have been changed and are in +need of a transfer. Without this option, rsync uses a "quick check" that +(by default) checks if each file's size and time of last modification match +between the sender and receiver. This option changes this to compare a +128-bit checksum for each file that has a matching size. Generating the +checksums means that both sides will expend a lot of disk I/O reading all +the data in the files in the transfer, so this can slow things down +significantly (and this is prior to any reading that will be done to +transfer changed files)</p> +<p>The sending side generates its checksums while it is doing the file-system +scan that builds the list of the available files. The receiver generates +its checksums when it is scanning for changed files, and will checksum any +file that has the same size as the corresponding sender's file: files with +either a changed size or a changed checksum are selected for transfer.</p> +<p>Note that rsync always verifies that each <u>transferred</u> file was correctly +reconstructed on the receiving side by checking a whole-file checksum that +is generated as the file is transferred, but that automatic +after-the-transfer verification has nothing to do with this option's +before-the-transfer "Does this file need to be updated?" check.</p> +<p>The checksum used is auto-negotiated between the client and the server, but +can be overridden using either the <code>--checksum-choice</code> option or an +environment variable that is discussed in that option's section.</p> +</dd> + +<dt><code>--archive</code>, <code>-a</code></dt><dd> +<p>This is equivalent to <code>-rlptgoD</code>. It is a quick way of saying you want +recursion and want to preserve almost everything (with <code>-H</code> being a notable +omission). The only exception to the above equivalence is when +<code>--files-from</code> is specified, in which case <code>-r</code> is not implied.</p> +<p>Note that <code>-a</code> <strong>does not preserve hardlinks</strong>, because finding +multiply-linked files is expensive. You must separately specify <code>-H</code>.</p> +</dd> + +<dt><code>--no-OPTION</code></dt><dd> +<p>You may turn off one or more implied options by prefixing the option name +with "no-". Not all options may be prefixed with a "no-": only options that +are implied by other options (e.g. <code>--no-D</code>, <code>--no-perms</code>) or have +different defaults in various circumstances (e.g. <code>--no-whole-file</code>, +<code>--no-blocking-io</code>, <code>--no-dirs</code>). You may specify either the short or the +long option name after the "no-" prefix (e.g. <code>--no-R</code> is the same as +<code>--no-relative</code>).</p> +<p>For example: if you want to use <code>-a</code> (<code>--archive</code>) but don't want <code>-o</code> +(<code>--owner</code>), instead of converting <code>-a</code> into <code>-rlptgD</code>, you could specify +<code>-a --no-o</code> (or <code>-a --no-owner</code>).</p> +<p>The order of the options is important: if you specify <code>--no-r -a</code>, the +<code>-r</code> option would end up being turned on, the opposite of <code>-a --no-r</code>. +Note also that the side-effects of the <code>--files-from</code> option are NOT +positional, as it affects the default state of several options and slightly +changes the meaning of <code>-a</code> (see the <code>--files-from</code> option for more +details).</p> +</dd> + +<dt><code>--recursive</code>, <code>-r</code></dt><dd> +<p>This tells rsync to copy directories recursively. See also <code>--dirs</code> (<code>-d</code>).</p> +<p>Beginning with rsync 3.0.0, the recursive algorithm used is now an +incremental scan that uses much less memory than before and begins the +transfer after the scanning of the first few directories have been +completed. This incremental scan only affects our recursion algorithm, and +does not change a non-recursive transfer. It is also only possible when +both ends of the transfer are at least version 3.0.0.</p> +<p>Some options require rsync to know the full file list, so these options +disable the incremental recursion mode. These include: <code>--delete-before</code>, +<code>--delete-after</code>, <code>--prune-empty-dirs</code>, and <code>--delay-updates</code>. Because of +this, the default delete mode when you specify <code>--delete</code> is now +<code>--delete-during</code> when both ends of the connection are at least 3.0.0 (use +<code>--del</code> or <code>--delete-during</code> to request this improved deletion mode +explicitly). See also the <code>--delete-delay</code> option that is a better choice +than using <code>--delete-after</code>.</p> +<p>Incremental recursion can be disabled using the <code>--no-inc-recursive</code> option +or its shorter <code>--no-i-r</code> alias.</p> +</dd> + +<dt><code>--relative</code>, <code>-R</code></dt><dd> +<p>Use relative paths. This means that the full path names specified on the +command line are sent to the server rather than just the last parts of the +filenames. This is particularly useful when you want to send several +different directories at the same time. For example, if you used this +command:</p> +<blockquote> +<pre><code>rsync -av /foo/bar/baz.c remote:/tmp/ +</code></pre> +</blockquote> +<p>would create a file named baz.c in /tmp/ on the remote machine. If instead +you used</p> +<blockquote> +<pre><code>rsync -avR /foo/bar/baz.c remote:/tmp/ +</code></pre> +</blockquote> +<p>then a file named /tmp/foo/bar/baz.c would be created on the remote +machine, preserving its full path. These extra path elements are called +"implied directories" (i.e. the "foo" and the "foo/bar" directories in the +above example).</p> +<p>Beginning with rsync 3.0.0, rsync always sends these implied directories as +real directories in the file list, even if a path element is really a +symlink on the sending side. This prevents some really unexpected behaviors +when copying the full path of a file that you didn't realize had a symlink +in its path. If you want to duplicate a server-side symlink, include both +the symlink via its path, and referent directory via its real path. If +you're dealing with an older rsync on the sending side, you may need to use +the <code>--no-implied-dirs</code> option.</p> +<p>It is also possible to limit the amount of path information that is sent as +implied directories for each path you specify. With a modern rsync on the +sending side (beginning with 2.6.7), you can insert a dot and a slash into +the source path, like this:</p> +<blockquote> +<pre><code>rsync -avR /foo/./bar/baz.c remote:/tmp/ +</code></pre> +</blockquote> +<p>That would create /tmp/bar/baz.c on the remote machine. (Note that the dot +must be followed by a slash, so "/foo/." would not be abbreviated.) For +older rsync versions, you would need to use a chdir to limit the source +path. For example, when pushing files:</p> +<blockquote> +<pre><code>(cd /foo; rsync -avR bar/baz.c remote:/tmp/) +</code></pre> +</blockquote> +<p>(Note that the parens put the two commands into a sub-shell, so that the +"cd" command doesn't remain in effect for future commands.) If you're +pulling files from an older rsync, use this idiom (but only for a +non-daemon transfer):</p> +<blockquote> +<pre><code>rsync -avR --rsync-path="cd /foo; rsync" \ + remote:bar/baz.c /tmp/ +</code></pre> +</blockquote> +</dd> + +<dt><code>--no-implied-dirs</code></dt><dd> +<p>This option affects the default behavior of the <code>--relative</code> option. When +it is specified, the attributes of the implied directories from the source +names are not included in the transfer. This means that the corresponding +path elements on the destination system are left unchanged if they exist, +and any missing implied directories are created with default attributes. +This even allows these implied path elements to have big differences, such +as being a symlink to a directory on the receiving side.</p> +<p>For instance, if a command-line arg or a files-from entry told rsync to +transfer the file "path/foo/file", the directories "path" and "path/foo" +are implied when <code>--relative</code> is used. If "path/foo" is a symlink to "bar" +on the destination system, the receiving rsync would ordinarily delete +"path/foo", recreate it as a directory, and receive the file into the new +directory. With <code>--no-implied-dirs</code>, the receiving rsync updates +"path/foo/file" using the existing path elements, which means that the file +ends up being created in "path/bar". Another way to accomplish this link +preservation is to use the <code>--keep-dirlinks</code> option (which will also affect +symlinks to directories in the rest of the transfer).</p> +<p>When pulling files from an rsync older than 3.0.0, you may need to use this +option if the sending side has a symlink in the path you request and you +wish the implied directories to be transferred as normal directories.</p> +</dd> + +<dt><code>--backup</code>, <code>-b</code></dt><dd> +<p>With this option, preexisting destination files are renamed as each file is +transferred or deleted. You can control where the backup file goes and +what (if any) suffix gets appended using the <code>--backup-dir</code> and <code>--suffix</code> +options.</p> +<p>Note that if you don't specify <code>--backup-dir</code>, (1) the <code>--omit-dir-times</code> +option will be forced on, and (2) if <code>--delete</code> is also in effect (without +<code>--delete-excluded</code>), rsync will add a "protect" filter-rule for the backup +suffix to the end of all your existing excludes (e.g. <code>-f "P *~"</code>). This +will prevent previously backed-up files from being deleted. Note that if +you are supplying your own filter rules, you may need to manually insert +your own exclude/protect rule somewhere higher up in the list so that it +has a high enough priority to be effective (e.g., if your rules specify a +trailing inclusion/exclusion of <code>*</code>, the auto-added rule would never be +reached).</p> +</dd> + +<dt><code>--backup-dir=DIR</code></dt><dd> +<p>In combination with the <code>--backup</code> option, this tells rsync to store all +backups in the specified directory on the receiving side. This can be used +for incremental backups. You can additionally specify a backup suffix +using the <code>--suffix</code> option (otherwise the files backed up in the specified +directory will keep their original filenames).</p> +<p>Note that if you specify a relative path, the backup directory will be +relative to the destination directory, so you probably want to specify +either an absolute path or a path that starts with "../". If an rsync +daemon is the receiver, the backup dir cannot go outside the module's path +hierarchy, so take extra care not to delete it or copy into it.</p> +</dd> + +<dt><code>--suffix=SUFFIX</code></dt><dd> +<p>This option allows you to override the default backup suffix used with the +<code>--backup</code> (<code>-b</code>) option. The default suffix is a <code>~</code> if no <code>--backup-dir</code> +was specified, otherwise it is an empty string.</p> +</dd> + +<dt><code>--update</code>, <code>-u</code></dt><dd> +<p>This forces rsync to skip any files which exist on the destination and have +a modified time that is newer than the source file. (If an existing +destination file has a modification time equal to the source file's, it +will be updated if the sizes are different.)</p> +<p>Note that this does not affect the copying of dirs, symlinks, or other +special files. Also, a difference of file format between the sender and +receiver is always considered to be important enough for an update, no +matter what date is on the objects. In other words, if the source has a +directory where the destination has a file, the transfer would occur +regardless of the timestamps.</p> +<p>This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. +It just limits the files that the receiver requests to be transferred.</p> +</dd> + +<dt><code>--inplace</code></dt><dd> +<p>This option changes how rsync transfers a file when its data needs to be +updated: instead of the default method of creating a new copy of the file +and moving it into place when it is complete, rsync instead writes the +updated data directly to the destination file.</p> +<p>This has several effects:</p> +<ul> +<li>Hard links are not broken. This means the new data will be visible +through other hard links to the destination file. Moreover, attempts to +copy differing source files onto a multiply-linked destination file will +result in a "tug of war" with the destination data changing back and +forth.</li> +<li>In-use binaries cannot be updated (either the OS will prevent this from +happening, or binaries that attempt to swap-in their data will misbehave +or crash).</li> +<li>The file's data will be in an inconsistent state during the transfer and +will be left that way if the transfer is interrupted or if an update +fails.</li> +<li>A file that rsync cannot write to cannot be updated. While a super user +can update any file, a normal user needs to be granted write permission +for the open of the file for writing to be successful.</li> +<li>The efficiency of rsync's delta-transfer algorithm may be reduced if some +data in the destination file is overwritten before it can be copied to a +position later in the file. This does not apply if you use <code>--backup</code>, +since rsync is smart enough to use the backup file as the basis file for +the transfer.</li> +</ul> +<p>WARNING: you should not use this option to update files that are being +accessed by others, so be careful when choosing to use this for a copy.</p> +<p>This option is useful for transferring large files with block-based changes +or appended data, and also on systems that are disk bound, not network +bound. It can also help keep a copy-on-write filesystem snapshot from +diverging the entire contents of a file that only has minor changes.</p> +<p>The option implies <code>--partial</code> (since an interrupted transfer does not +delete the file), but conflicts with <code>--partial-dir</code> and <code>--delay-updates</code>. +Prior to rsync 2.6.4 <code>--inplace</code> was also incompatible with +<code>--compare-dest</code> and <code>--link-dest</code>.</p> +</dd> + +<dt><code>--append</code></dt><dd> +<p>This causes rsync to update a file by appending data onto the end of the +file, which presumes that the data that already exists on the receiving +side is identical with the start of the file on the sending side. If a +file needs to be transferred and its size on the receiver is the same or +longer than the size on the sender, the file is skipped. This does not +interfere with the updating of a file's non-content attributes (e.g. +permissions, ownership, etc.) when the file does not need to be +transferred, nor does it affect the updating of any non-regular files. +Implies <code>--inplace</code>.</p> +<p>The use of <code>--append</code> can be dangerous if you aren't 100% sure that the +files that are longer have only grown by the appending of data onto the +end. You should thus use include/exclude/filter rules to ensure that such +a transfer is only affecting files that you know to be growing via appended +data.</p> +</dd> + +<dt><code>--append-verify</code></dt><dd> +<p>This works just like the <code>--append</code> option, but the existing data on the +receiving side is included in the full-file checksum verification step, +which will cause a file to be resent if the final verification step fails +(rsync uses a normal, non-appending <code>--inplace</code> transfer for the resend). +It otherwise has the exact same caveats for files that have not grown +larger, so don't use this for a general copy.</p> +<p>Note: prior to rsync 3.0.0, the <code>--append</code> option worked like +<code>--append-verify</code>, so if you are interacting with an older rsync (or the +transfer is using a protocol prior to 30), specifying either append option +will initiate an <code>--append-verify</code> transfer.</p> +</dd> + +<dt><code>--dirs</code>, <code>-d</code></dt><dd> +<p>Tell the sending side to include any directories that are encountered. +Unlike <code>--recursive</code>, a directory's contents are not copied unless the +directory name specified is "." or ends with a trailing slash (e.g. ".", +"dir/.", "dir/", etc.). Without this option or the <code>--recursive</code> option, +rsync will skip all directories it encounters (and output a message to that +effect for each one). If you specify both <code>--dirs</code> and <code>--recursive</code>, +<code>--recursive</code> takes precedence.</p> +<p>The <code>--dirs</code> option is implied by the <code>--files-from</code> option or the +<code>--list-only</code> option (including an implied <code>--list-only</code> usage) if +<code>--recursive</code> wasn't specified (so that directories are seen in the +listing). Specify <code>--no-dirs</code> (or <code>--no-d</code>) if you want to turn this off.</p> +<p>There is also a backward-compatibility helper option, <code>--old-dirs</code> (or +<code>--old-d</code>) that tells rsync to use a hack of <code>-r --exclude='/*/*'</code> to get +an older rsync to list a single directory without recursing.</p> +</dd> + +<dt><code>--links</code>, <code>-l</code></dt><dd> +<p>When symlinks are encountered, recreate the symlink on the destination.</p> +</dd> + +<dt><code>--copy-links</code>, <code>-L</code></dt><dd> +<p>When symlinks are encountered, the item that they point to (the referent) +is copied, rather than the symlink. In older versions of rsync, this +option also had the side-effect of telling the receiving side to follow +symlinks, such as symlinks to directories. In a modern rsync such as this +one, you'll need to specify <code>--keep-dirlinks</code> (<code>-K</code>) to get this extra +behavior. The only exception is when sending files to an rsync that is too +old to understand <code>-K</code> -⁠-⁠ in that case, the <code>-L</code> option will still have the +side-effect of <code>-K</code> on that older receiving rsync.</p> +</dd> + +<dt><code>--copy-unsafe-links</code></dt><dd> +<p>This tells rsync to copy the referent of symbolic links that point outside +the copied tree. Absolute symlinks are also treated like ordinary files, +and so are any symlinks in the source path itself when <code>--relative</code> is +used. This option has no additional effect if <code>--copy-links</code> was also +specified.</p> +<p>Note that the cut-off point is the top of the transfer, which is the part +of the path that rsync isn't mentioning in the verbose output. If you copy +"/src/subdir" to "/dest/" then the "subdir" directory is a name inside the +transfer tree, not the top of the transfer (which is /src) so it is legal +for created relative symlinks to refer to other names inside the /src and +/dest directories. If you instead copy "/src/subdir/" (with a trailing +slash) to "/dest/subdir" that would not allow symlinks to any files outside +of "subdir".</p> +</dd> + +<dt><code>--safe-links</code></dt><dd> +<p>This tells rsync to ignore any symbolic links which point outside the +copied tree. All absolute symlinks are also ignored. Using this option in +conjunction with <code>--relative</code> may give unexpected results.</p> +</dd> + +<dt><code>--munge-links</code></dt><dd> +<p>This option tells rsync to (1) modify all symlinks on the receiving side in +a way that makes them unusable but recoverable (see below), or (2) to +unmunge symlinks on the sending side that had been stored in a munged +state. This is useful if you don't quite trust the source of the data to +not try to slip in a symlink to a unexpected place.</p> +<p>The way rsync disables the use of symlinks is to prefix each one with the +string "/rsyncd-munged/". This prevents the links from being used as long +as that directory does not exist. When this option is enabled, rsync will +refuse to run if that path is a directory or a symlink to a directory.</p> +<p>The option only affects the client side of the transfer, so if you need it +to affect the server, specify it via <code>--remote-option</code>. (Note that in a +local transfer, the client side is the sender.)</p> +<p>This option has no affect on a daemon, since the daemon configures whether +it wants munged symlinks via its "<code>munge symlinks</code>" parameter. See also the +"munge-symlinks" perl script in the support directory of the source code.</p> +</dd> + +<dt><code>--copy-dirlinks</code>, <code>-k</code></dt><dd> +<p>This option causes the sending side to treat a symlink to a directory as +though it were a real directory. This is useful if you don't want symlinks +to non-directories to be affected, as they would be using <code>--copy-links</code>.</p> +<p>Without this option, if the sending side has replaced a directory with a +symlink to a directory, the receiving side will delete anything that is in +the way of the new symlink, including a directory hierarchy (as long as +<code>--force</code> or <code>--delete</code> is in effect).</p> +<p>See also <code>--keep-dirlinks</code> for an analogous option for the receiving side.</p> +<p><code>--copy-dirlinks</code> applies to all symlinks to directories in the source. If +you want to follow only a few specified symlinks, a trick you can use is to +pass them as additional source args with a trailing slash, using +<code>--relative</code> to make the paths match up right. For example:</p> +<blockquote> +<pre><code>rsync -r --relative src/./ src/./follow-me/ dest/ +</code></pre> +</blockquote> +<p>This works because rsync calls <strong>lstat</strong>(2) on the source arg as given, and +the trailing slash makes <strong>lstat</strong>(2) follow the symlink, giving rise to a +directory in the file-list which overrides the symlink found during the +scan of "src/./".</p> +</dd> + +<dt><code>--keep-dirlinks</code>, <code>-K</code></dt><dd> +<p>This option causes the receiving side to treat a symlink to a directory as +though it were a real directory, but only if it matches a real directory +from the sender. Without this option, the receiver's symlink would be +deleted and replaced with a real directory.</p> +<p>For example, suppose you transfer a directory "foo" that contains a file +"file", but "foo" is a symlink to directory "bar" on the receiver. Without +<code>--keep-dirlinks</code>, the receiver deletes symlink "foo", recreates it as a +directory, and receives the file into the new directory. With +<code>--keep-dirlinks</code>, the receiver keeps the symlink and "file" ends up in +"bar".</p> +<p>One note of caution: if you use <code>--keep-dirlinks</code>, you must trust all the +symlinks in the copy! If it is possible for an untrusted user to create +their own symlink to any directory, the user could then (on a subsequent +copy) replace the symlink with a real directory and affect the content of +whatever directory the symlink references. For backup copies, you are +better off using something like a bind mount instead of a symlink to modify +your receiving hierarchy.</p> +<p>See also <code>--copy-dirlinks</code> for an analogous option for the sending side.</p> +</dd> + +<dt><code>--hard-links</code>, <code>-H</code></dt><dd> +<p>This tells rsync to look for hard-linked files in the source and link +together the corresponding files on the destination. Without this option, +hard-linked files in the source are treated as though they were separate +files.</p> +<p>This option does NOT necessarily ensure that the pattern of hard links on +the destination exactly matches that on the source. Cases in which the +destination may end up with extra hard links include the following:</p> +<ul> +<li>If the destination contains extraneous hard-links (more linking than what +is present in the source file list), the copying algorithm will not break +them explicitly. However, if one or more of the paths have content +differences, the normal file-update process will break those extra links +(unless you are using the <code>--inplace</code> option).</li> +<li>If you specify a <code>--link-dest</code> directory that contains hard links, the +linking of the destination files against the <code>--link-dest</code> files can +cause some paths in the destination to become linked together due to the +<code>--link-dest</code> associations.</li> +</ul> +<p>Note that rsync can only detect hard links between files that are inside +the transfer set. If rsync updates a file that has extra hard-link +connections to files outside the transfer, that linkage will be broken. If +you are tempted to use the <code>--inplace</code> option to avoid this breakage, be +very careful that you know how your files are being updated so that you are +certain that no unintended changes happen due to lingering hard links (and +see the <code>--inplace</code> option for more caveats).</p> +<p>If incremental recursion is active (see <code>--recursive</code>), rsync may transfer +a missing hard-linked file before it finds that another link for that +contents exists elsewhere in the hierarchy. This does not affect the +accuracy of the transfer (i.e. which files are hard-linked together), just +its efficiency (i.e. copying the data for a new, early copy of a +hard-linked file that could have been found later in the transfer in +another member of the hard-linked set of files). One way to avoid this +inefficiency is to disable incremental recursion using the +<code>--no-inc-recursive</code> option.</p> +</dd> + +<dt><code>--perms</code>, <code>-p</code></dt><dd> +<p>This option causes the receiving rsync to set the destination permissions +to be the same as the source permissions. (See also the <code>--chmod</code> option +for a way to modify what rsync considers to be the source permissions.)</p> +<p>When this option is <u>off</u>, permissions are set as follows:</p> +<ul> +<li>Existing files (including updated files) retain their existing +permissions, though the <code>--executability</code> option might change just the +execute permission for the file.</li> +<li>New files get their "normal" permission bits set to the source file's +permissions masked with the receiving directory's default permissions +(either the receiving process's umask, or the permissions specified via +the destination directory's default ACL), and their special permission +bits disabled except in the case where a new directory inherits a setgid +bit from its parent directory.</li> +</ul> +<p>Thus, when <code>--perms</code> and <code>--executability</code> are both disabled, rsync's +behavior is the same as that of other file-copy utilities, such as <strong>cp</strong>(1) +and <strong>tar</strong>(1).</p> +<p>In summary: to give destination files (both old and new) the source +permissions, use <code>--perms</code>. To give new files the destination-default +permissions (while leaving existing files unchanged), make sure that the +<code>--perms</code> option is off and use <code>--chmod=ugo=rwX</code> (which ensures that all +non-masked bits get enabled). If you'd care to make this latter behavior +easier to type, you could define a popt alias for it, such as putting this +line in the file <code>~/.popt</code> (the following defines the <code>-Z</code> option, and +includes <code>--no-g</code> to use the default group of the destination dir):</p> +<blockquote> +<pre><code> rsync alias -Z --no-p --no-g --chmod=ugo=rwX +</code></pre> +</blockquote> +<p>You could then use this new option in a command such as this one:</p> +<blockquote> +<pre><code> rsync -avZ src/ dest/ +</code></pre> +</blockquote> +<p>(Caveat: make sure that <code>-a</code> does not follow <code>-Z</code>, or it will re-enable the +two <code>--no-*</code> options mentioned above.)</p> +<p>The preservation of the destination's setgid bit on newly-created +directories when <code>--perms</code> is off was added in rsync 2.6.7. Older rsync +versions erroneously preserved the three special permission bits for +newly-created files when <code>--perms</code> was off, while overriding the +destination's setgid bit setting on a newly-created directory. Default ACL +observance was added to the ACL patch for rsync 2.6.7, so older (or +non-ACL-enabled) rsyncs use the umask even if default ACLs are present. +(Keep in mind that it is the version of the receiving rsync that affects +these behaviors.)</p> +</dd> + +<dt><code>--executability</code>, <code>-E</code></dt><dd> +<p>This option causes rsync to preserve the executability (or +non-executability) of regular files when <code>--perms</code> is not enabled. A +regular file is considered to be executable if at least one 'x' is turned +on in its permissions. When an existing destination file's executability +differs from that of the corresponding source file, rsync modifies the +destination file's permissions as follows:</p> +<ul> +<li>To make a file non-executable, rsync turns off all its 'x' permissions.</li> +<li>To make a file executable, rsync turns on each 'x' permission that has a +corresponding 'r' permission enabled.</li> +</ul> +<p>If <code>--perms</code> is enabled, this option is ignored.</p> +</dd> + +<dt><code>--acls</code>, <code>-A</code></dt><dd> +<p>This option causes rsync to update the destination ACLs to be the same as +the source ACLs. The option also implies <code>--perms</code>.</p> +<p>The source and destination systems must have compatible ACL entries for +this option to work properly. See the <code>--fake-super</code> option for a way to +backup and restore ACLs that are not compatible.</p> +</dd> + +<dt><code>--xattrs</code>, <code>-X</code></dt><dd> +<p>This option causes rsync to update the destination extended attributes to +be the same as the source ones.</p> +<p>For systems that support extended-attribute namespaces, a copy being done +by a super-user copies all namespaces except system.*. A normal user only +copies the user.* namespace. To be able to backup and restore non-user +namespaces as a normal user, see the <code>--fake-super</code> option.</p> +<p>The above name filtering can be overridden by using one or more filter +options with the <strong>x</strong> modifier. When you specify an xattr-affecting +filter rule, rsync requires that you do your own system/user filtering, as +well as any additional filtering for what xattr names are copied and what +names are allowed to be deleted. For example, to skip the system +namespace, you could specify:</p> +<blockquote> +<pre><code>--filter='-x system.*' +</code></pre> +</blockquote> +<p>To skip all namespaces except the user namespace, you could specify a +negated-user match:</p> +<blockquote> +<pre><code>--filter='-x! user.*' +</code></pre> +</blockquote> +<p>To prevent any attributes from being deleted, you could specify a +receiver-only rule that excludes all names:</p> +<blockquote> +<pre><code>--filter='-xr *' +</code></pre> +</blockquote> +<p>Note that the <code>-X</code> option does not copy rsync's special xattr values (e.g. +those used by <code>--fake-super</code>) unless you repeat the option (e.g. <code>-XX</code>). +This "copy all xattrs" mode cannot be used with <code>--fake-super</code>.</p> +</dd> + +<dt><code>--chmod</code></dt><dd> +<p>This option tells rsync to apply one or more comma-separated "chmod" modes +to the permission of the files in the transfer. The resulting value is +treated as though it were the permissions that the sending side supplied +for the file, which means that this option can seem to have no effect on +existing files if <code>--perms</code> is not enabled.</p> +<p>In addition to the normal parsing rules specified in the <strong>chmod</strong>(1) +manpage, you can specify an item that should only apply to a directory by +prefixing it with a 'D', or specify an item that should only apply to a +file by prefixing it with a 'F'. For example, the following will ensure +that all directories get marked set-gid, that no files are other-writable, +that both are user-writable and group-writable, and that both have +consistent executability across all bits:</p> +<blockquote> +<pre><code>--chmod=Dg+s,ug+w,Fo-w,+X +</code></pre> +</blockquote> +<p>Using octal mode numbers is also allowed:</p> +<blockquote> +<pre><code>--chmod=D2775,F664 +</code></pre> +</blockquote> +<p>It is also legal to specify multiple <code>--chmod</code> options, as each additional +option is just appended to the list of changes to make.</p> +<p>See the <code>--perms</code> and <code>--executability</code> options for how the resulting +permission value can be applied to the files in the transfer.</p> +</dd> + +<dt><code>--owner</code>, <code>-o</code></dt><dd> +<p>This option causes rsync to set the owner of the destination file to be the +same as the source file, but only if the receiving rsync is being run as +the super-user (see also the <code>--super</code> and <code>--fake-super</code> options). Without +this option, the owner of new and/or transferred files are set to the +invoking user on the receiving side.</p> +<p>The preservation of ownership will associate matching names by default, but +may fall back to using the ID number in some circumstances (see also the +<code>--numeric-ids</code> option for a full discussion).</p> +</dd> + +<dt><code>--group</code>, <code>-g</code></dt><dd> +<p>This option causes rsync to set the group of the destination file to be the +same as the source file. If the receiving program is not running as the +super-user (or if <code>--no-super</code> was specified), only groups that the +invoking user on the receiving side is a member of will be preserved. +Without this option, the group is set to the default group of the invoking +user on the receiving side.</p> +<p>The preservation of group information will associate matching names by +default, but may fall back to using the ID number in some circumstances +(see also the <code>--numeric-ids</code> option for a full discussion).</p> +</dd> + +<dt><code>--devices</code></dt><dd> +<p>This option causes rsync to transfer character and block device files to +the remote system to recreate these devices. This option has no effect if +the receiving rsync is not run as the super-user (see also the <code>--super</code> +and <code>--fake-super</code> options).</p> +</dd> + +<dt><code>--specials</code></dt><dd> +<p>This option causes rsync to transfer special files such as named sockets +and fifos.</p> +</dd> + +<dt><code>-D</code></dt><dd> +<p>The <code>-D</code> option is equivalent to <code>--devices --specials</code>.</p> +</dd> + +<dt><code>--write-devices</code></dt><dd> +<p>This tells rsync to treat a device on the receiving side as a regular file, +allowing the writing of file data into a device.</p> +<p>This option implies the <code>--inplace</code> option.</p> +<p>Be careful using this, as you should know what devices are present on the +receiving side of the transfer, especially if running rsync as root.</p> +<p>This option is refused by an rsync daemon.</p> +</dd> + +<dt><code>--times</code>, <code>-t</code></dt><dd> +<p>This tells rsync to transfer modification times along with the files and +update them on the remote system. Note that if this option is not used, +the optimization that excludes files that have not been modified cannot be +effective; in other words, a missing <code>-t</code> or <code>-a</code> will cause the next +transfer to behave as if it used <code>-I</code>, causing all files to be updated +(though rsync's delta-transfer algorithm will make the update fairly +efficient if the files haven't actually changed, you're much better off +using <code>-t</code>).</p> +</dd> + +<dt><code>--atimes</code>, <code>-U</code></dt><dd> +<p>This tells rsync to set the access (use) times of the destination files to +the same value as the source files.</p> +<p>If repeated, it also sets the <code>--open-noatime</code> option, which can help you +to make the sending and receiving systems have the same access times on the +transferred files without needing to run rsync an extra time after a file +is transferred.</p> +<p>Note that some older rsync versions (prior to 3.2.0) may have been built +with a pre-release <code>--atimes</code> patch that does not imply <code>--open-noatime</code> +when this option is repeated.</p> +</dd> + +<dt><code>--open-noatime</code></dt><dd> +<p>This tells rsync to open files with the O_NOATIME flag (on systems that +support it) to avoid changing the access time of the files that are being +transferred. If your OS does not support the O_NOATIME flag then rsync +will silently ignore this option. Note also that some filesystems are +mounted to avoid updating the atime on read access even without the +O_NOATIME flag being set.</p> +</dd> + +<dt><code>--omit-dir-times</code>, <code>-O</code></dt><dd> +<p>This tells rsync to omit directories when it is preserving modification +times (see <code>--times</code>). If NFS is sharing the directories on the receiving +side, it is a good idea to use <code>-O</code>. This option is inferred if you use +<code>--backup</code> without <code>--backup-dir</code>.</p> +<p>This option also has the side-effect of avoiding early creation of +directories in incremental recursion copies. The default <code>--inc-recursive</code> +copying normally does an early-create pass of all the sub-directories in a +parent directory in order for it to be able to then set the modify time of +the parent directory right away (without having to delay that until a bunch +of recursive copying has finished). This early-create idiom is not +necessary if directory modify times are not being preserved, so it is +skipped. Since early-create directories don't have accurate mode, mtime, +or ownership, the use of this option can help when someone wants to avoid +these partially-finished directories.</p> +</dd> + +<dt><code>--omit-link-times</code>, <code>-J</code></dt><dd> +<p>This tells rsync to omit symlinks when it is preserving modification times +(see <code>--times</code>).</p> +</dd> + +<dt><code>--super</code></dt><dd> +<p>This tells the receiving side to attempt super-user activities even if the +receiving rsync wasn't run by the super-user. These activities include: +preserving users via the <code>--owner</code> option, preserving all groups (not just +the current user's groups) via the <code>--groups</code> option, and copying devices +via the <code>--devices</code> option. This is useful for systems that allow such +activities without being the super-user, and also for ensuring that you +will get errors if the receiving side isn't being run as the super-user. +To turn off super-user activities, the super-user can use <code>--no-super</code>.</p> +</dd> + +<dt><code>--fake-super</code></dt><dd> +<p>When this option is enabled, rsync simulates super-user activities by +saving/restoring the privileged attributes via special extended attributes +that are attached to each file (as needed). This includes the file's owner +and group (if it is not the default), the file's device info (device & +special files are created as empty text files), and any permission bits +that we won't allow to be set on the real file (e.g. the real file gets +u-s,g-s,o-t for safety) or that would limit the owner's access (since the +real super-user can always access/change a file, the files we create can +always be accessed/changed by the creating user). This option also handles +ACLs (if <code>--acls</code> was specified) and non-user extended attributes (if +<code>--xattrs</code> was specified).</p> +<p>This is a good way to backup data without using a super-user, and to store +ACLs from incompatible systems.</p> +<p>The <code>--fake-super</code> option only affects the side where the option is used. +To affect the remote side of a remote-shell connection, use the +<code>--remote-option</code> (<code>-M</code>) option:</p> +<blockquote> +<pre><code>rsync -av -M--fake-super /src/ host:/dest/ +</code></pre> +</blockquote> +<p>For a local copy, this option affects both the source and the destination. +If you wish a local copy to enable this option just for the destination +files, specify <code>-M--fake-super</code>. If you wish a local copy to enable this +option just for the source files, combine <code>--fake-super</code> with <code>-M--super</code>.</p> +<p>This option is overridden by both <code>--super</code> and <code>--no-super</code>.</p> +<p>See also the "<code>fake super</code>" setting in the daemon's rsyncd.conf file.</p> +</dd> + +<dt><code>--sparse</code>, <code>-S</code></dt><dd> +<p>Try to handle sparse files efficiently so they take up less space on the +destination. If combined with <code>--inplace</code> the file created might not end +up with sparse blocks with some combinations of kernel version and/or +filesystem type. If <code>--whole-file</code> is in effect (e.g. for a local copy) +then it will always work because rsync truncates the file prior to writing +out the updated version.</p> +<p>Note that versions of rsync older than 3.1.3 will reject the combination of +<code>--sparse</code> and <code>--inplace</code>.</p> +</dd> + +<dt><code>--preallocate</code></dt><dd> +<p>This tells the receiver to allocate each destination file to its eventual +size before writing data to the file. Rsync will only use the real +filesystem-level preallocation support provided by Linux's <strong>fallocate</strong>(2) +system call or Cygwin's <strong>posix_fallocate</strong>(3), not the slow glibc +implementation that writes a null byte into each block.</p> +<p>Without this option, larger files may not be entirely contiguous on the +filesystem, but with this option rsync will probably copy more slowly. If +the destination is not an extent-supporting filesystem (such as ext4, xfs, +NTFS, etc.), this option may have no positive effect at all.</p> +<p>If combined with <code>--sparse</code>, the file will only have sparse blocks (as +opposed to allocated sequences of null bytes) if the kernel version and +filesystem type support creating holes in the allocated data.</p> +</dd> + +<dt><code>--dry-run</code>, <code>-n</code></dt><dd> +<p>This makes rsync perform a trial run that doesn't make any changes (and +produces mostly the same output as a real run). It is most commonly used +in combination with the <code>--verbose</code>, <code>-v</code> and/or <code>--itemize-changes</code>, <code>-i</code> +options to see what an rsync command is going to do before one actually +runs it.</p> +<p>The output of <code>--itemize-changes</code> is supposed to be exactly the same on a +dry run and a subsequent real run (barring intentional trickery and system +call failures); if it isn't, that's a bug. Other output should be mostly +unchanged, but may differ in some areas. Notably, a dry run does not send +the actual data for file transfers, so <code>--progress</code> has no effect, the +"bytes sent", "bytes received", "literal data", and "matched data" +statistics are too small, and the "speedup" value is equivalent to a run +where no file transfers were needed.</p> +</dd> + +<dt><code>--whole-file</code>, <code>-W</code></dt><dd> +<p>This option disables rsync's delta-transfer algorithm, which causes all +transferred files to be sent whole. The transfer may be faster if this +option is used when the bandwidth between the source and destination +machines is higher than the bandwidth to disk (especially when the "disk" +is actually a networked filesystem). This is the default when both the +source and destination are specified as local paths, but only if no +batch-writing option is in effect.</p> +</dd> + +<dt><code>--checksum-choice=STR</code>, <code>--cc=STR</code></dt><dd> +<p>This option overrides the checksum algorithms. If one algorithm name is +specified, it is used for both the transfer checksums and (assuming +<code>--checksum</code> is specified) the pre-transfer checksums. If two +comma-separated names are supplied, the first name affects the transfer +checksums, and the second name affects the pre-transfer checksums (<code>-c</code>).</p> +<p>The checksum options that you may be able to use are:</p> +<ul> +<li><code>auto</code> (the default)</li> +<li><code>xxh64</code> (aka xxhash)</li> +<li><code>md5</code></li> +<li><code>md4</code></li> +<li><code>none</code></li> +</ul> +<p>Run <code>rsync -V</code> to see the default checksum list compiled into your version.</p> +<p>If "none" is specified for the first (or only) name, the <code>--whole-file</code> +option is forced on and no checksum verification is performed on the +transferred data. If "none" is specified for the second (or only) name, +the <code>--checksum</code> option cannot be used.</p> +<p>The "auto" option is the default, where rsync bases its algorithm choice on +a negotiation between the client and the server as follows:</p> +<p>If both the client and the server are at least version 3.2.0, they will +exchange a list of checksum names and choose the first one in the list that +they have in common. This typically means that they will choose xxh64 if +they both support it and fall back to MD5. If one side of the transfer is +not new enough to support this checksum negotiation, then a value is chosen +based on the protocol version (which chooses between MD5 and various +flavors of MD4 based on protocol age).</p> +<p>You can also override the checksum using the RSYNC_CHECKSUM_LIST +environment variable by setting it to a space-separated list of checksum +names that you consider acceptable. If no common checksum is found, the +client exits with an error. This method does not allow you to specify the +transfer checksum separately from the pre-transfer checksum, and it ignores +"auto" and all unknown checksum names. If the remote rsync is not new +enough to handle a checksum negotiation list, the list is silently ignored +unless it contains the string "FAIL".</p> +<p>The use of the <code>--checksum-choice</code> option overrides this environment list.</p> +</dd> + +<dt><code>--one-file-system</code>, <code>-x</code></dt><dd> +<p>This tells rsync to avoid crossing a filesystem boundary when recursing. +This does not limit the user's ability to specify items to copy from +multiple filesystems, just rsync's recursion through the hierarchy of each +directory that the user specified, and also the analogous recursion on the +receiving side during deletion. Also keep in mind that rsync treats a +"bind" mount to the same device as being on the same filesystem.</p> +<p>If this option is repeated, rsync omits all mount-point directories from +the copy. Otherwise, it includes an empty directory at each mount-point it +encounters (using the attributes of the mounted directory because those of +the underlying mount-point directory are inaccessible).</p> +<p>If rsync has been told to collapse symlinks (via <code>--copy-links</code> or +<code>--copy-unsafe-links</code>), a symlink to a directory on another device is +treated like a mount-point. Symlinks to non-directories are unaffected by +this option.</p> +</dd> + +<dt><code>--existing</code>, <code>--ignore-non-existing</code></dt><dd> +<p>This tells rsync to skip creating files (including directories) that do not +exist yet on the destination. If this option is combined with the +<code>--ignore-existing</code> option, no files will be updated (which can be useful +if all you want to do is delete extraneous files).</p> +<p>This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. +It just limits the files that the receiver requests to be transferred.</p> +</dd> + +<dt><code>--ignore-existing</code></dt><dd> +<p>This tells rsync to skip updating files that already exist on the +destination (this does <u>not</u> ignore existing directories, or nothing would +get done). See also <code>--existing</code>.</p> +<p>This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. +It just limits the files that the receiver requests to be transferred.</p> +<p>This option can be useful for those doing backups using the <code>--link-dest</code> +option when they need to continue a backup run that got interrupted. Since +a <code>--link-dest</code> run is copied into a new directory hierarchy (when it is +used properly), using <code>--ignore-existing</code> will ensure that the +already-handled files don't get tweaked (which avoids a change in +permissions on the hard-linked files). This does mean that this option is +only looking at the existing files in the destination hierarchy itself.</p> +</dd> + +<dt><code>--remove-source-files</code></dt><dd> +<p>This tells rsync to remove from the sending side the files (meaning +non-directories) that are a part of the transfer and have been successfully +duplicated on the receiving side.</p> +<p>Note that you should only use this option on source files that are +quiescent. If you are using this to move files that show up in a +particular directory over to another host, make sure that the finished +files get renamed into the source directory, not directly written into it, +so that rsync can't possibly transfer a file that is not yet fully written. +If you can't first write the files into a different directory, you should +use a naming idiom that lets rsync avoid transferring files that are not +yet finished (e.g. name the file "foo.new" when it is written, rename it to +"foo" when it is done, and then use the option <code>--exclude='*.new'</code> for the +rsync transfer).</p> +<p>Starting with 3.1.0, rsync will skip the sender-side removal (and output an +error) if the file's size or modify time has not stayed unchanged.</p> +</dd> + +<dt><code>--delete</code></dt><dd> +<p>This tells rsync to delete extraneous files from the receiving side (ones +that aren't on the sending side), but only for the directories that are +being synchronized. You must have asked rsync to send the whole directory +(e.g. "<code>dir</code>" or "<code>dir/</code>") without using a wildcard for the directory's +contents (e.g. "<code>dir/*</code>") since the wildcard is expanded by the shell and +rsync thus gets a request to transfer individual files, not the files' +parent directory. Files that are excluded from the transfer are also +excluded from being deleted unless you use the <code>--delete-excluded</code> option +or mark the rules as only matching on the sending side (see the +include/exclude modifiers in the FILTER RULES section).</p> +<p>Prior to rsync 2.6.7, this option would have no effect unless <code>--recursive</code> +was enabled. Beginning with 2.6.7, deletions will also occur when <code>--dirs</code> +(<code>-d</code>) is enabled, but only for directories whose contents are being +copied.</p> +<p>This option can be dangerous if used incorrectly! It is a very good idea to +first try a run using the <code>--dry-run</code> option (<code>-n</code>) to see what files are +going to be deleted.</p> +<p>If the sending side detects any I/O errors, then the deletion of any files +at the destination will be automatically disabled. This is to prevent +temporary filesystem failures (such as NFS errors) on the sending side from +causing a massive deletion of files on the destination. You can override +this with the <code>--ignore-errors</code> option.</p> +<p>The <code>--delete</code> option may be combined with one of the -⁠-⁠delete-WHEN options +without conflict, as well as <code>--delete-excluded</code>. However, if none of the +<code>--delete-WHEN</code> options are specified, rsync will choose the +<code>--delete-during</code> algorithm when talking to rsync 3.0.0 or newer, and the +<code>--delete-before</code> algorithm when talking to an older rsync. See also +<code>--delete-delay</code> and <code>--delete-after</code>.</p> +</dd> + +<dt><code>--delete-before</code></dt><dd> +<p>Request that the file-deletions on the receiving side be done before the +transfer starts. See <code>--delete</code> (which is implied) for more details on +file-deletion.</p> +<p>Deleting before the transfer is helpful if the filesystem is tight for +space and removing extraneous files would help to make the transfer +possible. However, it does introduce a delay before the start of the +transfer, and this delay might cause the transfer to timeout (if +<code>--timeout</code> was specified). It also forces rsync to use the old, +non-incremental recursion algorithm that requires rsync to scan all the +files in the transfer into memory at once (see <code>--recursive</code>).</p> +</dd> + +<dt><code>--delete-during</code>, <code>--del</code></dt><dd> +<p>Request that the file-deletions on the receiving side be done incrementally +as the transfer happens. The per-directory delete scan is done right +before each directory is checked for updates, so it behaves like a more +efficient <code>--delete-before</code>, including doing the deletions prior to any +per-directory filter files being updated. This option was first added in +rsync version 2.6.4. See <code>--delete</code> (which is implied) for more details on +file-deletion.</p> +</dd> + +<dt><code>--delete-delay</code></dt><dd> +<p>Request that the file-deletions on the receiving side be computed during +the transfer (like <code>--delete-during</code>), and then removed after the transfer +completes. This is useful when combined with <code>--delay-updates</code> and/or +<code>--fuzzy</code>, and is more efficient than using <code>--delete-after</code> (but can +behave differently, since <code>--delete-after</code> computes the deletions in a +separate pass after all updates are done). If the number of removed files +overflows an internal buffer, a temporary file will be created on the +receiving side to hold the names (it is removed while open, so you +shouldn't see it during the transfer). If the creation of the temporary +file fails, rsync will try to fall back to using <code>--delete-after</code> (which it +cannot do if <code>--recursive</code> is doing an incremental scan). See <code>--delete</code> +(which is implied) for more details on file-deletion.</p> +</dd> + +<dt><code>--delete-after</code></dt><dd> +<p>Request that the file-deletions on the receiving side be done after the +transfer has completed. This is useful if you are sending new +per-directory merge files as a part of the transfer and you want their +exclusions to take effect for the delete phase of the current transfer. It +also forces rsync to use the old, non-incremental recursion algorithm that +requires rsync to scan all the files in the transfer into memory at once +(see <code>--recursive</code>). See <code>--delete</code> (which is implied) for more details on +file-deletion.</p> +</dd> + +<dt><code>--delete-excluded</code></dt><dd> +<p>In addition to deleting the files on the receiving side that are not on the +sending side, this tells rsync to also delete any files on the receiving +side that are excluded (see <code>--exclude</code>). See the FILTER RULES section for +a way to make individual exclusions behave this way on the receiver, and +for a way to protect files from <code>--delete-excluded</code>. See <code>--delete</code> (which +is implied) for more details on file-deletion.</p> +</dd> + +<dt><code>--ignore-missing-args</code></dt><dd> +<p>When rsync is first processing the explicitly requested source files (e.g. +command-line arguments or <code>--files-from</code> entries), it is normally an error +if the file cannot be found. This option suppresses that error, and does +not try to transfer the file. This does not affect subsequent +vanished-file errors if a file was initially found to be present and later +is no longer there.</p> +</dd> + +<dt><code>--delete-missing-args</code></dt><dd> +<p>This option takes the behavior of (the implied) <code>--ignore-missing-args</code> +option a step farther: each missing arg will become a deletion request of +the corresponding destination file on the receiving side (should it exist). +If the destination file is a non-empty directory, it will only be +successfully deleted if <code>--force</code> or <code>--delete</code> are in effect. Other than +that, this option is independent of any other type of delete processing.</p> +<p>The missing source files are represented by special file-list entries which +display as a "<code>*missing</code>" entry in the <code>--list-only</code> output.</p> +</dd> + +<dt><code>--ignore-errors</code></dt><dd> +<p>Tells <code>--delete</code> to go ahead and delete files even when there are I/O +errors.</p> +</dd> + +<dt><code>--force</code></dt><dd> +<p>This option tells rsync to delete a non-empty directory when it is to be +replaced by a non-directory. This is only relevant if deletions are not +active (see <code>--delete</code> for details).</p> +<p>Note for older rsync versions: <code>--force</code> used to still be required when +using <code>--delete-after</code>, and it used to be non-functional unless the +<code>--recursive</code> option was also enabled.</p> +</dd> + +<dt><code>--max-delete=NUM</code></dt><dd> +<p>This tells rsync not to delete more than NUM files or directories. If that +limit is exceeded, all further deletions are skipped through the end of the +transfer. At the end, rsync outputs a warning (including a count of the +skipped deletions) and exits with an error code of 25 (unless some more +important error condition also occurred).</p> +<p>Beginning with version 3.0.0, you may specify <code>--max-delete=0</code> to be warned +about any extraneous files in the destination without removing any of them. +Older clients interpreted this as "unlimited", so if you don't know what +version the client is, you can use the less obvious <code>--max-delete=-1</code> as a +backward-compatible way to specify that no deletions be allowed (though +really old versions didn't warn when the limit was exceeded).</p> +</dd> + +<dt><code>--max-size=SIZE</code></dt><dd> +<p>This tells rsync to avoid transferring any file that is larger than the +specified SIZE. The SIZE value can be suffixed with a string to indicate a +size multiplier, and may be a fractional value (e.g. <code>--max-size=1.5m</code>).</p> +<p>This option is a transfer rule, not an exclude, so it doesn't affect the +data that goes into the file-lists, and thus it doesn't affect deletions. +It just limits the files that the receiver requests to be transferred.</p> +<p>The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or +"MiB") is a mebibyte (1024*1024), and "G" (or "GiB") is a gibibyte +(1024*1024*1024). If you want the multiplier to be 1000 instead of 1024, +use "KB", "MB", or "GB". (Note: lower-case is also accepted for all +values.) Finally, if the suffix ends in either "+1" or "-⁠1", the value will +be offset by one byte in the indicated direction.</p> +<p>Examples: <code>--max-size=1.5mb-1</code> is 1499999 bytes, and <code>--max-size=2g+1</code> is +2147483649 bytes.</p> +<p>Note that rsync versions prior to 3.1.0 did not allow <code>--max-size=0</code>.</p> +</dd> + +<dt><code>--min-size=SIZE</code></dt><dd> +<p>This tells rsync to avoid transferring any file that is smaller than the +specified SIZE, which can help in not transferring small, junk files. See +the <code>--max-size</code> option for a description of SIZE and other information.</p> +<p>Note that rsync versions prior to 3.1.0 did not allow <code>--min-size=0</code>.</p> +</dd> + +<dt><code>--block-size=BLOCKSIZE</code>, <code>-B</code></dt><dd> +<p>This forces the block size used in rsync's delta-transfer algorithm to a +fixed value. It is normally selected based on the size of each file being +updated. See the technical report for details.</p> +</dd> + +<dt><code>--rsh=COMMAND</code>, <code>-e</code></dt><dd> +<p>This option allows you to choose an alternative remote shell program to use +for communication between the local and remote copies of rsync. Typically, +rsync is configured to use ssh by default, but you may prefer to use rsh on +a local network.</p> +<p>If this option is used with <code>[user@]host::module/path</code>, then the remote +shell <u>COMMAND</u> will be used to run an rsync daemon on the remote host, and +all data will be transmitted through that remote shell connection, rather +than through a direct socket connection to a running rsync daemon on the +remote host. See the section "USING RSYNC-DAEMON FEATURES VIA A +REMOTE-SHELL CONNECTION" above.</p> +<p>Beginning with rsync 3.2.0, the RSYNC_PORT environment variable will be set +when a daemon connection is being made via a remote-shell connection. It +is set to 0 if the default daemon port is being assumed, or it is set to +the value of the rsync port that was specified via either the <code>--port</code> +option or a non-empty port value in an rsync:// URL. This allows the +script to discern if a non-default port is being requested, allowing for +things such as an SSL or stunnel helper script to connect to a default or +alternate port.</p> +<p>Command-line arguments are permitted in COMMAND provided that COMMAND is +presented to rsync as a single argument. You must use spaces (not tabs or +other whitespace) to separate the command and args from each other, and you +can use single- and/or double-quotes to preserve spaces in an argument (but +not backslashes). Note that doubling a single-quote inside a single-quoted +string gives you a single-quote; likewise for double-quotes (though you +need to pay attention to which quotes your shell is parsing and which +quotes rsync is parsing). Some examples:</p> +<blockquote> +<pre><code>-e 'ssh -p 2234' +-e 'ssh -o "ProxyCommand nohup ssh firewall nc -w1 %h %p"' +</code></pre> +</blockquote> +<p>(Note that ssh users can alternately customize site-specific connect +options in their .ssh/config file.)</p> +<p>You can also choose the remote shell program using the RSYNC_RSH +environment variable, which accepts the same range of values as <code>-e</code>.</p> +<p>See also the <code>--blocking-io</code> option which is affected by this option.</p> +</dd> + +<dt><code>--rsync-path=PROGRAM</code></dt><dd> +<p>Use this to specify what program is to be run on the remote machine to +start-up rsync. Often used when rsync is not in the default remote-shell's +path (e.g. <code>--rsync-path=/usr/local/bin/rsync</code>). Note that PROGRAM is run +with the help of a shell, so it can be any program, script, or command +sequence you'd care to run, so long as it does not corrupt the standard-in +& standard-out that rsync is using to communicate.</p> +<p>One tricky example is to set a different default directory on the remote +machine for use with the <code>--relative</code> option. For instance:</p> +<blockquote> +<pre><code>rsync -avR --rsync-path="cd /a/b && rsync" host:c/d /e/ +</code></pre> +</blockquote> +</dd> + +<dt><code>--remote-option=OPTION</code>, <code>-M</code></dt><dd> +<p>This option is used for more advanced situations where you want certain +effects to be limited to one side of the transfer only. For instance, if +you want to pass <code>--log-file=FILE</code> and <code>--fake-super</code> to the remote system, +specify it like this:</p> +<blockquote> +<pre><code>rsync -av -M --log-file=foo -M--fake-super src/ dest/ +</code></pre> +</blockquote> +<p>If you want to have an option affect only the local side of a transfer when +it normally affects both sides, send its negation to the remote side. Like +this:</p> +<blockquote> +<pre><code>rsync -av -x -M--no-x src/ dest/ +</code></pre> +</blockquote> +<p>Be cautious using this, as it is possible to toggle an option that will +cause rsync to have a different idea about what data to expect next over +the socket, and that will make it fail in a cryptic fashion.</p> +<p>Note that it is best to use a separate <code>--remote-option</code> for each option +you want to pass. This makes your usage compatible with the +<code>--protect-args</code> option. If that option is off, any spaces in your remote +options will be split by the remote shell unless you take steps to protect +them.</p> +<p>When performing a local transfer, the "local" side is the sender and the +"remote" side is the receiver.</p> +<p>Note some versions of the popt option-parsing library have a bug in them +that prevents you from using an adjacent arg with an equal in it next to a +short option letter (e.g. <code>-M--log-file=/tmp/foo</code>). If this bug affects +your version of popt, you can use the version of popt that is included with +rsync.</p> +</dd> + +<dt><code>--cvs-exclude</code>, <code>-C</code></dt><dd> +<p>This is a useful shorthand for excluding a broad range of files that you +often don't want to transfer between systems. It uses a similar algorithm +to CVS to determine if a file should be ignored.</p> +<p>The exclude list is initialized to exclude the following items (these +initial items are marked as perishable -⁠-⁠ see the FILTER RULES section):</p> +<blockquote> +<p><code>RCS</code> +<code>SCCS</code> +<code>CVS</code> +<code>CVS.adm</code> +<code>RCSLOG</code> +<code>cvslog.*</code> +<code>tags</code> +<code>TAGS</code> +<code>.make.state</code> +<code>.nse_depinfo</code> +<code>*~</code> +<code>#*</code> +<code>.#*</code> +<code>,*</code> +<code>_$*</code> +<code>*$</code> +<code>*.old</code> +<code>*.bak</code> +<code>*.BAK</code> +<code>*.orig</code> +<code>*.rej</code> +<code>.del-*</code> +<code>*.a</code> +<code>*.olb</code> +<code>*.o</code> +<code>*.obj</code> +<code>*.so</code> +<code>*.exe</code> +<code>*.Z</code> +<code>*.elc</code> +<code>*.ln</code> +<code>core</code> +<code>.svn/</code> +<code>.git/</code> +<code>.hg/</code> +<code>.bzr/</code></p> +</blockquote> +<p>then, files listed in a $HOME/.cvsignore are added to the list and any +files listed in the CVSIGNORE environment variable (all cvsignore names are +delimited by whitespace).</p> +<p>Finally, any file is ignored if it is in the same directory as a .cvsignore +file and matches one of the patterns listed therein. Unlike rsync's +filter/exclude files, these patterns are split on whitespace. See the +<strong>cvs</strong>(1) manual for more information.</p> +<p>If you're combining <code>-C</code> with your own <code>--filter</code> rules, you should note +that these CVS excludes are appended at the end of your own rules, +regardless of where the <code>-C</code> was placed on the command-line. This makes +them a lower priority than any rules you specified explicitly. If you want +to control where these CVS excludes get inserted into your filter rules, +you should omit the <code>-C</code> as a command-line option and use a combination of +<code>--filter=:C</code> and <code>--filter=-C</code> (either on your command-line or by putting +the ":C" and "-⁠C" rules into a filter file with your other rules). The +first option turns on the per-directory scanning for the .cvsignore file. +The second option does a one-time import of the CVS excludes mentioned +above.</p> +</dd> + +<dt><code>--filter=RULE</code>, <code>-f</code></dt><dd> +<p>This option allows you to add rules to selectively exclude certain files +from the list of files to be transferred. This is most useful in +combination with a recursive transfer.</p> +<p>You may use as many <code>--filter</code> options on the command line as you like to +build up the list of files to exclude. If the filter contains whitespace, +be sure to quote it so that the shell gives the rule to rsync as a single +argument. The text below also mentions that you can use an underscore to +replace the space that separates a rule from its arg.</p> +<p>See the FILTER RULES section for detailed information on this option.</p> +</dd> + +<dt><code>-F</code></dt><dd> +<p>The <code>-F</code> option is a shorthand for adding two <code>--filter</code> rules to your +command. The first time it is used is a shorthand for this rule:</p> +<blockquote> +<pre><code>--filter='dir-merge /.rsync-filter' +</code></pre> +</blockquote> +<p>This tells rsync to look for per-directory .rsync-filter files that have +been sprinkled through the hierarchy and use their rules to filter the +files in the transfer. If <code>-F</code> is repeated, it is a shorthand for this +rule:</p> +<blockquote> +<pre><code>--filter='exclude .rsync-filter' +</code></pre> +</blockquote> +<p>This filters out the .rsync-filter files themselves from the transfer.</p> +<p>See the FILTER RULES section for detailed information on how these options +work.</p> +</dd> + +<dt><code>--exclude=PATTERN</code></dt><dd> +<p>This option is a simplified form of the <code>--filter</code> option that defaults to +an exclude rule and does not allow the full rule-parsing syntax of normal +filter rules.</p> +<p>See the FILTER RULES section for detailed information on this option.</p> +</dd> + +<dt><code>--exclude-from=FILE</code></dt><dd> +<p>This option is related to the <code>--exclude</code> option, but it specifies a FILE +that contains exclude patterns (one per line). Blank lines in the file and +lines starting with '<code>;</code>' or '<code>#</code>' are ignored. If <u>FILE</u> is '<code>-</code>', the +list will be read from standard input.</p> +</dd> + +<dt><code>--include=PATTERN</code></dt><dd> +<p>This option is a simplified form of the <code>--filter</code> option that defaults to +an include rule and does not allow the full rule-parsing syntax of normal +filter rules.</p> +<p>See the FILTER RULES section for detailed information on this option.</p> +</dd> + +<dt><code>--include-from=FILE</code></dt><dd> +<p>This option is related to the <code>--include</code> option, but it specifies a FILE +that contains include patterns (one per line). Blank lines in the file and +lines starting with '<code>;</code>' or '<code>#</code>' are ignored. If <u>FILE</u> is '<code>-</code>', the +list will be read from standard input.</p> +</dd> + +<dt><code>--files-from=FILE</code></dt><dd> +<p>Using this option allows you to specify the exact list of files to transfer +(as read from the specified FILE or '<code>-</code>' for standard input). It also +tweaks the default behavior of rsync to make transferring just the +specified files and directories easier:</p> +<ul> +<li>The <code>--relative</code> (<code>-R</code>) option is implied, which preserves the path +information that is specified for each item in the file (use +<code>--no-relative</code> or <code>--no-R</code> if you want to turn that off).</li> +<li>The <code>--dirs</code> (<code>-d</code>) option is implied, which will create directories +specified in the list on the destination rather than noisily skipping +them (use <code>--no-dirs</code> or <code>--no-d</code> if you want to turn that off).</li> +<li>The <code>--archive</code> (<code>-a</code>) option's behavior does not imply <code>--recursive</code> +(<code>-r</code>), so specify it explicitly, if you want it.</li> +<li>These side-effects change the default state of rsync, so the position of +the <code>--files-from</code> option on the command-line has no bearing on how other +options are parsed (e.g. <code>-a</code> works the same before or after +<code>--files-from</code>, as does <code>--no-R</code> and all other options).</li> +</ul> +<p>The filenames that are read from the FILE are all relative to the source +dir -⁠-⁠ any leading slashes are removed and no ".." references are allowed +to go higher than the source dir. For example, take this command:</p> +<blockquote> +<pre><code>rsync -a --files-from=/tmp/foo /usr remote:/backup +</code></pre> +</blockquote> +<p>If /tmp/foo contains the string "bin" (or even "/bin"), the /usr/bin +directory will be created as /backup/bin on the remote host. If it +contains "bin/" (note the trailing slash), the immediate contents of the +directory would also be sent (without needing to be explicitly mentioned in +the file -⁠-⁠ this began in version 2.6.4). In both cases, if the <code>-r</code> +option was enabled, that dir's entire hierarchy would also be transferred +(keep in mind that <code>-r</code> needs to be specified explicitly with +<code>--files-from</code>, since it is not implied by <code>-a</code>). Also note that the +effect of the (enabled by default) <code>--relative</code> option is to duplicate only +the path info that is read from the file -⁠-⁠ it does not force the +duplication of the source-spec path (/usr in this case).</p> +<p>In addition, the <code>--files-from</code> file can be read from the remote host +instead of the local host if you specify a "host:" in front of the file +(the host must match one end of the transfer). As a short-cut, you can +specify just a prefix of ":" to mean "use the remote end of the transfer". +For example:</p> +<blockquote> +<pre><code>rsync -a --files-from=:/path/file-list src:/ /tmp/copy +</code></pre> +</blockquote> +<p>This would copy all the files specified in the /path/file-list file that +was located on the remote "src" host.</p> +<p>If the <code>--iconv</code> and <code>--protect-args</code> options are specified and the +<code>--files-from</code> filenames are being sent from one host to another, the +filenames will be translated from the sending host's charset to the +receiving host's charset.</p> +<p>NOTE: sorting the list of files in the <code>--files-from</code> input helps rsync to +be more efficient, as it will avoid re-visiting the path elements that are +shared between adjacent entries. If the input is not sorted, some path +elements (implied directories) may end up being scanned multiple times, and +rsync will eventually unduplicate them after they get turned into file-list +elements.</p> +</dd> + +<dt><code>--from0</code>, <code>-0</code></dt><dd> +<p>This tells rsync that the rules/filenames it reads from a file are +terminated by a null ('\0') character, not a NL, CR, or CR+LF. This +affects <code>--exclude-from</code>, <code>--include-from</code>, <code>--files-from</code>, and any merged +files specified in a <code>--filter</code> rule. It does not affect <code>--cvs-exclude</code> +(since all names read from a .cvsignore file are split on whitespace).</p> +</dd> + +<dt><code>--protect-args</code>, <code>-s</code></dt><dd> +<p>This option sends all filenames and most options to the remote rsync +without allowing the remote shell to interpret them. This means that +spaces are not split in names, and any non-wildcard special characters are +not translated (such as <code>~</code>, <code>$</code>, <code>;</code>, <code>&</code>, etc.). Wildcards are expanded +on the remote host by rsync (instead of the shell doing it).</p> +<p>If you use this option with <code>--iconv</code>, the args related to the remote side +will also be translated from the local to the remote character-set. The +translation happens before wild-cards are expanded. See also the +<code>--files-from</code> option.</p> +<p>You may also control this option via the RSYNC_PROTECT_ARGS environment +variable. If this variable has a non-zero value, this option will be +enabled by default, otherwise it will be disabled by default. Either state +is overridden by a manually specified positive or negative version of this +option (note that <code>--no-s</code> and <code>--no-protect-args</code> are the negative +versions). Since this option was first introduced in 3.0.0, you'll need to +make sure it's disabled if you ever need to interact with a remote rsync +that is older than that.</p> +<p>Rsync can also be configured (at build time) to have this option enabled by +default (with is overridden by both the environment and the command-line). +This option will eventually become a new default setting at some +as-yet-undetermined point in the future.</p> +</dd> + +<dt><code>--copy-as=USER[:GROUP]</code></dt><dd> +<p>This option instructs rsync to use the USER and (if specified after a +colon) the GROUP for the copy operations. This only works if the user that +is running rsync has the ability to change users. If the group is not +specified then the user's default groups are used.</p> +<p>This option can help to reduce the risk of an rsync being run as root into +or out of a directory that might have live changes happening to it and you +want to make sure that root-level read or write actions of system files are +not possible. While you could alternatively run all of rsync as the +specified user, sometimes you need the root-level host-access credentials +to be used, so this allows rsync to drop root for the copying part of the +operation after the remote-shell or daemon connection is established.</p> +<p>The option only affects one side of the transfer unless the transfer is +local, in which case it affects both sides. Use the <code>--remote-option</code> to +affect the remote side, such as <code>-M--copy-as=joe</code>. For a local transfer, +the lsh (or lsh.sh) support file provides a local-shell helper script that +can be used to allow a "localhost:" or "lh:" host-spec to be specified +without needing to setup any remote shells, allowing you to specify remote +options that affect the side of the transfer that is using the host-spec +(and using hostname "lh" avoids the overriding of the remote directory to +the user's home dir).</p> +<p>For example, the following rsync writes the local files as user "joe":</p> +<blockquote> +<pre><code>sudo rsync -aiv --copy-as=joe host1:backups/joe/ /home/joe/ +</code></pre> +</blockquote> +<p>This makes all files owned by user "joe", limits the groups to those that +are available to that user, and makes it impossible for the joe user to do +a timed exploit of the path to induce a change to a file that the joe user +has no permissions to change.</p> +<p>The following command does a local copy into the "dest/" dir as user "joe" +(assumimg you've installed support/lsh into a dir on your $PATH):</p> +<blockquote> +<pre><code>sudo rsync -aive lsh -M--copy-as=joe src/ lh:dest/ +</code></pre> +</blockquote> +</dd> + +<dt><code>--temp-dir=DIR</code>, <code>-T</code></dt><dd> +<p>This option instructs rsync to use DIR as a scratch directory when creating +temporary copies of the files transferred on the receiving side. The +default behavior is to create each temporary file in the same directory as +the associated destination file. Beginning with rsync 3.1.1, the temp-file +names inside the specified DIR will not be prefixed with an extra dot +(though they will still have a random suffix added).</p> +<p>This option is most often used when the receiving disk partition does not +have enough free space to hold a copy of the largest file in the transfer. +In this case (i.e. when the scratch directory is on a different disk +partition), rsync will not be able to rename each received temporary file +over the top of the associated destination file, but instead must copy it +into place. Rsync does this by copying the file over the top of the +destination file, which means that the destination file will contain +truncated data during this copy. If this were not done this way (even if +the destination file were first removed, the data locally copied to a +temporary file in the destination directory, and then renamed into place) +it would be possible for the old file to continue taking up disk space (if +someone had it open), and thus there might not be enough room to fit the +new version on the disk at the same time.</p> +<p>If you are using this option for reasons other than a shortage of disk +space, you may wish to combine it with the <code>--delay-updates</code> option, which +will ensure that all copied files get put into subdirectories in the +destination hierarchy, awaiting the end of the transfer. If you don't have +enough room to duplicate all the arriving files on the destination +partition, another way to tell rsync that you aren't overly concerned about +disk space is to use the <code>--partial-dir</code> option with a relative path; +because this tells rsync that it is OK to stash off a copy of a single file +in a subdir in the destination hierarchy, rsync will use the partial-dir as +a staging area to bring over the copied file, and then rename it into place +from there. (Specifying a <code>--partial-dir</code> with an absolute path does not +have this side-effect.)</p> +</dd> + +<dt><code>--fuzzy</code>, <code>-y</code></dt><dd> +<p>This option tells rsync that it should look for a basis file for any +destination file that is missing. The current algorithm looks in the same +directory as the destination file for either a file that has an identical +size and modified-time, or a similarly-named file. If found, rsync uses +the fuzzy basis file to try to speed up the transfer.</p> +<p>If the option is repeated, the fuzzy scan will also be done in any matching +alternate destination directories that are specified via <code>--compare-dest</code>, +<code>--copy-dest</code>, or <code>--link-dest</code>.</p> +<p>Note that the use of the <code>--delete</code> option might get rid of any potential +fuzzy-match files, so either use <code>--delete-after</code> or specify some filename +exclusions if you need to prevent this.</p> +</dd> + +<dt><code>--compare-dest=DIR</code></dt><dd> +<p>This option instructs rsync to use <u>DIR</u> on the destination machine as an +additional hierarchy to compare destination files against doing transfers +(if the files are missing in the destination directory). If a file is +found in <u>DIR</u> that is identical to the sender's file, the file will NOT be +transferred to the destination directory. This is useful for creating a +sparse backup of just files that have changed from an earlier backup. This +option is typically used to copy into an empty (or newly created) +directory.</p> +<p>Beginning in version 2.6.4, multiple <code>--compare-dest</code> directories may be +provided, which will cause rsync to search the list in the order specified +for an exact match. If a match is found that differs only in attributes, a +local copy is made and the attributes updated. If a match is not found, a +basis file from one of the <u>DIRs</u> will be selected to try to speed up the +transfer.</p> +<p>If <u>DIR</u> is a relative path, it is relative to the destination directory. +See also <code>--copy-dest</code> and <code>--link-dest</code>.</p> +<p>NOTE: beginning with version 3.1.0, rsync will remove a file from a +non-empty destination hierarchy if an exact match is found in one of the +compare-dest hierarchies (making the end result more closely match a fresh +copy).</p> +</dd> + +<dt><code>--copy-dest=DIR</code></dt><dd> +<p>This option behaves like <code>--compare-dest</code>, but rsync will also copy +unchanged files found in <u>DIR</u> to the destination directory using a local +copy. This is useful for doing transfers to a new destination while +leaving existing files intact, and then doing a flash-cutover when all +files have been successfully transferred.</p> +<p>Multiple <code>--copy-dest</code> directories may be provided, which will cause rsync +to search the list in the order specified for an unchanged file. If a +match is not found, a basis file from one of the <u>DIRs</u> will be selected to +try to speed up the transfer.</p> +<p>If <u>DIR</u> is a relative path, it is relative to the destination directory. +See also <code>--compare-dest</code> and <code>--link-dest</code>.</p> +</dd> + +<dt><code>--link-dest=DIR</code></dt><dd> +<p>This option behaves like <code>--copy-dest</code>, but unchanged files are hard linked +from <u>DIR</u> to the destination directory. The files must be identical in +all preserved attributes (e.g. permissions, possibly ownership) in order +for the files to be linked together. An example:</p> +<blockquote> +<pre><code>rsync -av --link-dest=$PWD/prior_dir host:src_dir/ new_dir/ +</code></pre> +</blockquote> +<p>If file's aren't linking, double-check their attributes. Also check if +some attributes are getting forced outside of rsync's control, such a mount +option that squishes root to a single user, or mounts a removable drive +with generic ownership (such as OS X's "Ignore ownership on this volume" +option).</p> +<p>Beginning in version 2.6.4, multiple <code>--link-dest</code> directories may be +provided, which will cause rsync to search the list in the order specified +for an exact match (there is a limit of 20 such directories). If a match +is found that differs only in attributes, a local copy is made and the +attributes updated. If a match is not found, a basis file from one of the +<u>DIRs</u> will be selected to try to speed up the transfer.</p> +<p>This option works best when copying into an empty destination hierarchy, as +existing files may get their attributes tweaked, and that can affect +alternate destination files via hard-links. Also, itemizing of changes can +get a bit muddled. Note that prior to version 3.1.0, an +alternate-directory exact match would never be found (nor linked into the +destination) when a destination file already exists.</p> +<p>Note that if you combine this option with <code>--ignore-times</code>, rsync will not +link any files together because it only links identical files together as a +substitute for transferring the file, never as an additional check after +the file is updated.</p> +<p>If <u>DIR</u> is a relative path, it is relative to the destination directory. +See also <code>--compare-dest</code> and <code>--copy-dest</code>.</p> +<p>Note that rsync versions prior to 2.6.1 had a bug that could prevent +<code>--link-dest</code> from working properly for a non-super-user when <code>-o</code> was +specified (or implied by <code>-a</code>). You can work-around this bug by avoiding +the <code>-o</code> option when sending to an old rsync.</p> +</dd> + +<dt><code>--compress</code>, <code>-z</code></dt><dd> +<p>With this option, rsync compresses the file data as it is sent to the +destination machine, which reduces the amount of data being transmitted -⁠-⁠ +something that is useful over a slow connection.</p> +<p>The "zlib" compression method typically achieves better compression ratios +than can be achieved by using a compressing remote shell or a compressing +transport because it takes advantage of the implicit information in the +matching data blocks that are not explicitly sent over the connection. +This matching-data compression comes at a cost of CPU, though, and can be +disabled by using the "zlibx" compresson method instead. This can be +selected by repeating the <code>-z</code> option or specifying +<code>--compress-choice=zlibx</code>, but it only works if both sides of the transfer +are at least version 3.1.1.</p> +<p>Note that if you see an error about an option named <code>--old-compress</code> or +<code>--new-compress</code>, this is rsync trying to send the <code>--compress-choice=zlib</code> +or <code>--compress-choice=zlibx</code> option in a backward-compatible manner that +more rsync versions understand. This error indicates that the older rsync +version will not allow you to force the compression type.</p> +<p>See the <code>--skip-compress</code> option for the default list of file suffixes that +will not be compressed.</p> +</dd> + +<dt><code>--compress-choice=STR</code>, <code>--zc=STR</code></dt><dd> +<p>This option can be used to override the automatic selection of the +compression algorithm that is the default when <code>--compress</code> is used.</p> +<p>The compression options that you may be able to use are:</p> +<ul> +<li><code>zstd</code></li> +<li><code>lz4</code></li> +<li><code>zlibx</code></li> +<li><code>zlib</code></li> +<li><code>none</code></li> +</ul> +<p>Run <code>rsync -V</code> to see the compress list compiled into your version.</p> +<p>The "zlibx" algorithm is given preference over "zlib" if both sides of the +transfer are at least version 3.2.0, otherwise it will choose "zlib" unless +you override it via something like <code>-zz</code>. These 2 algorithms are the stame +except that "zlibx" does not try to include matched data that was not +transferred in the compression computations.</p> +<p>If "none" is specified, that is equivalent to using <code>--no-compress</code>.</p> +<p>This option implies <code>--compress</code> unless "none" was specified.</p> +<p>You can also override the compression negotiation using the +RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated +list of compression names that you consider acceptable. If no common +compress choice is found, the client exits with an error. It ignores +"auto" and all unknown compression names. If the remote rsync is not new +enough to handle a compression negotiation list, the list is silently +ignored unless it contains the string "FAIL".</p> +</dd> + +<dt><code>--compress-level=NUM</code></dt><dd> +<p>Explicitly set the compression level to use (see <code>--compress</code>) instead of +letting it default. If NUM is non-zero, the <code>--compress</code> option is +implied.</p> +</dd> + +<dt><code>--skip-compress=LIST</code></dt><dd> +<p>Override the list of file suffixes that will be compressed as little as +possible. Rsync sets the compression level on a per-file basis based on +the file's suffix. If the compression algorithm has an "off" level (such +as zlib/zlibx) then no compression occurs for those files. Other +algorithms have the level minimized to reduces the CPU usage as much as +possible.</p> +<p>The <strong>LIST</strong> should be one or more file suffixes (without the dot) separated +by slashes (<code>/</code>). You may specify an empty string to indicate that no files +should be skipped.</p> +<p>Simple character-class matching is supported: each must consist of a list +of letters inside the square brackets (e.g. no special classes, such as +"[:alpha:]", are supported, and '-⁠' has no special meaning).</p> +<p>The characters asterisk (<code>*</code>) and question-mark (<code>?</code>) have no special meaning.</p> +<p>Here's an example that specifies 6 suffixes to skip (since 1 of the 5 rules +matches 2 suffixes):</p> +<blockquote> +<pre><code>--skip-compress=gz/jpg/mp[34]/7z/bz2 +</code></pre> +</blockquote> +<p>The default file suffixes in the skip-compress list in this version of +rsync are:</p> +<blockquote> +<p>7z +ace +apk +avi +bz2 +deb +flac +gpg +gz +iso +jar +jpeg +jpg +lz +lz4 +lzma +lzo +mkv +mov +mp3 +mp4 +odb +odf +odg +odi +odm +odp +ods +odt +ogg +ogv +opus +otg +oth +otp +ots +ott +oxt +png +rar +rpm +rz +rzip +squashfs +sxc +sxd +sxg +sxm +sxw +tbz +tgz +tlz +txz +tzo +webm +webp +xz +z +zip +zst</p> +</blockquote> +<p>This list will be replaced by your <code>--skip-compress</code> list in all but one +situation: a copy from a daemon rsync will add your skipped suffixes to its +list of non-compressing files (and its list may be configured to a +different default).</p> +</dd> + +<dt><code>--numeric-ids</code></dt><dd> +<p>With this option rsync will transfer numeric group and user IDs rather than +using user and group names and mapping them at both ends.</p> +<p>By default rsync will use the username and groupname to determine what +ownership to give files. The special uid 0 and the special group 0 are +never mapped via user/group names even if the <code>--numeric-ids</code> option is not +specified.</p> +<p>If a user or group has no name on the source system or it has no match on +the destination system, then the numeric ID from the source system is used +instead. See also the comments on the "<code>use chroot</code>" setting in the +rsyncd.conf manpage for information on how the chroot setting affects +rsync's ability to look up the names of the users and groups and what you +can do about it.</p> +</dd> + +<dt><code>--usermap=STRING</code>, <code>--groupmap=STRING</code></dt><dd> +<p>These options allow you to specify users and groups that should be mapped +to other values by the receiving side. The <strong>STRING</strong> is one or more +<strong>FROM</strong>:<strong>TO</strong> pairs of values separated by commas. Any matching <strong>FROM</strong> +value from the sender is replaced with a <strong>TO</strong> value from the receiver. +You may specify usernames or user IDs for the <strong>FROM</strong> and <strong>TO</strong> values, +and the <strong>FROM</strong> value may also be a wild-card string, which will be +matched against the sender's names (wild-cards do NOT match against ID +numbers, though see below for why a '<code>*</code>' matches everything). You may +instead specify a range of ID numbers via an inclusive range: LOW-HIGH. +For example:</p> +<blockquote> +<pre><code>--usermap=0-99:nobody,wayne:admin,*:normal --groupmap=usr:1,1:usr +</code></pre> +</blockquote> +<p>The first match in the list is the one that is used. You should specify +all your user mappings using a single <code>--usermap</code> option, and/or all your +group mappings using a single <code>--groupmap</code> option.</p> +<p>Note that the sender's name for the 0 user and group are not transmitted to +the receiver, so you should either match these values using a 0, or use the +names in effect on the receiving side (typically "root"). All other +<strong>FROM</strong> names match those in use on the sending side. All <strong>TO</strong> names +match those in use on the receiving side.</p> +<p>Any IDs that do not have a name on the sending side are treated as having +an empty name for the purpose of matching. This allows them to be matched +via a "<code>*</code>" or using an empty name. For instance:</p> +<blockquote> +<pre><code>--usermap=:nobody --groupmap=*:nobody +</code></pre> +</blockquote> +<p>When the <code>--numeric-ids</code> option is used, the sender does not send any +names, so all the IDs are treated as having an empty name. This means that +you will need to specify numeric <strong>FROM</strong> values if you want to map these +nameless IDs to different values.</p> +<p>For the <code>--usermap</code> option to have any effect, the <code>-o</code> (<code>--owner</code>) option +must be used (or implied), and the receiver will need to be running as a +super-user (see also the <code>--fake-super</code> option). For the <code>--groupmap</code> +option to have any effect, the <code>-g</code> (<code>--groups</code>) option must be used (or +implied), and the receiver will need to have permissions to set that group.</p> +</dd> + +<dt><code>--chown=USER:GROUP</code></dt><dd> +<p>This option forces all files to be owned by USER with group GROUP. This is +a simpler interface than using <code>--usermap</code> and <code>--groupmap</code> directly, but +it is implemented using those options internally, so you cannot mix them. +If either the USER or GROUP is empty, no mapping for the omitted user/group +will occur. If GROUP is empty, the trailing colon may be omitted, but if +USER is empty, a leading colon must be supplied.</p> +<p>If you specify "<code>--chown=foo:bar</code>", this is exactly the same as specifying +"<code>--usermap=*:foo --groupmap=*:bar</code>", only easier.</p> +</dd> + +<dt><code>--timeout=TIMEOUT</code></dt><dd> +<p>This option allows you to set a maximum I/O timeout in seconds. If no data +is transferred for the specified time then rsync will exit. The default is +0, which means no timeout.</p> +</dd> + +<dt><code>--contimeout</code></dt><dd> +<p>This option allows you to set the amount of time that rsync will wait for +its connection to an rsync daemon to succeed. If the timeout is reached, +rsync exits with an error.</p> +</dd> + +<dt><code>--address</code></dt><dd> +<p>By default rsync will bind to the wildcard address when connecting to an +rsync daemon. The <code>--address</code> option allows you to specify a specific IP +address (or hostname) to bind to. See also this option in the <code>--daemon</code> +mode section.</p> +</dd> + +<dt><code>--port=PORT</code></dt><dd> +<p>This specifies an alternate TCP port number to use rather than the default +of 873. This is only needed if you are using the double-colon (::) syntax +to connect with an rsync daemon (since the URL syntax has a way to specify +the port as a part of the URL). See also this option in the <code>--daemon</code> +mode section.</p> +</dd> + +<dt><code>--sockopts</code></dt><dd> +<p>This option can provide endless fun for people who like to tune their +systems to the utmost degree. You can set all sorts of socket options +which may make transfers faster (or slower!). Read the man page for the +<code>setsockopt()</code> system call for details on some of the options you may be +able to set. By default no special socket options are set. This only +affects direct socket connections to a remote rsync daemon.</p> +<p>This option also exists in the <code>--daemon</code> mode section.</p> +</dd> + +<dt><code>--blocking-io</code></dt><dd> +<p>This tells rsync to use blocking I/O when launching a remote shell +transport. If the remote shell is either rsh or remsh, rsync defaults to +using blocking I/O, otherwise it defaults to using non-blocking I/O. (Note +that ssh prefers non-blocking I/O.)</p> +</dd> + +<dt><code>--outbuf=MODE</code></dt><dd> +<p>This sets the output buffering mode. The mode can be None (aka +Unbuffered), Line, or Block (aka Full). You may specify as little as a +single letter for the mode, and use upper or lower case.</p> +<p>The main use of this option is to change Full buffering to Line buffering +when rsync's output is going to a file or pipe.</p> +</dd> + +<dt><code>--itemize-changes</code>, <code>-i</code></dt><dd> +<p>Requests a simple itemized list of the changes that are being made to each +file, including attribute changes. This is exactly the same as specifying +<code>--out-format='%i %n%L'</code>. If you repeat the option, unchanged files will +also be output, but only if the receiving rsync is at least version 2.6.7 +(you can use <code>-vv</code> with older versions of rsync, but that also turns on the +output of other verbose messages).</p> +<p>The "%i" escape has a cryptic output that is 11 letters long. The general +format is like the string <code>YXcstpoguax</code>, where <strong>Y</strong> is replaced by the type +of update being done, <strong>X</strong> is replaced by the file-type, and the other +letters represent attributes that may be output if they are being modified.</p> +<p>The update types that replace the <strong>Y</strong> are as follows:</p> +<ul> +<li>A <code><</code> means that a file is being transferred to the remote host (sent).</li> +<li>A <code>></code> means that a file is being transferred to the local host +(received).</li> +<li>A <code>c</code> means that a local change/creation is occurring for the item (such +as the creation of a directory or the changing of a symlink, etc.).</li> +<li>A <code>h</code> means that the item is a hard link to another item (requires +<code>--hard-links</code>).</li> +<li>A <code>.</code> means that the item is not being updated (though it might have +attributes that are being modified).</li> +<li>A <code>*</code> means that the rest of the itemized-output area contains a message +(e.g. "deleting").</li> +</ul> +<p>The file-types that replace the <strong>X</strong> are: <code>f</code> for a file, a <code>d</code> for a +directory, an <code>L</code> for a symlink, a <code>D</code> for a device, and a <code>S</code> for a +special file (e.g. named sockets and fifos).</p> +<p>The other letters in the string above are the actual letters that will be +output if the associated attribute for the item is being updated or a "." +for no change. Three exceptions to this are: (1) a newly created item +replaces each letter with a "+", (2) an identical item replaces the dots +with spaces, and (3) an unknown attribute replaces each letter with a "?" +(this can happen when talking to an older rsync).</p> +<p>The attribute that is associated with each letter is as follows:</p> +<ul> +<li>A <code>c</code> means either that a regular file has a different checksum (requires +<code>--checksum</code>) or that a symlink, device, or special file has a changed +value. Note that if you are sending files to an rsync prior to 3.0.1, +this change flag will be present only for checksum-differing regular +files.</li> +<li>A <code>s</code> means the size of a regular file is different and will be updated +by the file transfer.</li> +<li>A <code>t</code> means the modification time is different and is being updated to +the sender's value (requires <code>--times</code>). An alternate value of <code>T</code> means +that the modification time will be set to the transfer time, which +happens when a file/symlink/device is updated without <code>--times</code> and when +a symlink is changed and the receiver can't set its time. (Note: when +using an rsync 3.0.0 client, you might see the <code>s</code> flag combined with <code>t</code> +instead of the proper <code>T</code> flag for this time-setting failure.)</li> +<li>A <code>p</code> means the permissions are different and are being updated to the +sender's value (requires <code>--perms</code>).</li> +<li>An <code>o</code> means the owner is different and is being updated to the sender's +value (requires <code>--owner</code> and super-user privileges).</li> +<li>A <code>g</code> means the group is different and is being updated to the sender's +value (requires <code>--group</code> and the authority to set the group).</li> +<li>A <code>u</code> means the access (use) time is different and is being updated to +the sender's value (requires <code>--atimes</code>). An alternate value of <code>U</code> +means that the access time will be set to the transfer time, which +happens when a symlink or directory is updated.</li> +<li>The <code>a</code> means that the ACL information changed.</li> +<li>The <code>x</code> means that the extended attribute information changed.</li> +</ul> +<p>One other output is possible: when deleting files, the "%i" will output the +string "<code>*deleting</code>" for each item that is being removed (assuming that you +are talking to a recent enough rsync that it logs deletions instead of +outputting them as a verbose message).</p> +</dd> + +<dt><code>--out-format=FORMAT</code></dt><dd> +<p>This allows you to specify exactly what the rsync client outputs to the +user on a per-update basis. The format is a text string containing +embedded single-character escape sequences prefixed with a percent (%) +character. A default format of "%n%L" is assumed if either <code>--info=name</code> +or <code>-v</code> is specified (this tells you just the name of the file and, if the +item is a link, where it points). For a full list of the possible escape +characters, see the "<code>log format</code>" setting in the rsyncd.conf manpage.</p> +<p>Specifying the <code>--out-format</code> option implies the <code>--info=name</code> option, +which will mention each file, dir, etc. that gets updated in a significant +way (a transferred file, a recreated symlink/device, or a touched +directory). In addition, if the itemize-changes escape (%i) is included in +the string (e.g. if the <code>--itemize-changes</code> option was used), the logging +of names increases to mention any item that is changed in any way (as long +as the receiving side is at least 2.6.4). See the <code>--itemize-changes</code> +option for a description of the output of "%i".</p> +<p>Rsync will output the out-format string prior to a file's transfer unless +one of the transfer-statistic escapes is requested, in which case the +logging is done at the end of the file's transfer. When this late logging +is in effect and <code>--progress</code> is also specified, rsync will also output the +name of the file being transferred prior to its progress information +(followed, of course, by the out-format output).</p> +</dd> + +<dt><code>--log-file=FILE</code></dt><dd> +<p>This option causes rsync to log what it is doing to a file. This is +similar to the logging that a daemon does, but can be requested for the +client side and/or the server side of a non-daemon transfer. If specified +as a client option, transfer logging will be enabled with a default format +of "%i %n%L". See the <code>--log-file-format</code> option if you wish to override +this.</p> +<p>Here's a example command that requests the remote side to log what is +happening:</p> +<blockquote> +<pre><code>rsync -av --remote-option=--log-file=/tmp/rlog src/ dest/ +</code></pre> +</blockquote> +<p>This is very useful if you need to debug why a connection is closing +unexpectedly.</p> +</dd> + +<dt><code>--log-file-format=FORMAT</code></dt><dd> +<p>This allows you to specify exactly what per-update logging is put into the +file specified by the <code>--log-file</code> option (which must also be specified for +this option to have any effect). If you specify an empty string, updated +files will not be mentioned in the log file. For a list of the possible +escape characters, see the "<code>log format</code>" setting in the rsyncd.conf manpage.</p> +<p>The default FORMAT used if <code>--log-file</code> is specified and this option is not +is '%i %n%L'.</p> +</dd> + +<dt><code>--stats</code></dt><dd> +<p>This tells rsync to print a verbose set of statistics on the file transfer, +allowing you to tell how effective rsync's delta-transfer algorithm is for +your data. This option is equivalent to <code>--info=stats2</code> if combined with 0 +or 1 <code>-v</code> options, or <code>--info=stats3</code> if combined with 2 or more <code>-v</code> +options.</p> +<p>The current statistics are as follows:</p> +<ul> +<li><code>Number of files</code> is the count of all "files" (in the generic sense), +which includes directories, symlinks, etc. The total count will be +followed by a list of counts by filetype (if the total is non-zero). For +example: "(reg: 5, dir: 3, link: 2, dev: 1, special: 1)" lists the totals +for regular files, directories, symlinks, devices, and special files. If +any of value is 0, it is completely omitted from the list.</li> +<li><code>Number of created files</code> is the count of how many "files" (generic +sense) were created (as opposed to updated). The total count will be +followed by a list of counts by filetype (if the total is non-zero).</li> +<li><code>Number of deleted files</code> is the count of how many "files" (generic +sense) were created (as opposed to updated). The total count will be +followed by a list of counts by filetype (if the total is non-zero). +Note that this line is only output if deletions are in effect, and only +if protocol 31 is being used (the default for rsync 3.1.x).</li> +<li><code>Number of regular files transferred</code> is the count of normal files that +were updated via rsync's delta-transfer algorithm, which does not include +dirs, symlinks, etc. Note that rsync 3.1.0 added the word "regular" into +this heading.</li> +<li><code>Total file size</code> is the total sum of all file sizes in the transfer. +This does not count any size for directories or special files, but does +include the size of symlinks.</li> +<li><code>Total transferred file size</code> is the total sum of all files sizes for +just the transferred files.</li> +<li><code>Literal data</code> is how much unmatched file-update data we had to send to +the receiver for it to recreate the updated files.</li> +<li><code>Matched data</code> is how much data the receiver got locally when recreating +the updated files.</li> +<li><code>File list size</code> is how big the file-list data was when the sender sent +it to the receiver. This is smaller than the in-memory size for the file +list due to some compressing of duplicated data when rsync sends the +list.</li> +<li><code>File list generation time</code> is the number of seconds that the sender +spent creating the file list. This requires a modern rsync on the +sending side for this to be present.</li> +<li><code>File list transfer time</code> is the number of seconds that the sender spent +sending the file list to the receiver.</li> +<li><code>Total bytes sent</code> is the count of all the bytes that rsync sent from the +client side to the server side.</li> +<li><code>Total bytes received</code> is the count of all non-message bytes that rsync +received by the client side from the server side. "Non-message" bytes +means that we don't count the bytes for a verbose message that the server +sent to us, which makes the stats more consistent.</li> +</ul> +</dd> + +<dt><code>--8-bit-output</code>, <code>-8</code></dt><dd> +<p>This tells rsync to leave all high-bit characters unescaped in the output +instead of trying to test them to see if they're valid in the current +locale and escaping the invalid ones. All control characters (but never +tabs) are always escaped, regardless of this option's setting.</p> +<p>The escape idiom that started in 2.6.7 is to output a literal backslash +(<code>\</code>) and a hash (<code>#</code>), followed by exactly 3 octal digits. For example, a +newline would output as "<code>\#012</code>". A literal backslash that is in a +filename is not escaped unless it is followed by a hash and 3 digits (0-9).</p> +</dd> + +<dt><code>--human-readable</code>, <code>-h</code></dt><dd> +<p>Output numbers in a more human-readable format. There are 3 possible +levels: (1) output numbers with a separator between each set of 3 digits +(either a comma or a period, depending on if the decimal point is +represented by a period or a comma); (2) output numbers in units of 1000 +(with a character suffix for larger units -⁠-⁠ see below); (3) output +numbers in units of 1024.</p> +<p>The default is human-readable level 1. Each <code>-h</code> option increases the +level by one. You can take the level down to 0 (to output numbers as pure +digits) by specifying the <code>--no-human-readable</code> (<code>--no-h</code>) option.</p> +<p>The unit letters that are appended in levels 2 and 3 are: K (kilo), M +(mega), G (giga), or T (tera). For example, a 1234567-byte file would +output as 1.23M in level-2 (assuming that a period is your local decimal +point).</p> +<p>Backward compatibility note: versions of rsync prior to 3.1.0 do not +support human-readable level 1, and they default to level 0. Thus, +specifying one or two <code>-h</code> options will behave in a comparable manner in +old and new versions as long as you didn't specify a <code>--no-h</code> option prior +to one or more <code>-h</code> options. See the <code>--list-only</code> option for one +difference.</p> +</dd> + +<dt><code>--partial</code></dt><dd> +<p>By default, rsync will delete any partially transferred file if the +transfer is interrupted. In some circumstances it is more desirable to +keep partially transferred files. Using the <code>--partial</code> option tells rsync +to keep the partial file which should make a subsequent transfer of the +rest of the file much faster.</p> +</dd> + +<dt><code>--partial-dir=DIR</code></dt><dd> +<p>A better way to keep partial files than the <code>--partial</code> option is to +specify a <u>DIR</u> that will be used to hold the partial data (instead of +writing it out to the destination file). On the next transfer, rsync will +use a file found in this dir as data to speed up the resumption of the +transfer and then delete it after it has served its purpose.</p> +<p>Note that if <code>--whole-file</code> is specified (or implied), any partial-dir file +that is found for a file that is being updated will simply be removed +(since rsync is sending files without using rsync's delta-transfer +algorithm).</p> +<p>Rsync will create the <u>DIR</u> if it is missing (just the last dir -⁠-⁠ not the +whole path). This makes it easy to use a relative path (such as +"<code>--partial-dir=.rsync-partial</code>") to have rsync create the +partial-directory in the destination file's directory when needed, and then +remove it again when the partial file is deleted. Note that the directory +is only removed if it is a relative pathname, as it is expected that an +absolute path is to a directory that is reserved for partial-dir work.</p> +<p>If the partial-dir value is not an absolute path, rsync will add an exclude +rule at the end of all your existing excludes. This will prevent the +sending of any partial-dir files that may exist on the sending side, and +will also prevent the untimely deletion of partial-dir items on the +receiving side. An example: the above <code>--partial-dir</code> option would add the +equivalent of "<code>-f '-p .rsync-partial/'</code>" at the end of any other filter +rules.</p> +<p>If you are supplying your own exclude rules, you may need to add your own +exclude/hide/protect rule for the partial-dir because (1) the auto-added +rule may be ineffective at the end of your other rules, or (2) you may wish +to override rsync's exclude choice. For instance, if you want to make +rsync clean-up any left-over partial-dirs that may be lying around, you +should specify <code>--delete-after</code> and add a "risk" filter rule, e.g. +<code>-f 'R .rsync-partial/'</code>. (Avoid using <code>--delete-before</code> or +<code>--delete-during</code> unless you don't need rsync to use any of the left-over +partial-dir data during the current run.)</p> +<p>IMPORTANT: the <code>--partial-dir</code> should not be writable by other users or it +is a security risk. E.g. AVOID "/tmp".</p> +<p>You can also set the partial-dir value the RSYNC_PARTIAL_DIR environment +variable. Setting this in the environment does not force <code>--partial</code> to be +enabled, but rather it affects where partial files go when <code>--partial</code> is +specified. For instance, instead of using <code>--partial-dir=.rsync-tmp</code> along +with <code>--progress</code>, you could set RSYNC_PARTIAL_DIR=.rsync-tmp in your +environment and then just use the <code>-P</code> option to turn on the use of the +.rsync-tmp dir for partial transfers. The only times that the <code>--partial</code> +option does not look for this environment value are (1) when <code>--inplace</code> +was specified (since <code>--inplace</code> conflicts with <code>--partial-dir</code>), and (2) +when <code>--delay-updates</code> was specified (see below).</p> +<p>When a modern rsync resumes the transfer of a file in the partial-dir, that +partial file is now updated in-place instead of creating yet another +tmp-file copy (so it maxes out at dest + tmp instead of dest + partial + +tmp). This requires both ends of the transfer to be at least version +3.2.0.</p> +<p>For the purposes of the daemon-config's "<code>refuse options</code>" setting, +<code>--partial-dir</code> does <u>not</u> imply <code>--partial</code>. This is so that a refusal of +the <code>--partial</code> option can be used to disallow the overwriting of +destination files with a partial transfer, while still allowing the safer +idiom provided by <code>--partial-dir</code>.</p> +</dd> + +<dt><code>--delay-updates</code></dt><dd> +<p>This option puts the temporary file from each updated file into a holding +directory until the end of the transfer, at which time all the files are +renamed into place in rapid succession. This attempts to make the updating +of the files a little more atomic. By default the files are placed into a +directory named <code>.~tmp~</code> in each file's destination directory, but if +you've specified the <code>--partial-dir</code> option, that directory will be used +instead. See the comments in the <code>--partial-dir</code> section for a discussion +of how this <code>.~tmp~</code> dir will be excluded from the transfer, and what you +can do if you want rsync to cleanup old <code>.~tmp~</code> dirs that might be lying +around. Conflicts with <code>--inplace</code> and <code>--append</code>.</p> +<p>This option uses more memory on the receiving side (one bit per file +transferred) and also requires enough free disk space on the receiving side +to hold an additional copy of all the updated files. Note also that you +should not use an absolute path to <code>--partial-dir</code> unless (1) there is no +chance of any of the files in the transfer having the same name (since all +the updated files will be put into a single directory if the path is +absolute) and (2) there are no mount points in the hierarchy (since the +delayed updates will fail if they can't be renamed into place).</p> +<p>See also the "atomic-rsync" perl script in the "support" subdir for an +update algorithm that is even more atomic (it uses <code>--link-dest</code> and a +parallel hierarchy of files).</p> +</dd> + +<dt><code>--prune-empty-dirs</code>, <code>-m</code></dt><dd> +<p>This option tells the receiving rsync to get rid of empty directories from +the file-list, including nested directories that have no non-directory +children. This is useful for avoiding the creation of a bunch of useless +directories when the sending rsync is recursively scanning a hierarchy of +files using include/exclude/filter rules.</p> +<p>Note that the use of transfer rules, such as the <code>--min-size</code> option, does +not affect what goes into the file list, and thus does not leave +directories empty, even if none of the files in a directory match the +transfer rule.</p> +<p>Because the file-list is actually being pruned, this option also affects +what directories get deleted when a delete is active. However, keep in +mind that excluded files and directories can prevent existing items from +being deleted due to an exclude both hiding source files and protecting +destination files. See the perishable filter-rule option for how to avoid +this.</p> +<p>You can prevent the pruning of certain empty directories from the file-list +by using a global "protect" filter. For instance, this option would ensure +that the directory "emptydir" was kept in the file-list:</p> +<blockquote> +<pre><code>--filter 'protect emptydir/' +</code></pre> +</blockquote> +<p>Here's an example that copies all .pdf files in a hierarchy, only creating +the necessary destination directories to hold the .pdf files, and ensures +that any superfluous files and directories in the destination are removed +(note the hide filter of non-directories being used instead of an exclude):</p> +<blockquote> +<pre><code>rsync -avm --del --include='*.pdf' -f 'hide,! */' src/ dest +</code></pre> +</blockquote> +<p>If you didn't want to remove superfluous destination files, the more +time-honored options of <code>--include='*/' --exclude='*'</code> would work +fine in place of the hide-filter (if that is more natural to you).</p> +</dd> + +<dt><code>--progress</code></dt><dd> +<p>This option tells rsync to print information showing the progress of the +transfer. This gives a bored user something to watch. With a modern rsync +this is the same as specifying <code>--info=flist2,name,progress</code>, but any +user-supplied settings for those info flags takes precedence (e.g. +"<code>--info=flist0 --progress</code>").</p> +<p>While rsync is transferring a regular file, it updates a progress line that +looks like this:</p> +<blockquote> +<pre><code>782448 63% 110.64kB/s 0:00:04 +</code></pre> +</blockquote> +<p>In this example, the receiver has reconstructed 782448 bytes or 63% of the +sender's file, which is being reconstructed at a rate of 110.64 kilobytes +per second, and the transfer will finish in 4 seconds if the current rate +is maintained until the end.</p> +<p>These statistics can be misleading if rsync's delta-transfer algorithm is +in use. For example, if the sender's file consists of the basis file +followed by additional data, the reported rate will probably drop +dramatically when the receiver gets to the literal data, and the transfer +will probably take much longer to finish than the receiver estimated as it +was finishing the matched part of the file.</p> +<p>When the file transfer finishes, rsync replaces the progress line with a +summary line that looks like this:</p> +<blockquote> +<pre><code>1,238,099 100% 146.38kB/s 0:00:08 (xfr#5, to-chk=169/396) +</code></pre> +</blockquote> +<p>In this example, the file was 1,238,099 bytes long in total, the average +rate of transfer for the whole file was 146.38 kilobytes per second over +the 8 seconds that it took to complete, it was the 5th transfer of a +regular file during the current rsync session, and there are 169 more files +for the receiver to check (to see if they are up-to-date or not) remaining +out of the 396 total files in the file-list.</p> +<p>In an incremental recursion scan, rsync won't know the total number of +files in the file-list until it reaches the ends of the scan, but since it +starts to transfer files during the scan, it will display a line with the +text "ir-chk" (for incremental recursion check) instead of "to-chk" until +the point that it knows the full size of the list, at which point it will +switch to using "to-chk". Thus, seeing "ir-chk" lets you know that the +total count of files in the file list is still going to increase (and each +time it does, the count of files left to check will increase by the number +of the files added to the list).</p> +</dd> + +<dt><code>-P</code></dt><dd> +<p>The <code>-P</code> option is equivalent to <code>--partial --progress</code>. Its purpose is +to make it much easier to specify these two options for a long transfer +that may be interrupted.</p> +<p>There is also a <code>--info=progress2</code> option that outputs statistics based on +the whole transfer, rather than individual files. Use this flag without +outputting a filename (e.g. avoid <code>-v</code> or specify <code>--info=name0</code>) if you +want to see how the transfer is doing without scrolling the screen with a +lot of names. (You don't need to specify the <code>--progress</code> option in order +to use <code>--info=progress2</code>.)</p> +<p>Finally, you can get an instant progress report by sending rsync a signal +of either SIGINFO or SIGVTALRM. On BSD systems, a SIGINFO is generated by +typing a Ctrl+T (Linux doesn't currently support a SIGINFO signal). When +the client-side process receives one of those signals, it sets a flag to +output a single progress report which is output when the current file +transfer finishes (so it may take a little time if a big file is being +handled when the signal arrives). A filename is output (if needed) +followed by the <code>--info=progress2</code> format of progress info. If you don't +know which of the 3 rsync processes is the client process, it's OK to +signal all of them (since the non-client processes ignore the signal).</p> +<p>CAUTION: sending SIGVTALRM to an older rsync (pre-3.2.0) will kill it.</p> +</dd> + +<dt><code>--password-file=FILE</code></dt><dd> +<p>This option allows you to provide a password for accessing an rsync daemon +via a file or via standard input if <strong>FILE</strong> is <code>-</code>. The file should +contain just the password on the first line (all other lines are ignored). +Rsync will exit with an error if <strong>FILE</strong> is world readable or if a +root-run rsync command finds a non-root-owned file.</p> +<p>This option does not supply a password to a remote shell transport such as +ssh; to learn how to do that, consult the remote shell's documentation. +When accessing an rsync daemon using a remote shell as the transport, this +option only comes into effect after the remote shell finishes its +authentication (i.e. if you have also specified a password in the daemon's +config file).</p> +</dd> + +<dt><code>--list-only</code></dt><dd> +<p>This option will cause the source files to be listed instead of +transferred. This option is inferred if there is a single source arg and +no destination specified, so its main uses are: (1) to turn a copy command +that includes a destination arg into a file-listing command, or (2) to be +able to specify more than one source arg (note: be sure to include the +destination). Caution: keep in mind that a source arg with a wild-card is +expanded by the shell into multiple args, so it is never safe to try to +list such an arg without using this option. For example:</p> +<blockquote> +<pre><code>rsync -av --list-only foo* dest/ +</code></pre> +</blockquote> +<p>Starting with rsync 3.1.0, the sizes output by <code>--list-only</code> are affected +by the <code>--human-readable</code> option. By default they will contain digit +separators, but higher levels of readability will output the sizes with +unit suffixes. Note also that the column width for the size output has +increased from 11 to 14 characters for all human-readable levels. Use +<code>--no-h</code> if you want just digits in the sizes, and the old column width of +11 characters.</p> +<p>Compatibility note: when requesting a remote listing of files from an rsync +that is version 2.6.3 or older, you may encounter an error if you ask for a +non-recursive listing. This is because a file listing implies the <code>--dirs</code> +option w/o <code>--recursive</code>, and older rsyncs don't have that option. To +avoid this problem, either specify the <code>--no-dirs</code> option (if you don't +need to expand a directory's content), or turn on recursion and exclude the +content of subdirectories: <code>-r --exclude='/*/*'</code>.</p> +</dd> + +<dt><code>--bwlimit=RATE</code></dt><dd> +<p>This option allows you to specify the maximum transfer rate for the data +sent over the socket, specified in units per second. The RATE value can be +suffixed with a string to indicate a size multiplier, and may be a +fractional value (e.g. "<code>--bwlimit=1.5m</code>"). If no suffix is specified, the +value will be assumed to be in units of 1024 bytes (as if "K" or "KiB" had +been appended). See the <code>--max-size</code> option for a description of all the +available suffixes. A value of zero specifies no limit.</p> +<p>For backward-compatibility reasons, the rate limit will be rounded to the +nearest KiB unit, so no rate smaller than 1024 bytes per second is +possible.</p> +<p>Rsync writes data over the socket in blocks, and this option both limits +the size of the blocks that rsync writes, and tries to keep the average +transfer rate at the requested limit. Some burstiness may be seen where +rsync writes out a block of data and then sleeps to bring the average rate +into compliance.</p> +<p>Due to the internal buffering of data, the <code>--progress</code> option may not be +an accurate reflection on how fast the data is being sent. This is because +some files can show up as being rapidly sent when the data is quickly +buffered, while other can show up as very slow when the flushing of the +output buffer occurs. This may be fixed in a future version.</p> +</dd> + +<dt><code>--write-batch=FILE</code></dt><dd> +<p>Record a file that can later be applied to another identical destination +with <code>--read-batch</code>. See the "BATCH MODE" section for details, and also +the <code>--only-write-batch</code> option.</p> +</dd> + +<dt><code>--only-write-batch=FILE</code></dt><dd> +<p>Works like <code>--write-batch</code>, except that no updates are made on the +destination system when creating the batch. This lets you transport the +changes to the destination system via some other means and then apply the +changes via <code>--read-batch</code>.</p> +<p>Note that you can feel free to write the batch directly to some portable +media: if this media fills to capacity before the end of the transfer, you +can just apply that partial transfer to the destination and repeat the +whole process to get the rest of the changes (as long as you don't mind a +partially updated destination system while the multi-update cycle is +happening).</p> +<p>Also note that you only save bandwidth when pushing changes to a remote +system because this allows the batched data to be diverted from the sender +into the batch file without having to flow over the wire to the receiver +(when pulling, the sender is remote, and thus can't write the batch).</p> +</dd> + +<dt><code>--read-batch=FILE</code></dt><dd> +<p>Apply all of the changes stored in FILE, a file previously generated by +<code>--write-batch</code>. If <u>FILE</u> is <code>-</code>, the batch data will be read from +standard input. See the "BATCH MODE" section for details.</p> +</dd> + +<dt><code>--protocol=NUM</code></dt><dd> +<p>Force an older protocol version to be used. This is useful for creating a +batch file that is compatible with an older version of rsync. For +instance, if rsync 2.6.4 is being used with the <code>--write-batch</code> option, but +rsync 2.6.3 is what will be used to run the <code>--read-batch</code> option, you +should use "-⁠-⁠protocol=28" when creating the batch file to force the older +protocol version to be used in the batch file (assuming you can't upgrade +the rsync on the reading system).</p> +</dd> + +<dt><code>--iconv=CONVERT_SPEC</code></dt><dd> +<p>Rsync can convert filenames between character sets using this option. +Using a CONVERT_SPEC of "." tells rsync to look up the default +character-set via the locale setting. Alternately, you can fully specify +what conversion to do by giving a local and a remote charset separated by a +comma in the order <code>--iconv=LOCAL,REMOTE</code>, e.g. <code>--iconv=utf8,iso88591</code>. +This order ensures that the option will stay the same whether you're +pushing or pulling files. Finally, you can specify either <code>--no-iconv</code> or +a CONVERT_SPEC of "-⁠" to turn off any conversion. The default setting of +this option is site-specific, and can also be affected via the RSYNC_ICONV +environment variable.</p> +<p>For a list of what charset names your local iconv library supports, you can +run "<code>iconv --list</code>".</p> +<p>If you specify the <code>--protect-args</code> option (<code>-s</code>), rsync will translate the +filenames you specify on the command-line that are being sent to the remote +host. See also the <code>--files-from</code> option.</p> +<p>Note that rsync does not do any conversion of names in filter files +(including include/exclude files). It is up to you to ensure that you're +specifying matching rules that can match on both sides of the transfer. +For instance, you can specify extra include/exclude rules if there are +filename differences on the two sides that need to be accounted for.</p> +<p>When you pass an <code>--iconv</code> option to an rsync daemon that allows it, the +daemon uses the charset specified in its "charset" configuration parameter +regardless of the remote charset you actually pass. Thus, you may feel +free to specify just the local charset for a daemon transfer (e.g. +<code>--iconv=utf8</code>).</p> +</dd> + +<dt><code>--ipv4</code>, <code>-4</code> or <code>--ipv6</code>, <code>-6</code></dt><dd> +<p>Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh. This +affects sockets that rsync has direct control over, such as the outgoing +socket when directly contacting an rsync daemon, as well as the forwarding +of the <code>-4</code> or <code>-6</code> option to ssh when rsync can deduce that ssh is being +used as the remote shell. For other remote shells you'll need to specify +the "<code>--rsh SHELL -4</code>" option directly (or whatever ipv4/ipv6 hint options +it uses).</p> +<p>These options also exist in the <code>--daemon</code> mode section.</p> +<p>If rsync was complied without support for IPv6, the <code>--ipv6</code> option will +have no effect. The <code>rsync -V</code> output will contain "<code>no IPv6</code>" if is the +case.</p> +</dd> + +<dt><code>--checksum-seed=NUM</code></dt><dd> +<p>Set the checksum seed to the integer NUM. This 4 byte checksum seed is +included in each block and MD4 file checksum calculation (the more modern +MD5 file checksums don't use a seed). By default the checksum seed is +generated by the server and defaults to the current <strong>time</strong>(). This +option is used to set a specific checksum seed, which is useful for +applications that want repeatable block checksums, or in the case where the +user wants a more random checksum seed. Setting NUM to 0 causes rsync to +use the default of <strong>time</strong>() for checksum seed.</p> +</dd> +</dl> +<h1>DAEMON OPTIONS</h1> +<p>The options allowed when starting an rsync daemon are as follows:</p> +<dl> + +<dt><code>--daemon</code></dt><dd> +<p>This tells rsync that it is to run as a daemon. The daemon you start +running may be accessed using an rsync client using the <code>host::module</code> or +<code>rsync://host/module/</code> syntax.</p> +<p>If standard input is a socket then rsync will assume that it is being run +via inetd, otherwise it will detach from the current terminal and become a +background daemon. The daemon will read the config file (rsyncd.conf) on +each connect made by a client and respond to requests accordingly. See the +<strong>rsyncd.conf</strong>(5) man page for more details.</p> +</dd> + +<dt><code>--address</code></dt><dd> +<p>By default rsync will bind to the wildcard address when run as a daemon +with the <code>--daemon</code> option. The <code>--address</code> option allows you to specify a +specific IP address (or hostname) to bind to. This makes virtual hosting +possible in conjunction with the <code>--config</code> option. See also the "address" +global option in the rsyncd.conf manpage.</p> +</dd> + +<dt><code>--bwlimit=RATE</code></dt><dd> +<p>This option allows you to specify the maximum transfer rate for the data +the daemon sends over the socket. The client can still specify a smaller +<code>--bwlimit</code> value, but no larger value will be allowed. See the client +version of this option (above) for some extra details.</p> +</dd> + +<dt><code>--config=FILE</code></dt><dd> +<p>This specifies an alternate config file than the default. This is only +relevant when <code>--daemon</code> is specified. The default is /etc/rsyncd.conf +unless the daemon is running over a remote shell program and the remote +user is not the super-user; in that case the default is rsyncd.conf in the +current directory (typically $HOME).</p> +</dd> + +<dt><code>--dparam=OVERRIDE</code>, <code>-M</code></dt><dd> +<p>This option can be used to set a daemon-config parameter when starting up +rsync in daemon mode. It is equivalent to adding the parameter at the end +of the global settings prior to the first module's definition. The +parameter names can be specified without spaces, if you so desire. For +instance:</p> +<blockquote> +<pre><code>rsync --daemon -M pidfile=/path/rsync.pid +</code></pre> +</blockquote> +</dd> + +<dt><code>--no-detach</code></dt><dd> +<p>When running as a daemon, this option instructs rsync to not detach itself +and become a background process. This option is required when running as a +service on Cygwin, and may also be useful when rsync is supervised by a +program such as <code>daemontools</code> or AIX's <code>System Resource Controller</code>. +<code>--no-detach</code> is also recommended when rsync is run under a debugger. This +option has no effect if rsync is run from inetd or sshd.</p> +</dd> + +<dt><code>--port=PORT</code></dt><dd> +<p>This specifies an alternate TCP port number for the daemon to listen on +rather than the default of 873. See also the "port" global option in the +rsyncd.conf manpage.</p> +</dd> + +<dt><code>--log-file=FILE</code></dt><dd> +<p>This option tells the rsync daemon to use the given log-file name instead +of using the "<code>log file</code>" setting in the config file.</p> +</dd> + +<dt><code>--log-file-format=FORMAT</code></dt><dd> +<p>This option tells the rsync daemon to use the given FORMAT string instead +of using the "<code>log format</code>" setting in the config file. It also enables +"<code>transfer logging</code>" unless the string is empty, in which case transfer +logging is turned off.</p> +</dd> + +<dt><code>--sockopts</code></dt><dd> +<p>This overrides the <code>socket options</code> setting in the rsyncd.conf file and has +the same syntax.</p> +</dd> + +<dt><code>--verbose</code>, <code>-v</code></dt><dd> +<p>This option increases the amount of information the daemon logs during its +startup phase. After the client connects, the daemon's verbosity level +will be controlled by the options that the client used and the +"<code>max verbosity</code>" setting in the module's config section.</p> +</dd> + +<dt><code>--ipv4</code>, <code>-4</code> or <code>--ipv6</code>, <code>-6</code></dt><dd> +<p>Tells rsync to prefer IPv4/IPv6 when creating the incoming sockets that the +rsync daemon will use to listen for connections. One of these options may +be required in older versions of Linux to work around an IPv6 bug in the +kernel (if you see an "address already in use" error when nothing else is +using the port, try specifying <code>--ipv6</code> or <code>--ipv4</code> when starting the +daemon).</p> +<p>These options also exist in the regular rsync options section.</p> +<p>If rsync was complied without support for IPv6, the <code>--ipv6</code> option will +have no effect. The <code>rsync -V</code> output will contain "<code>no IPv6</code>" if is the +case.</p> +</dd> + +<dt><code>--help</code>, <code>-h</code></dt><dd> +<p>When specified after <code>--daemon</code>, print a short help page describing the +options available for starting an rsync daemon.</p> +</dd> +</dl> +<h1>FILTER RULES</h1> +<p>The filter rules allow for flexible selection of which files to transfer +(include) and which files to skip (exclude). The rules either directly specify +include/exclude patterns or they specify a way to acquire more include/exclude +patterns (e.g. to read them from a file).</p> +<p>As the list of files/directories to transfer is built, rsync checks each name +to be transferred against the list of include/exclude patterns in turn, and the +first matching pattern is acted on: if it is an exclude pattern, then that file +is skipped; if it is an include pattern then that filename is not skipped; if +no matching pattern is found, then the filename is not skipped.</p> +<p>Rsync builds an ordered list of filter rules as specified on the command-line. +Filter rules have the following syntax:</p> +<blockquote> +<pre><code>RULE [PATTERN_OR_FILENAME] +RULE,MODIFIERS [PATTERN_OR_FILENAME] +</code></pre> +</blockquote> +<p>You have your choice of using either short or long RULE names, as described +below. If you use a short-named rule, the ',' separating the RULE from the +MODIFIERS is optional. The PATTERN or FILENAME that follows (when present) +must come after either a single space or an underscore (_). Here are the +available rule prefixes:</p> +<dl> +<dt><code>exclude, '-'</code></dt><dd> specifies an exclude pattern.</dd> +<dt><code>include, '+'</code></dt><dd> specifies an include pattern.</dd> +<dt><code>merge, '.'</code></dt><dd> specifies a merge-file to read for more rules.</dd> +<dt><code>dir-merge, ':'</code></dt><dd> specifies a per-directory merge-file.</dd> +<dt><code>hide, 'H'</code></dt><dd> specifies a pattern for hiding files from the transfer.</dd> +<dt><code>show, 'S'</code></dt><dd> files that match the pattern are not hidden.</dd> +<dt><code>protect, 'P'</code></dt><dd> specifies a pattern for protecting files from deletion.</dd> +<dt><code>risk, 'R'</code></dt><dd> files that match the pattern are not protected.</dd> +<dt><code>clear, '!'</code></dt><dd> clears the current include/exclude list (takes no arg)</dd> +</dl> +<p>When rules are being read from a file, empty lines are ignored, as are comment +lines that start with a "#".</p> +<p>Note that the <code>--include</code> & <code>--exclude</code> command-line options do not allow the +full range of rule parsing as described above -⁠-⁠ they only allow the +specification of include / exclude patterns plus a "<code>!</code>" token to clear the +list (and the normal comment parsing when rules are read from a file). If a +pattern does not begin with "<code>- </code>" (dash, space) or "<code>+ </code>" (plus, space), then +the rule will be interpreted as if "<code>+ </code>" (for an include option) or "<code>- </code>" +(for an exclude option) were prefixed to the string. A <code>--filter</code> option, on +the other hand, must always contain either a short or long rule name at the +start of the rule.</p> +<p>Note also that the <code>--filter</code>, <code>--include</code>, and <code>--exclude</code> options take one +rule/pattern each. To add multiple ones, you can repeat the options on the +command-line, use the merge-file syntax of the <code>--filter</code> option, or the +<code>--include-from</code> / <code>--exclude-from</code> options.</p> +<h1>INCLUDE/EXCLUDE PATTERN RULES</h1> +<p>You can include and exclude files by specifying patterns using the "+", "-⁠", +etc. filter rules (as introduced in the FILTER RULES section above). The +include/exclude rules each specify a pattern that is matched against the names +of the files that are going to be transferred. These patterns can take several +forms:</p> +<ul> +<li>if the pattern starts with a <code>/</code> then it is anchored to a particular spot in +the hierarchy of files, otherwise it is matched against the end of the +pathname. This is similar to a leading <code>^</code> in regular expressions. Thus +<code>/foo</code> would match a name of "foo" at either the "root of the transfer" (for +a global rule) or in the merge-file's directory (for a per-directory rule). +An unqualified <code>foo</code> would match a name of "foo" anywhere in the tree because +the algorithm is applied recursively from the top down; it behaves as if each +path component gets a turn at being the end of the filename. Even the +unanchored "sub/foo" would match at any point in the hierarchy where a "foo" +was found within a directory named "sub". See the section on ANCHORING +INCLUDE/EXCLUDE PATTERNS for a full discussion of how to specify a pattern +that matches at the root of the transfer.</li> +<li>if the pattern ends with a <code>/</code> then it will only match a directory, not a +regular file, symlink, or device.</li> +<li>rsync chooses between doing a simple string match and wildcard matching by +checking if the pattern contains one of these three wildcard characters: +'<code>*</code>', '<code>?</code>', and '<code>[</code>' .</li> +<li>a '<code>*</code>' matches any path component, but it stops at slashes.</li> +<li>use '<code>**</code>' to match anything, including slashes.</li> +<li>a '<code>?</code>' matches any character except a slash (<code>/</code>).</li> +<li>a '<code>[</code>' introduces a character class, such as <code>[a-z]</code> or <code>[[:alpha:]]</code>.</li> +<li>in a wildcard pattern, a backslash can be used to escape a wildcard +character, but it is matched literally when no wildcards are present. This +means that there is an extra level of backslash removal when a pattern +contains wildcard characters compared to a pattern that has none. e.g. if +you add a wildcard to "<code>foo\bar</code>" (which matches the backslash) you would +need to use "<code>foo\\bar*</code>" to avoid the "<code>\b</code>" becoming just "b".</li> +<li>if the pattern contains a <code>/</code> (not counting a trailing /) or a "<code>**</code>", then it +is matched against the full pathname, including any leading directories. If +the pattern doesn't contain a <code>/</code> or a "<code>**</code>", then it is matched only against +the final component of the filename. (Remember that the algorithm is applied +recursively so "full filename" can actually be any portion of a path from the +starting directory on down.)</li> +<li>a trailing "<code>dir_name/***</code>" will match both the directory (as if "dir_name/" +had been specified) and everything in the directory (as if "<code>dir_name/**</code>" +had been specified). This behavior was added in version 2.6.7.</li> +</ul> +<p>Note that, when using the <code>--recursive</code> (<code>-r</code>) option (which is implied by +<code>-a</code>), every subdir component of every path is visited left to right, with each +directory having a chance for exclusion before its content. In this way +include/exclude patterns are applied recursively to the pathname of each node +in the filesystem's tree (those inside the transfer). The exclude patterns +short-circuit the directory traversal stage as rsync finds the files to send.</p> +<p>For instance, to include "<code>/foo/bar/baz</code>", the directories "<code>/foo</code>" and "<code>/foo/bar</code>" +must not be excluded. Excluding one of those parent directories prevents the +examination of its content, cutting off rsync's recursion into those paths and +rendering the include for "<code>/foo/bar/baz</code>" ineffectual (since rsync can't match +something it never sees in the cut-off section of the directory hierarchy).</p> +<p>The concept path exclusion is particularly important when using a trailing '<code>*</code>' +rule. For instance, this won't work:</p> +<blockquote> +<pre><code>+ /some/path/this-file-will-not-be-found ++ /file-is-included +- * +</code></pre> +</blockquote> +<p>This fails because the parent directory "some" is excluded by the '<code>*</code>' rule, so +rsync never visits any of the files in the "some" or "some/path" directories. +One solution is to ask for all directories in the hierarchy to be included by +using a single rule: "<code>+ */</code>" (put it somewhere before the "<code>- *</code>" rule), and +perhaps use the <code>--prune-empty-dirs</code> option. Another solution is to add +specific include rules for all the parent dirs that need to be visited. For +instance, this set of rules works fine:</p> +<blockquote> +<pre><code>+ /some/ ++ /some/path/ ++ /some/path/this-file-is-found ++ /file-also-included +- * +</code></pre> +</blockquote> +<p>Here are some examples of exclude/include matching:</p> +<ul> +<li>"<code>- *.o</code>" would exclude all names matching <code>*.o</code></li> +<li>"<code>- /foo</code>" would exclude a file (or directory) named foo in the transfer-root +directory</li> +<li>"<code>- foo/</code>" would exclude any directory named foo</li> +<li>"<code>- /foo/*/bar</code>" would exclude any file named bar which is at two levels +below a directory named foo in the transfer-root directory</li> +<li>"<code>- /foo/**/bar</code>" would exclude any file named bar two or more levels below a +directory named foo in the transfer-root directory</li> +<li>The combination of "<code>+ */</code>", "<code>+ *.c</code>", and "<code>- *</code>" would include all +directories and C source files but nothing else (see also the +<code>--prune-empty-dirs</code> option)</li> +<li>The combination of "<code>+ foo/</code>", "<code>+ foo/bar.c</code>", and "<code>- *</code>" would include +only the foo directory and foo/bar.c (the foo directory must be explicitly +included or it would be excluded by the "<code>*</code>")</li> +</ul> +<p>The following modifiers are accepted after a "<code>+</code>" or "<code>-</code>":</p> +<ul> +<li>A <code>/</code> specifies that the include/exclude rule should be matched against the +absolute pathname of the current item. For example, "<code>-/ /etc/passwd</code>" would +exclude the passwd file any time the transfer was sending files from the +"/etc" directory, and "-⁠/ subdir/foo" would always exclude "foo" when it is +in a dir named "subdir", even if "foo" is at the root of the current +transfer.</li> +<li>A <code>!</code> specifies that the include/exclude should take effect if the pattern +fails to match. For instance, "<code>-! */</code>" would exclude all non-directories.</li> +<li>A <code>C</code> is used to indicate that all the global CVS-exclude rules should be +inserted as excludes in place of the "-⁠C". No arg should follow.</li> +<li>An <code>s</code> is used to indicate that the rule applies to the sending side. When a +rule affects the sending side, it prevents files from being transferred. The +default is for a rule to affect both sides unless <code>--delete-excluded</code> was +specified, in which case default rules become sender-side only. See also the +hide (H) and show (S) rules, which are an alternate way to specify +sending-side includes/excludes.</li> +<li>An <code>r</code> is used to indicate that the rule applies to the receiving side. When +a rule affects the receiving side, it prevents files from being deleted. See +the <code>s</code> modifier for more info. See also the protect (P) and risk (R) rules, +which are an alternate way to specify receiver-side includes/excludes.</li> +<li>A <code>p</code> indicates that a rule is perishable, meaning that it is ignored in +directories that are being deleted. For instance, the <code>-C</code> option's default +rules that exclude things like "CVS" and "<code>*.o</code>" are marked as perishable, +and will not prevent a directory that was removed on the source from being +deleted on the destination.</li> +<li>An <code>x</code> indicates that a rule affects xattr names in xattr copy/delete +operations (and is thus ignored when matching file/dir names). If no +xattr-matching rules are specified, a default xattr filtering rule is used +(see the <code>--xattrs</code> option).</li> +</ul> +<h1>MERGE-FILE FILTER RULES</h1> +<p>You can merge whole files into your filter rules by specifying either a merge +(.) or a dir-merge (:) filter rule (as introduced in the FILTER RULES section +above).</p> +<p>There are two kinds of merged files -⁠-⁠ single-instance ('.') and per-directory +(':'). A single-instance merge file is read one time, and its rules are +incorporated into the filter list in the place of the "." rule. For +per-directory merge files, rsync will scan every directory that it traverses +for the named file, merging its contents when the file exists into the current +list of inherited rules. These per-directory rule files must be created on the +sending side because it is the sending side that is being scanned for the +available files to transfer. These rule files may also need to be transferred +to the receiving side if you want them to affect what files don't get deleted +(see PER-DIRECTORY RULES AND DELETE below).</p> +<p>Some examples:</p> +<blockquote> +<pre><code>merge /etc/rsync/default.rules +. /etc/rsync/default.rules +dir-merge .per-dir-filter +dir-merge,n- .non-inherited-per-dir-excludes +:n- .non-inherited-per-dir-excludes +</code></pre> +</blockquote> +<p>The following modifiers are accepted after a merge or dir-merge rule:</p> +<ul> +<li>A <code>-</code> specifies that the file should consist of only exclude patterns, with +no other rule-parsing except for in-file comments.</li> +<li>A <code>+</code> specifies that the file should consist of only include patterns, with +no other rule-parsing except for in-file comments.</li> +<li>A <code>C</code> is a way to specify that the file should be read in a CVS-compatible +manner. This turns on 'n', 'w', and '-⁠', but also allows the list-clearing +token (!) to be specified. If no filename is provided, ".cvsignore" is +assumed.</li> +<li>A <code>e</code> will exclude the merge-file name from the transfer; e.g. "dir-merge,e +.rules" is like "dir-merge .rules" and "-⁠ .rules".</li> +<li>An <code>n</code> specifies that the rules are not inherited by subdirectories.</li> +<li>A <code>w</code> specifies that the rules are word-split on whitespace instead of the +normal line-splitting. This also turns off comments. Note: the space that +separates the prefix from the rule is treated specially, so "-⁠ foo + bar" is +parsed as two rules (assuming that prefix-parsing wasn't also disabled).</li> +<li>You may also specify any of the modifiers for the "+" or "-⁠" rules (above) in +order to have the rules that are read in from the file default to having that +modifier set (except for the <code>!</code> modifier, which would not be useful). For +instance, "merge,-⁠/ .excl" would treat the contents of .excl as absolute-path +excludes, while "dir-merge,s .filt" and ":sC" would each make all their +per-directory rules apply only on the sending side. If the merge rule +specifies sides to affect (via the <code>s</code> or <code>r</code> modifier or both), then the +rules in the file must not specify sides (via a modifier or a rule prefix +such as <code>hide</code>).</li> +</ul> +<p>Per-directory rules are inherited in all subdirectories of the directory where +the merge-file was found unless the 'n' modifier was used. Each subdirectory's +rules are prefixed to the inherited per-directory rules from its parents, which +gives the newest rules a higher priority than the inherited rules. The entire +set of dir-merge rules are grouped together in the spot where the merge-file +was specified, so it is possible to override dir-merge rules via a rule that +got specified earlier in the list of global rules. When the list-clearing rule +("!") is read from a per-directory file, it only clears the inherited rules for +the current merge file.</p> +<p>Another way to prevent a single rule from a dir-merge file from being inherited +is to anchor it with a leading slash. Anchored rules in a per-directory +merge-file are relative to the merge-file's directory, so a pattern "/foo" +would only match the file "foo" in the directory where the dir-merge filter +file was found.</p> +<p>Here's an example filter file which you'd specify via <code>--filter=". file":</code></p> +<blockquote> +<pre><code>merge /home/user/.global-filter +- *.gz +dir-merge .rules ++ *.[ch] +- *.o +- foo* +</code></pre> +</blockquote> +<p>This will merge the contents of the /home/user/.global-filter file at the start +of the list and also turns the ".rules" filename into a per-directory filter +file. All rules read in prior to the start of the directory scan follow the +global anchoring rules (i.e. a leading slash matches at the root of the +transfer).</p> +<p>If a per-directory merge-file is specified with a path that is a parent +directory of the first transfer directory, rsync will scan all the parent dirs +from that starting point to the transfer directory for the indicated +per-directory file. For instance, here is a common filter (see <code>-F</code>):</p> +<blockquote> +<pre><code>--filter=': /.rsync-filter' +</code></pre> +</blockquote> +<p>That rule tells rsync to scan for the file .rsync-filter in all directories +from the root down through the parent directory of the transfer prior to the +start of the normal directory scan of the file in the directories that are sent +as a part of the transfer. (Note: for an rsync daemon, the root is always the +same as the module's "path".)</p> +<p>Some examples of this pre-scanning for per-directory files:</p> +<blockquote> +<pre><code>rsync -avF /src/path/ /dest/dir +rsync -av --filter=': ../../.rsync-filter' /src/path/ /dest/dir +rsync -av --filter=': .rsync-filter' /src/path/ /dest/dir +</code></pre> +</blockquote> +<p>The first two commands above will look for ".rsync-filter" in "/" and "/src" +before the normal scan begins looking for the file in "/src/path" and its +subdirectories. The last command avoids the parent-dir scan and only looks for +the ".rsync-filter" files in each directory that is a part of the transfer.</p> +<p>If you want to include the contents of a ".cvsignore" in your patterns, you +should use the rule ":C", which creates a dir-merge of the .cvsignore file, but +parsed in a CVS-compatible manner. You can use this to affect where the +<code>--cvs-exclude</code> (<code>-C</code>) option's inclusion of the per-directory .cvsignore file +gets placed into your rules by putting the ":C" wherever you like in your +filter rules. Without this, rsync would add the dir-merge rule for the +.cvsignore file at the end of all your other rules (giving it a lower priority +than your command-line rules). For example:</p> +<blockquote> +<pre><code>cat <<EOT | rsync -avC --filter='. -' a/ b ++ foo.o +:C +- *.old +EOT +rsync -avC --include=foo.o -f :C --exclude='*.old' a/ b +</code></pre> +</blockquote> +<p>Both of the above rsync commands are identical. Each one will merge all the +per-directory .cvsignore rules in the middle of the list rather than at the +end. This allows their dir-specific rules to supersede the rules that follow +the :C instead of being subservient to all your rules. To affect the other CVS +exclude rules (i.e. the default list of exclusions, the contents of +$HOME/.cvsignore, and the value of $CVSIGNORE) you should omit the <code>-C</code> +command-line option and instead insert a "-⁠C" rule into your filter rules; e.g. +"<code>--filter=-C</code>".</p> +<h1>LIST-CLEARING FILTER RULE</h1> +<p>You can clear the current include/exclude list by using the "!" filter rule (as +introduced in the FILTER RULES section above). The "current" list is either +the global list of rules (if the rule is encountered while parsing the filter +options) or a set of per-directory rules (which are inherited in their own +sub-list, so a subdirectory can use this to clear out the parent's rules).</p> +<h1>ANCHORING INCLUDE/EXCLUDE PATTERNS</h1> +<p>As mentioned earlier, global include/exclude patterns are anchored at the "root +of the transfer" (as opposed to per-directory patterns, which are anchored at +the merge-file's directory). If you think of the transfer as a subtree of +names that are being sent from sender to receiver, the transfer-root is where +the tree starts to be duplicated in the destination directory. This root +governs where patterns that start with a / match.</p> +<p>Because the matching is relative to the transfer-root, changing the trailing +slash on a source path or changing your use of the <code>--relative</code> option affects +the path you need to use in your matching (in addition to changing how much of +the file tree is duplicated on the destination host). The following examples +demonstrate this.</p> +<p>Let's say that we want to match two source files, one with an absolute +path of "/home/me/foo/bar", and one with a path of "/home/you/bar/baz". +Here is how the various command choices differ for a 2-source transfer:</p> +<blockquote> +<pre><code>Example cmd: rsync -a /home/me /home/you /dest ++/- pattern: /me/foo/bar ++/- pattern: /you/bar/baz +Target file: /dest/me/foo/bar +Target file: /dest/you/bar/baz +</code></pre> +</blockquote> +<blockquote> +<pre><code>Example cmd: rsync -a /home/me/ /home/you/ /dest ++/- pattern: /foo/bar (note missing "me") ++/- pattern: /bar/baz (note missing "you") +Target file: /dest/foo/bar +Target file: /dest/bar/baz +</code></pre> +</blockquote> +<blockquote> +<pre><code>Example cmd: rsync -a --relative /home/me/ /home/you /dest ++/- pattern: /home/me/foo/bar (note full path) ++/- pattern: /home/you/bar/baz (ditto) +Target file: /dest/home/me/foo/bar +Target file: /dest/home/you/bar/baz +</code></pre> +</blockquote> +<blockquote> +<pre><code>Example cmd: cd /home; rsync -a --relative me/foo you/ /dest ++/- pattern: /me/foo/bar (starts at specified path) ++/- pattern: /you/bar/baz (ditto) +Target file: /dest/me/foo/bar +Target file: /dest/you/bar/baz +</code></pre> +</blockquote> +<p>The easiest way to see what name you should filter is to just +look at the output when using <code>--verbose</code> and put a / in front of the name +(use the <code>--dry-run</code> option if you're not yet ready to copy any files).</p> +<h1>PER-DIRECTORY RULES AND DELETE</h1> +<p>Without a delete option, per-directory rules are only relevant on the sending +side, so you can feel free to exclude the merge files themselves without +affecting the transfer. To make this easy, the 'e' modifier adds this exclude +for you, as seen in these two equivalent commands:</p> +<blockquote> +<pre><code>rsync -av --filter=': .excl' --exclude=.excl host:src/dir /dest +rsync -av --filter=':e .excl' host:src/dir /dest +</code></pre> +</blockquote> +<p>However, if you want to do a delete on the receiving side AND you want some +files to be excluded from being deleted, you'll need to be sure that the +receiving side knows what files to exclude. The easiest way is to include the +per-directory merge files in the transfer and use <code>--delete-after</code>, because +this ensures that the receiving side gets all the same exclude rules as the +sending side before it tries to delete anything:</p> +<blockquote> +<pre><code>rsync -avF --delete-after host:src/dir /dest +</code></pre> +</blockquote> +<p>However, if the merge files are not a part of the transfer, you'll need to +either specify some global exclude rules (i.e. specified on the command line), +or you'll need to maintain your own per-directory merge files on the receiving +side. An example of the first is this (assume that the remote .rules files +exclude themselves):</p> +<blockquote> +<pre><code>rsync -av --filter=': .rules' --filter='. /my/extra.rules' + --delete host:src/dir /dest +</code></pre> +</blockquote> +<p>In the above example the extra.rules file can affect both sides of the +transfer, but (on the sending side) the rules are subservient to the rules +merged from the .rules files because they were specified after the +per-directory merge rule.</p> +<p>In one final example, the remote side is excluding the .rsync-filter files from +the transfer, but we want to use our own .rsync-filter files to control what +gets deleted on the receiving side. To do this we must specifically exclude +the per-directory merge files (so that they don't get deleted) and then put +rules into the local files to control what else should not get deleted. Like +one of these commands:</p> +<blockquote> +<pre><code>rsync -av --filter=':e /.rsync-filter' --delete \ + host:src/dir /dest +rsync -avFF --delete host:src/dir /dest +</code></pre> +</blockquote> +<h1>BATCH MODE</h1> +<p>Batch mode can be used to apply the same set of updates to many identical +systems. Suppose one has a tree which is replicated on a number of hosts. Now +suppose some changes have been made to this source tree and those changes need +to be propagated to the other hosts. In order to do this using batch mode, +rsync is run with the write-batch option to apply the changes made to the +source tree to one of the destination trees. The write-batch option causes the +rsync client to store in a "batch file" all the information needed to repeat +this operation against other, identical destination trees.</p> +<p>Generating the batch file once saves having to perform the file status, +checksum, and data block generation more than once when updating multiple +destination trees. Multicast transport protocols can be used to transfer the +batch update files in parallel to many hosts at once, instead of sending the +same data to every host individually.</p> +<p>To apply the recorded changes to another destination tree, run rsync with the +read-batch option, specifying the name of the same batch file, and the +destination tree. Rsync updates the destination tree using the information +stored in the batch file.</p> +<p>For your convenience, a script file is also created when the write-batch option +is used: it will be named the same as the batch file with ".sh" appended. This +script file contains a command-line suitable for updating a destination tree +using the associated batch file. It can be executed using a Bourne (or +Bourne-like) shell, optionally passing in an alternate destination tree +pathname which is then used instead of the original destination path. This is +useful when the destination tree path on the current host differs from the one +used to create the batch file.</p> +<p>Examples:</p> +<blockquote> +<pre><code>$ rsync --write-batch=foo -a host:/source/dir/ /adest/dir/ +$ scp foo* remote: +$ ssh remote ./foo.sh /bdest/dir/ +</code></pre> +</blockquote> +<blockquote> +<pre><code>$ rsync --write-batch=foo -a /source/dir/ /adest/dir/ +$ ssh remote rsync --read-batch=- -a /bdest/dir/ <foo +</code></pre> +</blockquote> +<p>In these examples, rsync is used to update /adest/dir/ from /source/dir/ and +the information to repeat this operation is stored in "foo" and "foo.sh". The +host "remote" is then updated with the batched data going into the directory +/bdest/dir. The differences between the two examples reveals some of the +flexibility you have in how you deal with batches:</p> +<ul> +<li>The first example shows that the initial copy doesn't have to be local -⁠-⁠ you +can push or pull data to/from a remote host using either the remote-shell +syntax or rsync daemon syntax, as desired.</li> +<li>The first example uses the created "foo.sh" file to get the right rsync +options when running the read-batch command on the remote host.</li> +<li>The second example reads the batch data via standard input so that the batch +file doesn't need to be copied to the remote machine first. This example +avoids the foo.sh script because it needed to use a modified <code>--read-batch</code> +option, but you could edit the script file if you wished to make use of it +(just be sure that no other option is trying to use standard input, such as +the "<code>--exclude-from=-</code>" option).</li> +</ul> +<p>Caveats:</p> +<p>The read-batch option expects the destination tree that it is updating to be +identical to the destination tree that was used to create the batch update +fileset. When a difference between the destination trees is encountered the +update might be discarded with a warning (if the file appears to be up-to-date +already) or the file-update may be attempted and then, if the file fails to +verify, the update discarded with an error. This means that it should be safe +to re-run a read-batch operation if the command got interrupted. If you wish +to force the batched-update to always be attempted regardless of the file's +size and date, use the <code>-I</code> option (when reading the batch). If an error +occurs, the destination tree will probably be in a partially updated state. In +that case, rsync can be used in its regular (non-batch) mode of operation to +fix up the destination tree.</p> +<p>The rsync version used on all destinations must be at least as new as the one +used to generate the batch file. Rsync will die with an error if the protocol +version in the batch file is too new for the batch-reading rsync to handle. +See also the <code>--protocol</code> option for a way to have the creating rsync generate +a batch file that an older rsync can understand. (Note that batch files +changed format in version 2.6.3, so mixing versions older than that with newer +versions will not work.)</p> +<p>When reading a batch file, rsync will force the value of certain options to +match the data in the batch file if you didn't set them to the same as the +batch-writing command. Other options can (and should) be changed. For +instance <code>--write-batch</code> changes to <code>--read-batch</code>, <code>--files-from</code> is dropped, +and the <code>--filter</code> / <code>--include</code> / <code>--exclude</code> options are not needed unless +one of the <code>--delete</code> options is specified.</p> +<p>The code that creates the BATCH.sh file transforms any filter/include/exclude +options into a single list that is appended as a "here" document to the shell +script file. An advanced user can use this to modify the exclude list if a +change in what gets deleted by <code>--delete</code> is desired. A normal user can ignore +this detail and just use the shell script as an easy way to run the appropriate +<code>--read-batch</code> command for the batched data.</p> +<p>The original batch mode in rsync was based on "rsync+", but the latest +version uses a new implementation.</p> +<h1>SYMBOLIC LINKS</h1> +<p>Three basic behaviors are possible when rsync encounters a symbolic +link in the source directory.</p> +<p>By default, symbolic links are not transferred at all. A message "skipping +non-regular" file is emitted for any symlinks that exist.</p> +<p>If <code>--links</code> is specified, then symlinks are recreated with the same target on +the destination. Note that <code>--archive</code> implies <code>--links</code>.</p> +<p>If <code>--copy-links</code> is specified, then symlinks are "collapsed" by +copying their referent, rather than the symlink.</p> +<p>Rsync can also distinguish "safe" and "unsafe" symbolic links. An example +where this might be used is a web site mirror that wishes to ensure that the +rsync module that is copied does not include symbolic links to <code>/etc/passwd</code> in +the public section of the site. Using <code>--copy-unsafe-links</code> will cause any +links to be copied as the file they point to on the destination. Using +<code>--safe-links</code> will cause unsafe links to be omitted altogether. (Note that you +must specify <code>--links</code> for <code>--safe-links</code> to have any effect.)</p> +<p>Symbolic links are considered unsafe if they are absolute symlinks +(start with <code>/</code>), empty, or if they contain enough ".." +components to ascend from the directory being copied.</p> +<p>Here's a summary of how the symlink options are interpreted. The list is in +order of precedence, so if your combination of options isn't mentioned, use the +first line that is a complete subset of your options:</p> +<dl> +<dt><code>--copy-links</code></dt><dd> Turn all symlinks into normal files (leaving no symlinks for +any other options to affect).</dd> +<dt><code>--links --copy-unsafe-links</code></dt><dd> Turn all unsafe symlinks into files and +duplicate all safe symlinks.</dd> +<dt><code>--copy-unsafe-links</code></dt><dd> Turn all unsafe symlinks into files, noisily skip all +safe symlinks.</dd> +<dt><code>--links --safe-links</code></dt><dd> Duplicate safe symlinks and skip unsafe ones.</dd> +<dt><code>--links</code></dt><dd> Duplicate all symlinks.</dd> +</dl> +<h1>DIAGNOSTICS</h1> +<p>rsync occasionally produces error messages that may seem a little cryptic. The +one that seems to cause the most confusion is "protocol version mismatch -⁠-⁠ is +your shell clean?".</p> +<p>This message is usually caused by your startup scripts or remote shell facility +producing unwanted garbage on the stream that rsync is using for its transport. +The way to diagnose this problem is to run your remote shell like this:</p> +<blockquote> +<pre><code>ssh remotehost /bin/true > out.dat +</code></pre> +</blockquote> +<p>then look at out.dat. If everything is working correctly then out.dat should +be a zero length file. If you are getting the above error from rsync then you +will probably find that out.dat contains some text or data. Look at the +contents and try to work out what is producing it. The most common cause is +incorrectly configured shell startup scripts (such as .cshrc or .profile) that +contain output statements for non-interactive logins.</p> +<p>If you are having trouble debugging filter patterns, then try specifying the +<code>-vv</code> option. At this level of verbosity rsync will show why each individual +file is included or excluded.</p> +<h1>EXIT VALUES</h1> +<dl> +<dt><strong>0</strong></dt><dd> Success</dd> +<dt><strong>1</strong></dt><dd> Syntax or usage error</dd> +<dt><strong>2</strong></dt><dd> Protocol incompatibility</dd> +<dt><strong>3</strong></dt><dd> Errors selecting input/output files, dirs</dd> +<dt><strong>4</strong></dt><dd> Requested action not supported: an attempt was made to manipulate +64-bit files on a platform that cannot support them; or an option was +specified that is supported by the client and not by the server.</dd> +<dt><strong>5</strong></dt><dd> Error starting client-server protocol</dd> +<dt><strong>6</strong></dt><dd> Daemon unable to append to log-file</dd> +<dt><strong>10</strong></dt><dd> Error in socket I/O</dd> +<dt><strong>11</strong></dt><dd> Error in file I/O</dd> +<dt><strong>12</strong></dt><dd> Error in rsync protocol data stream</dd> +<dt><strong>13</strong></dt><dd> Errors with program diagnostics</dd> +<dt><strong>14</strong></dt><dd> Error in IPC code</dd> +<dt><strong>20</strong></dt><dd> Received SIGUSR1 or SIGINT</dd> +<dt><strong>21</strong></dt><dd> Some error returned by <strong>waitpid()</strong></dd> +<dt><strong>22</strong></dt><dd> Error allocating core memory buffers</dd> +<dt><strong>23</strong></dt><dd> Partial transfer due to error</dd> +<dt><strong>24</strong></dt><dd> Partial transfer due to vanished source files</dd> +<dt><strong>25</strong></dt><dd> The -⁠-⁠max-delete limit stopped deletions</dd> +<dt><strong>30</strong></dt><dd> Timeout in data send/receive</dd> +<dt><strong>35</strong></dt><dd> Timeout waiting for daemon connection</dd> +</dl> +<h1>ENVIRONMENT VARIABLES</h1> +<dl> + +<dt><code>CVSIGNORE</code></dt><dd> +<p>The CVSIGNORE environment variable supplements any ignore patterns in +.cvsignore files. See the <code>--cvs-exclude</code> option for more details.</p> +</dd> + +<dt><code>RSYNC_ICONV</code></dt><dd> +<p>Specify a default <code>--iconv</code> setting using this environment variable. (First +supported in 3.0.0.)</p> +</dd> + +<dt><code>RSYNC_PROTECT_ARGS</code></dt><dd> +<p>Specify a non-zero numeric value if you want the <code>--protect-args</code> option to +be enabled by default, or a zero value to make sure that it is disabled by +default. (First supported in 3.1.0.)</p> +</dd> + +<dt><code>RSYNC_RSH</code></dt><dd> +<p>The RSYNC_RSH environment variable allows you to override the default shell +used as the transport for rsync. Command line options are permitted after +the command name, just as in the <code>-e</code> option.</p> +</dd> + +<dt><code>RSYNC_PROXY</code></dt><dd> +<p>The RSYNC_PROXY environment variable allows you to redirect your rsync +client to use a web proxy when connecting to a rsync daemon. You should +set RSYNC_PROXY to a hostname:port pair.</p> +</dd> + +<dt><code>RSYNC_PASSWORD</code></dt><dd> +<p>Setting RSYNC_PASSWORD to the required password allows you to run +authenticated rsync connections to an rsync daemon without user +intervention. Note that this does not supply a password to a remote shell +transport such as ssh; to learn how to do that, consult the remote shell's +documentation.</p> +</dd> + +<dt><code>USER</code> or <code>LOGNAME</code></dt><dd> +<p>The USER or LOGNAME environment variables are used to determine the default +username sent to an rsync daemon. If neither is set, the username defaults +to "nobody".</p> +</dd> + +<dt><code>HOME</code></dt><dd> +<p>The HOME environment variable is used to find the user's default .cvsignore +file.</p> +</dd> +</dl> +<h1>FILES</h1> +<p>/etc/rsyncd.conf or rsyncd.conf</p> +<h1>SEE ALSO</h1> +<p><strong>rsync-ssl</strong>(1), <strong>rsyncd.conf</strong>(5)</p> +<h1>BUGS</h1> +<p>times are transferred as *nix time_t values</p> +<p>When transferring to FAT filesystems rsync may re-sync +unmodified files. +See the comments on the <code>--modify-window</code> option.</p> +<p>file permissions, devices, etc. are transferred as native numerical +values</p> +<p>see also the comments on the <code>--delete</code> option</p> +<p>Please report bugs! See the web site at <a href="http://rsync.samba.org/">http://rsync.samba.org/</a>.</p> +<h1>VERSION</h1> +<p>This man page is current for version 3.2.0 of rsync.</p> +<h1>INTERNAL OPTIONS</h1> +<p>The options <code>--server</code> and <code>--sender</code> are used internally by rsync, and should +never be typed by a user under normal circumstances. Some awareness of these +options may be needed in certain scenarios, such as when setting up a login +that can only run an rsync command. For instance, the support directory of the +rsync distribution has an example script named rrsync (for restricted rsync) +that can be used with a restricted ssh login.</p> +<h1>CREDITS</h1> +<p>rsync is distributed under the GNU General Public License. See the file +COPYING for details.</p> +<p>A web site is available at <a href="http://rsync.samba.org/">http://rsync.samba.org/</a>. The site includes an +FAQ-O-Matic which may cover questions unanswered by this manual page.</p> +<p>The primary ftp site for rsync is <a href="ftp://rsync.samba.org/pub/rsync">ftp://rsync.samba.org/pub/rsync</a></p> +<p>We would be delighted to hear from you if you like this program. Please +contact the mailing-list at <a href="mailto:rsync@lists.samba.org">rsync@lists.samba.org</a>.</p> +<p>This program uses the excellent zlib compression library written by Jean-loup +Gailly and Mark Adler.</p> +<h1>THANKS</h1> +<p>Special thanks go out to: John Van Essen, Matt McCutchen, Wesley W. Terpstra, +David Dykstra, Jos Backus, Sebastian Krahmer, Martin Pool, and our +gone-but-not-forgotten compadre, J.W. Schultz.</p> +<p>Thanks also to Richard Brent, Brendan Mackay, Bill Waite, Stephen Rothwell and +David Bell. I've probably missed some people, my apologies if I have.</p> +<h1>AUTHOR</h1> +<p>rsync was originally written by Andrew Tridgell and Paul Mackerras. Many +people have later contributed to it. It is currently maintained by Wayne +Davison.</p> +<p>Mailing lists for support and development are available at +<a href="http://lists.samba.org/">http://lists.samba.org/</a>.</p> +<div style="float: right"><p><i>19 Jun 2020</i></p></div> +</body></html> diff --git a/rsync.1.md b/rsync.1.md new file mode 100644 index 00000000..375f92ed --- /dev/null +++ b/rsync.1.md @@ -0,0 +1,3966 @@ +# NAME + +rsync - a fast, versatile, remote (and local) file-copying tool + +# SYNOPSIS + +``` +Local: + rsync [OPTION...] SRC... [DEST] + +Access via remote shell: + Pull: + rsync [OPTION...] [USER@]HOST:SRC... [DEST] + Push: + rsync [OPTION...] SRC... [USER@]HOST:DEST + +Access via rsync daemon: + Pull: + rsync [OPTION...] [USER@]HOST::SRC... [DEST] + rsync [OPTION...] rsync://[USER@]HOST[:PORT]/SRC... [DEST] + Push: + rsync [OPTION...] SRC... [USER@]HOST::DEST + rsync [OPTION...] SRC... rsync://[USER@]HOST[:PORT]/DEST) +``` + +Usages with just one SRC arg and no DEST arg will list the source files instead +of copying. + +# DESCRIPTION + +Rsync is a fast and extraordinarily versatile file copying tool. It can copy +locally, to/from another host over any remote shell, or to/from a remote rsync +daemon. It offers a large number of options that control every aspect of its +behavior and permit very flexible specification of the set of files to be +copied. It is famous for its delta-transfer algorithm, which reduces the +amount of data sent over the network by sending only the differences between +the source files and the existing files in the destination. Rsync is widely +used for backups and mirroring and as an improved copy command for everyday +use. + +Rsync finds files that need to be transferred using a "quick check" algorithm +(by default) that looks for files that have changed in size or in last-modified +time. Any changes in the other preserved attributes (as requested by options) +are made on the destination file directly when the quick check indicates that +the file's data does not need to be updated. + +Some of the additional features of rsync are: + +- support for copying links, devices, owners, groups, and permissions +- exclude and exclude-from options similar to GNU tar +- a CVS exclude mode for ignoring the same files that CVS would ignore +- can use any transparent remote shell, including ssh or rsh +- does not require super-user privileges +- pipelining of file transfers to minimize latency costs +- support for anonymous or authenticated rsync daemons (ideal for mirroring) + +# GENERAL + +Rsync copies files either to or from a remote host, or locally on the current +host (it does not support copying files between two remote hosts). + +There are two different ways for rsync to contact a remote system: using a +remote-shell program as the transport (such as ssh or rsh) or contacting an +rsync daemon directly via TCP. The remote-shell transport is used whenever the +source or destination path contains a single colon (:) separator after a host +specification. Contacting an rsync daemon directly happens when the source or +destination path contains a double colon (::) separator after a host +specification, OR when an rsync:// URL is specified (see also the "USING +RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION" section for an exception +to this latter rule). + +As a special case, if a single source arg is specified without a destination, +the files are listed in an output format similar to "`ls -l`". + +As expected, if neither the source or destination path specify a remote host, +the copy occurs locally (see also the `--list-only` option). + +Rsync refers to the local side as the client and the remote side as the server. +Don't confuse server with an rsync daemon. A daemon is always a server, but a +server can be either a daemon or a remote-shell spawned process. + +# SETUP + +See the file README.md for installation instructions. + +Once installed, you can use rsync to any machine that you can access via a +remote shell (as well as some that you can access using the rsync daemon-mode +protocol). For remote transfers, a modern rsync uses ssh for its +communications, but it may have been configured to use a different remote shell +by default, such as rsh or remsh. + +You can also specify any remote shell you like, either by using the `-e` +command line option, or by setting the RSYNC_RSH environment variable. + +Note that rsync must be installed on both the source and destination machines. + +# USAGE + +You use rsync in the same way you use rcp. You must specify a source and a +destination, one of which may be remote. + +Perhaps the best way to explain the syntax is with some examples: + +> rsync -t *.c foo:src/ + +This would transfer all files matching the pattern `*.c` from the current +directory to the directory src on the machine foo. If any of the files already +exist on the remote system then the rsync remote-update protocol is used to +update the file by sending only the differences in the data. Note that the +expansion of wildcards on the commandline (`*.c`) into a list of files is +handled by the shell before it runs rsync and not by rsync itself (exactly the +same as all other posix-style programs). + +> rsync -avz foo:src/bar /data/tmp + +This would recursively transfer all files from the directory src/bar on the +machine foo into the /data/tmp/bar directory on the local machine. The files +are transferred in archive mode, which ensures that symbolic links, devices, +attributes, permissions, ownerships, etc. are preserved in the transfer. +Additionally, compression will be used to reduce the size of data portions of +the transfer. + +> rsync -avz foo:src/bar/ /data/tmp + +A trailing slash on the source changes this behavior to avoid creating an +additional directory level at the destination. You can think of a trailing / +on a source as meaning "copy the contents of this directory" as opposed to +"copy the directory by name", but in both cases the attributes of the +containing directory are transferred to the containing directory on the +destination. In other words, each of the following commands copies the files +in the same way, including their setting of the attributes of /dest/foo: + +> rsync -av /src/foo /dest +> rsync -av /src/foo/ /dest/foo + +Note also that host and module references don't require a trailing slash to +copy the contents of the default directory. For example, both of these copy +the remote directory's contents into "/dest": + +> rsync -av host: /dest +> rsync -av host::module /dest + +You can also use rsync in local-only mode, where both the source and +destination don't have a ':' in the name. In this case it behaves like an +improved copy command. + +Finally, you can list all the (listable) modules available from a particular +rsync daemon by leaving off the module name: + +> rsync somehost.mydomain.com:: + +See the following section for more details. + +# ADVANCED USAGE + +The syntax for requesting multiple files from a remote host is done by +specifying additional remote-host args in the same style as the first, or with +the hostname omitted. For instance, all these work: + +> rsync -av host:file1 :file2 host:file{3,4} /dest/ +> rsync -av host::modname/file{1,2} host::modname/file3 /dest/ +> rsync -av host::modname/file1 ::modname/file{3,4} + +Older versions of rsync required using quoted spaces in the SRC, like these +examples: + +> rsync -av host:'dir1/file1 dir2/file2' /dest +> rsync host::'modname/dir1/file1 modname/dir2/file2' /dest + +This word-splitting still works (by default) in the latest rsync, but is not as +easy to use as the first method. + +If you need to transfer a filename that contains whitespace, you can either +specify the `--protect-args` (`-s`) option, or you'll need to escape the +whitespace in a way that the remote shell will understand. For instance: + +> rsync -av host:'file\ name\ with\ spaces' /dest + +# CONNECTING TO AN RSYNC DAEMON + +It is also possible to use rsync without a remote shell as the transport. In +this case you will directly connect to a remote rsync daemon, typically using +TCP port 873. (This obviously requires the daemon to be running on the remote +system, so refer to the STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS section +below for information on that.) + +Using rsync in this way is the same as using it with a remote shell except +that: + +- you either use a double colon :: instead of a single colon to separate the + hostname from the path, or you use an rsync:// URL. +- the first word of the "path" is actually a module name. +- the remote daemon may print a message of the day when you connect. +- if you specify no path name on the remote daemon then the list of accessible + paths on the daemon will be shown. +- if you specify no local destination then a listing of the specified files on + the remote daemon is provided. +- you must not specify the `--rsh` (`-e`) option (since that overrides the + daemon connection to use ssh -- see USING RSYNC-DAEMON FEATURES VIA A + REMOTE-SHELL CONNECTION below). + +An example that copies all the files in a remote module named "src": + +> rsync -av host::src /dest + +Some modules on the remote daemon may require authentication. If so, you will +receive a password prompt when you connect. You can avoid the password prompt +by setting the environment variable RSYNC_PASSWORD to the password you want to +use or using the `--password-file` option. This may be useful when scripting +rsync. + +WARNING: On some systems environment variables are visible to all users. On +those systems using `--password-file` is recommended. + +You may establish the connection via a web proxy by setting the environment +variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note +that your web proxy's configuration must support proxy connections to port 873. + +You may also establish a daemon connection using a program as a proxy by +setting the environment variable RSYNC_CONNECT_PROG to the commands you wish to +run in place of making a direct socket connection. The string may contain the +escape "%H" to represent the hostname specified in the rsync command (so use +"%%" if you need a single "%" in your string). For example: + +> export RSYNC_CONNECT_PROG='ssh proxyhost nc %H 873' +> rsync -av targethost1::module/src/ /dest/ +> rsync -av rsync://targethost2/module/src/ /dest/ + +The command specified above uses ssh to run nc (netcat) on a proxyhost, which +forwards all data to port 873 (the rsync daemon) on the targethost (%H). + +Note also that if the RSYNC_SHELL environment variable is set, that program +will be used to run the RSYNC_CONNECT_PROG command instead of using the default +shell of the **system()** call. + +# USING RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION + +It is sometimes useful to use various features of an rsync daemon (such as +named modules) without actually allowing any new socket connections into a +system (other than what is already required to allow remote-shell access). +Rsync supports connecting to a host using a remote shell and then spawning a +single-use "daemon" server that expects to read its config file in the home dir +of the remote user. This can be useful if you want to encrypt a daemon-style +transfer's data, but since the daemon is started up fresh by the remote user, +you may not be able to use features such as chroot or change the uid used by +the daemon. (For another way to encrypt a daemon transfer, consider using ssh +to tunnel a local port to a remote machine and configure a normal rsync daemon +on that remote host to only allow connections from "localhost".) + +From the user's perspective, a daemon transfer via a remote-shell connection +uses nearly the same command-line syntax as a normal rsync-daemon transfer, +with the only exception being that you must explicitly set the remote shell +program on the command-line with the `--rsh=COMMAND` option. (Setting the +RSYNC_RSH in the environment will not turn on this functionality.) For example: + +> rsync -av --rsh=ssh host::module /dest + +If you need to specify a different remote-shell user, keep in mind that the +user@ prefix in front of the host is specifying the rsync-user value (for a +module that requires user-based authentication). This means that you must give +the '-l user' option to ssh when specifying the remote-shell, as in this +example that uses the short version of the `--rsh` option: + +> rsync -av -e "ssh -l ssh-user" rsync-user@host::module /dest + +The "ssh-user" will be used at the ssh level; the "rsync-user" will be used to +log-in to the "module". + +# STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS + +In order to connect to an rsync daemon, the remote system needs to have a +daemon already running (or it needs to have configured something like inetd to +spawn an rsync daemon for incoming connections on a particular port). For full +information on how to start a daemon that will handling incoming socket +connections, see the **rsyncd.conf**(5) man page -- that is the config file for +the daemon, and it contains the full details for how to run the daemon +(including stand-alone and inetd configurations). + +If you're using one of the remote-shell transports for the transfer, there is +no need to manually start an rsync daemon. + +# SORTED TRANSFER ORDER + +Rsync always sorts the specified filenames into its internal transfer list. +This handles the merging together of the contents of identically named +directories, makes it easy to remove duplicate filenames, and may confuse +someone when the files are transferred in a different order than what was given +on the command-line. + +If you need a particular file to be transferred prior to another, either +separate the files into different rsync calls, or consider using +`--delay-updates` (which doesn't affect the sorted transfer order, but does +make the final file-updating phase happen much more rapidly). + +# EXAMPLES + +Here are some examples of how I use rsync. + +To backup my wife's home directory, which consists of large MS Word files and +mail folders, I use a cron job that runs + +> rsync -Cavz . arvidsjaur:backup + +each night over a PPP connection to a duplicate directory on my machine +"arvidsjaur". + +To synchronize my samba source trees I use the following Makefile targets: + +> get: +> rsync -avuzb --exclude '*~' samba:samba/ . +> put: +> rsync -Cavuzb . samba:samba/ +> sync: get put + +This allows me to sync with a CVS directory at the other end of the connection. +I then do CVS operations on the remote machine, which saves a lot of time as +the remote CVS protocol isn't very efficient. + +I mirror a directory between my "old" and "new" ftp sites with the command: + +> rsync -az -e ssh --delete ~ftp/pub/samba nimbus:"~ftp/pub/tridge" + +This is launched from cron every few hours. + +# OPTION SUMMARY + +Here is a short summary of the options available in rsync. Please refer to the +detailed description below for a complete description. + +[comment]: # (help-rsync.h) +[comment]: # (Keep these short enough that they'll be under 80 chars when indented by 8 chars.) + +``` +--verbose, -v increase verbosity +--info=FLAGS fine-grained informational verbosity +--debug=FLAGS fine-grained debug verbosity +--msgs2stderr output messages directly to stderr +--quiet, -q suppress non-error messages +--no-motd suppress daemon-mode MOTD +--checksum, -c skip based on checksum, not mod-time & size +--archive, -a archive mode; equals -rlptgoD (no -H,-A,-X) +--no-OPTION turn off an implied OPTION (e.g. --no-D) +--recursive, -r recurse into directories +--relative, -R use relative path names +--no-implied-dirs don't send implied dirs with --relative +--backup, -b make backups (see --suffix & --backup-dir) +--backup-dir=DIR make backups into hierarchy based in DIR +--suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) +--update, -u skip files that are newer on the receiver +--inplace update destination files in-place +--append append data onto shorter files +--append-verify --append w/old data in file checksum +--dirs, -d transfer directories without recursing +--links, -l copy symlinks as symlinks +--copy-links, -L transform symlink into referent file/dir +--copy-unsafe-links only "unsafe" symlinks are transformed +--safe-links ignore symlinks that point outside the tree +--munge-links munge symlinks to make them safe & unusable +--copy-dirlinks, -k transform symlink to dir into referent dir +--keep-dirlinks, -K treat symlinked dir on receiver as dir +--hard-links, -H preserve hard links +--perms, -p preserve permissions +--executability, -E preserve executability +--chmod=CHMOD affect file and/or directory permissions +--acls, -A preserve ACLs (implies --perms) +--xattrs, -X preserve extended attributes +--owner, -o preserve owner (super-user only) +--group, -g preserve group +--devices preserve device files (super-user only) +--specials preserve special files +-D same as --devices --specials +--times, -t preserve modification times +--atimes, -U preserve access (use) times +--open-noatime avoid changing the atime on opened files +--omit-dir-times, -O omit directories from --times +--omit-link-times, -J omit symlinks from --times +--super receiver attempts super-user activities +--fake-super store/recover privileged attrs using xattrs +--sparse, -S turn sequences of nulls into sparse blocks +--preallocate allocate dest files before writing them +--write-devices write to devices as files (implies --inplace) +--dry-run, -n perform a trial run with no changes made +--whole-file, -W copy files whole (w/o delta-xfer algorithm) +--checksum-choice=STR choose the checksum algorithms +--one-file-system, -x don't cross filesystem boundaries +--block-size=SIZE, -B force a fixed checksum block-size +--rsh=COMMAND, -e specify the remote shell to use +--rsync-path=PROGRAM specify the rsync to run on remote machine +--existing skip creating new files on receiver +--ignore-existing skip updating files that exist on receiver +--remove-source-files sender removes synchronized files (non-dir) +--del an alias for --delete-during +--delete delete extraneous files from dest dirs +--delete-before receiver deletes before xfer, not during +--delete-during receiver deletes during the transfer +--delete-delay find deletions during, delete after +--delete-after receiver deletes after transfer, not during +--delete-excluded also delete excluded files from dest dirs +--ignore-missing-args ignore missing source args without error +--delete-missing-args delete missing source args from destination +--ignore-errors delete even if there are I/O errors +--force force deletion of dirs even if not empty +--max-delete=NUM don't delete more than NUM files +--max-size=SIZE don't transfer any file larger than SIZE +--min-size=SIZE don't transfer any file smaller than SIZE +--partial keep partially transferred files +--partial-dir=DIR put a partially transferred file into DIR +--delay-updates put all updated files into place at end +--prune-empty-dirs, -m prune empty directory chains from file-list +--numeric-ids don't map uid/gid values by user/group name +--usermap=STRING custom username mapping +--groupmap=STRING custom groupname mapping +--chown=USER:GROUP simple username/groupname mapping +--timeout=SECONDS set I/O timeout in seconds +--contimeout=SECONDS set daemon connection timeout in seconds +--ignore-times, -I don't skip files that match size and time +--size-only skip files that match in size +--modify-window=NUM, -@ set the accuracy for mod-time comparisons +--temp-dir=DIR, -T create temporary files in directory DIR +--fuzzy, -y find similar file for basis if no dest file +--compare-dest=DIR also compare destination files relative to DIR +--copy-dest=DIR ... and include copies of unchanged files +--link-dest=DIR hardlink to files in DIR when unchanged +--compress, -z compress file data during the transfer +--compress-level=NUM explicitly set compression level +--skip-compress=LIST skip compressing files with suffix in LIST +--cvs-exclude, -C auto-ignore files in the same way CVS does +--filter=RULE, -f add a file-filtering RULE +-F same as --filter='dir-merge /.rsync-filter' + repeated: --filter='- .rsync-filter' +--exclude=PATTERN exclude files matching PATTERN +--exclude-from=FILE read exclude patterns from FILE +--include=PATTERN don't exclude files matching PATTERN +--include-from=FILE read include patterns from FILE +--files-from=FILE read list of source-file names from FILE +--from0, -0 all *-from/filter files are delimited by 0s +--protect-args, -s no space-splitting; wildcard chars only +--copy-as=USER[:GROUP] specify user & optional group for the copy +--address=ADDRESS bind address for outgoing socket to daemon +--port=PORT specify double-colon alternate port number +--sockopts=OPTIONS specify custom TCP options +--blocking-io use blocking I/O for the remote shell +--outbuf=N|L|B set out buffering to None, Line, or Block +--stats give some file-transfer stats +--8-bit-output, -8 leave high-bit chars unescaped in output +--human-readable, -h output numbers in a human-readable format +--progress show progress during transfer +-P same as --partial --progress +--itemize-changes, -i output a change-summary for all updates +--remote-option=OPT, -M send OPTION to the remote side only +--out-format=FORMAT output updates using the specified FORMAT +--log-file=FILE log what we're doing to the specified FILE +--log-file-format=FMT log updates using the specified FMT +--password-file=FILE read daemon-access password from FILE +--list-only list the files instead of copying them +--bwlimit=RATE limit socket I/O bandwidth +--write-batch=FILE write a batched update to FILE +--only-write-batch=FILE like --write-batch but w/o updating dest +--read-batch=FILE read a batched update from FILE +--protocol=NUM force an older protocol version to be used +--iconv=CONVERT_SPEC request charset conversion of filenames +--checksum-seed=NUM set block/file checksum seed (advanced) +--ipv4, -4 prefer IPv4 +--ipv6, -6 prefer IPv6 +--version, -V print the version + other info and exit +--help, -h (*) show this help (* -h is help only on its own) +``` + +Rsync can also be run as a daemon, in which case the following options are +accepted: + +[comment]: # (help-rsyncd.h) + +``` +--daemon run as an rsync daemon +--address=ADDRESS bind to the specified address +--bwlimit=RATE limit socket I/O bandwidth +--config=FILE specify alternate rsyncd.conf file +--dparam=OVERRIDE, -M override global daemon config parameter +--no-detach do not detach from the parent +--port=PORT listen on alternate port number +--log-file=FILE override the "log file" setting +--log-file-format=FMT override the "log format" setting +--sockopts=OPTIONS specify custom TCP options +--verbose, -v increase verbosity +--ipv4, -4 prefer IPv4 +--ipv6, -6 prefer IPv6 +--help, -h show this help (when used with --daemon) +``` + +# OPTIONS + +Rsync accepts both long (double-dash + word) and short (single-dash + letter) +options. The full list of the available options are described below. If an +option can be specified in more than one way, the choices are comma-separated. +Some options only have a long variant, not a short. If the option takes a +parameter, the parameter is only listed after the long variant, even though it +must also be specified for the short. When specifying a parameter, you can +either use the form `--option=param` or replace the '=' with whitespace. The +parameter may need to be quoted in some manner for it to survive the shell's +command-line parsing. Keep in mind that a leading tilde (`~`) in a filename is +substituted by your shell, so `--option=~/foo` will not change the tilde into +your home directory (remove the '=' for that). + +[comment]: # (An OL starting at 0 is converted into a DL by the parser.) + +0. `--help`, `-h` `(*)` + + Print a short help page describing the options available in rsync and exit. + (*) The `-h` short option will only invoke `--help` when used without other + options since it normally means `--human-readable`. + +0. `--version`, `-V` + + Print the rsync version plus other info and exit. + + The output includes the default list of checksum algorithms, the default + list of compression algorithms, a list of compiled-in capabilities, a link + to the rsync web site, and some license/copyright info. + +0. `--verbose`, `-v` + + This option increases the amount of information you are given during the + transfer. By default, rsync works silently. A single `-v` will give you + information about what files are being transferred and a brief summary at + the end. Two `-v` options will give you information on what files are + being skipped and slightly more information at the end. More than two `-v` + options should only be used if you are debugging rsync. + + In a modern rsync, the `-v` option is equivalent to the setting of groups + of `--info` and `--debug` options. You can choose to use these newer + options in addition to, or in place of using `--verbose`, as any + fine-grained settings override the implied settings of `-v`. Both `--info` + and `--debug` have a way to ask for help that tells you exactly what flags + are set for each increase in verbosity. + + However, do keep in mind that a daemon's "`max verbosity`" setting will limit + how high of a level the various individual flags can be set on the daemon + side. For instance, if the max is 2, then any info and/or debug flag that + is set to a higher value than what would be set by `-vv` will be downgraded + to the `-vv` level in the daemon's logging. + +0. `--info=FLAGS` + + This option lets you have fine-grained control over the information output + you want to see. An individual flag name may be followed by a level + number, with 0 meaning to silence that output, 1 being the default output + level, and higher numbers increasing the output of that flag (for those + that support higher levels). Use `--info=help` to see all the available + flag names, what they output, and what flag names are added for each + increase in the verbose level. Some examples: + + > rsync -a --info=progress2 src/ dest/ + > rsync -avv --info=stats2,misc1,flist0 src/ dest/ + + Note that `--info=name`'s output is affected by the `--out-format` and + `--itemize-changes` (`-i`) options. See those options for more information + on what is output and when. + + This option was added to 3.1.0, so an older rsync on the server side might + reject your attempts at fine-grained control (if one or more flags needed + to be send to the server and the server was too old to understand them). + See also the "`max verbosity`" caveat above when dealing with a daemon. + +0. `--debug=FLAGS` + + This option lets you have fine-grained control over the debug output you + want to see. An individual flag name may be followed by a level number, + with 0 meaning to silence that output, 1 being the default output level, + and higher numbers increasing the output of that flag (for those that + support higher levels). Use `--debug=help` to see all the available flag + names, what they output, and what flag names are added for each increase in + the verbose level. Some examples: + + > rsync -avvv --debug=none src/ dest/ + > rsync -avA --del --debug=del2,acl src/ dest/ + + Note that some debug messages will only be output when `--msgs2stderr` is + specified, especially those pertaining to I/O and buffer debugging. + + Beginning in 3.2.0, this option is no longer auto-forwared to the server + side in order to allow you to specify different debug values for each side + of the transfer, as well as to specify a new debug option that is only + present in one of the rsync versions. If you want to duplicate the same + option on both sides, using brace expansion is an easy way to save you some + typing. This works in zsh and bash: + + > rsync -aiv {-M,}--debug=del2 src/ dest/ + +0. `--msgs2stderr` + + This option changes rsync to send all its output directly to stderr rather + than to send messages to the client side via the protocol. The protocol + allows rsync to output normal messages via stdout and errors via stderr, + but it can delay messages behind a slew of data. + + One case where this is helpful is when sending really large files, since + errors that happen on a remote receiver tend to get delayed until after the + file's data is fully sent. It is also helpful for debugging, since it + helps to avoid overpopulating the protocol data with extra message data. + + The option does not affect the remote side of a transfer without using + `--remote-option` -- e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`. + + Also keep in mind that connecting to a normal (non-remote-shell) daemon + does not have a stderr channel to send messages back to the client side, so + a modern rsync only allows the option on a remote-shell-run daemon. + + This option has the side-effect of making stderr output get line-buffered + so that the merging of the output of 3 programs happens in a more readable + manner. + +0. `--quiet`, `-q` + + This option decreases the amount of information you are given during the + transfer, notably suppressing information messages from the remote server. + This option is useful when invoking rsync from cron. + +0. `--no-motd` + + This option affects the information that is output by the client at the + start of a daemon transfer. This suppresses the message-of-the-day (MOTD) + text, but it also affects the list of modules that the daemon sends in + response to the "rsync host::" request (due to a limitation in the rsync + protocol), so omit this option if you want to request the list of modules + from the daemon. + +0. `--ignore-times`, `-I` + + Normally rsync will skip any files that are already the same size and have + the same modification timestamp. This option turns off this "quick check" + behavior, causing all files to be updated. + +0. `--size-only` + + This modifies rsync's "quick check" algorithm for finding files that need + to be transferred, changing it from the default of transferring files with + either a changed size or a changed last-modified time to just looking for + files that have changed in size. This is useful when starting to use rsync + after using another mirroring system which may not preserve timestamps + exactly. + +0. `--modify-window`, `-@` + + When comparing two timestamps, rsync treats the timestamps as being equal + if they differ by no more than the modify-window value. The default is 0, + which matches just integer seconds. If you specify a negative value (and + the receiver is at least version 3.1.3) then nanoseconds will also be taken + into account. Specifying 1 is useful for copies to/from MS Windows FAT + filesystems, because FAT represents times with a 2-second resolution + (allowing times to differ from the original by up to 1 second). + + If you want all your transfers to default to comparing nanoseconds, you can + create a `~/.popt` file and put these lines in it: + + > rsync alias -a -a@-1 + > rsync alias -t -t@-1 + + With that as the default, you'd need to specify `--modify-window=0` (aka + `-@0`) to override it and ignore nanoseconds, e.g. if you're copying + between ext3 and ext4, or if the receiving rsync is older than 3.1.3. + +0. `--checksum`, `-c` + + This changes the way rsync checks if the files have been changed and are in + need of a transfer. Without this option, rsync uses a "quick check" that + (by default) checks if each file's size and time of last modification match + between the sender and receiver. This option changes this to compare a + 128-bit checksum for each file that has a matching size. Generating the + checksums means that both sides will expend a lot of disk I/O reading all + the data in the files in the transfer, so this can slow things down + significantly (and this is prior to any reading that will be done to + transfer changed files) + + The sending side generates its checksums while it is doing the file-system + scan that builds the list of the available files. The receiver generates + its checksums when it is scanning for changed files, and will checksum any + file that has the same size as the corresponding sender's file: files with + either a changed size or a changed checksum are selected for transfer. + + Note that rsync always verifies that each _transferred_ file was correctly + reconstructed on the receiving side by checking a whole-file checksum that + is generated as the file is transferred, but that automatic + after-the-transfer verification has nothing to do with this option's + before-the-transfer "Does this file need to be updated?" check. + + The checksum used is auto-negotiated between the client and the server, but + can be overridden using either the `--checksum-choice` option or an + environment variable that is discussed in that option's section. + +0. `--archive`, `-a` + + This is equivalent to `-rlptgoD`. It is a quick way of saying you want + recursion and want to preserve almost everything (with `-H` being a notable + omission). The only exception to the above equivalence is when + `--files-from` is specified, in which case `-r` is not implied. + + Note that `-a` **does not preserve hardlinks**, because finding + multiply-linked files is expensive. You must separately specify `-H`. + +0. `--no-OPTION` + + You may turn off one or more implied options by prefixing the option name + with "no-". Not all options may be prefixed with a "no-": only options that + are implied by other options (e.g. `--no-D`, `--no-perms`) or have + different defaults in various circumstances (e.g. `--no-whole-file`, + `--no-blocking-io`, `--no-dirs`). You may specify either the short or the + long option name after the "no-" prefix (e.g. `--no-R` is the same as + `--no-relative`). + + For example: if you want to use `-a` (`--archive`) but don't want `-o` + (`--owner`), instead of converting `-a` into `-rlptgD`, you could specify + `-a --no-o` (or `-a --no-owner`). + + The order of the options is important: if you specify `--no-r -a`, the + `-r` option would end up being turned on, the opposite of `-a --no-r`. + Note also that the side-effects of the `--files-from` option are NOT + positional, as it affects the default state of several options and slightly + changes the meaning of `-a` (see the `--files-from` option for more + details). + +0. `--recursive`, `-r` + + This tells rsync to copy directories recursively. See also `--dirs` (`-d`). + + Beginning with rsync 3.0.0, the recursive algorithm used is now an + incremental scan that uses much less memory than before and begins the + transfer after the scanning of the first few directories have been + completed. This incremental scan only affects our recursion algorithm, and + does not change a non-recursive transfer. It is also only possible when + both ends of the transfer are at least version 3.0.0. + + Some options require rsync to know the full file list, so these options + disable the incremental recursion mode. These include: `--delete-before`, + `--delete-after`, `--prune-empty-dirs`, and `--delay-updates`. Because of + this, the default delete mode when you specify `--delete` is now + `--delete-during` when both ends of the connection are at least 3.0.0 (use + `--del` or `--delete-during` to request this improved deletion mode + explicitly). See also the `--delete-delay` option that is a better choice + than using `--delete-after`. + + Incremental recursion can be disabled using the `--no-inc-recursive` option + or its shorter `--no-i-r` alias. + +0. `--relative`, `-R` + + Use relative paths. This means that the full path names specified on the + command line are sent to the server rather than just the last parts of the + filenames. This is particularly useful when you want to send several + different directories at the same time. For example, if you used this + command: + + > rsync -av /foo/bar/baz.c remote:/tmp/ + + would create a file named baz.c in /tmp/ on the remote machine. If instead + you used + + > rsync -avR /foo/bar/baz.c remote:/tmp/ + + then a file named /tmp/foo/bar/baz.c would be created on the remote + machine, preserving its full path. These extra path elements are called + "implied directories" (i.e. the "foo" and the "foo/bar" directories in the + above example). + + Beginning with rsync 3.0.0, rsync always sends these implied directories as + real directories in the file list, even if a path element is really a + symlink on the sending side. This prevents some really unexpected behaviors + when copying the full path of a file that you didn't realize had a symlink + in its path. If you want to duplicate a server-side symlink, include both + the symlink via its path, and referent directory via its real path. If + you're dealing with an older rsync on the sending side, you may need to use + the `--no-implied-dirs` option. + + It is also possible to limit the amount of path information that is sent as + implied directories for each path you specify. With a modern rsync on the + sending side (beginning with 2.6.7), you can insert a dot and a slash into + the source path, like this: + + > rsync -avR /foo/./bar/baz.c remote:/tmp/ + + That would create /tmp/bar/baz.c on the remote machine. (Note that the dot + must be followed by a slash, so "/foo/." would not be abbreviated.) For + older rsync versions, you would need to use a chdir to limit the source + path. For example, when pushing files: + + > (cd /foo; rsync -avR bar/baz.c remote:/tmp/) + + (Note that the parens put the two commands into a sub-shell, so that the + "cd" command doesn't remain in effect for future commands.) If you're + pulling files from an older rsync, use this idiom (but only for a + non-daemon transfer): + + > rsync -avR --rsync-path="cd /foo; rsync" \ + > remote:bar/baz.c /tmp/ + +0. `--no-implied-dirs` + + This option affects the default behavior of the `--relative` option. When + it is specified, the attributes of the implied directories from the source + names are not included in the transfer. This means that the corresponding + path elements on the destination system are left unchanged if they exist, + and any missing implied directories are created with default attributes. + This even allows these implied path elements to have big differences, such + as being a symlink to a directory on the receiving side. + + For instance, if a command-line arg or a files-from entry told rsync to + transfer the file "path/foo/file", the directories "path" and "path/foo" + are implied when `--relative` is used. If "path/foo" is a symlink to "bar" + on the destination system, the receiving rsync would ordinarily delete + "path/foo", recreate it as a directory, and receive the file into the new + directory. With `--no-implied-dirs`, the receiving rsync updates + "path/foo/file" using the existing path elements, which means that the file + ends up being created in "path/bar". Another way to accomplish this link + preservation is to use the `--keep-dirlinks` option (which will also affect + symlinks to directories in the rest of the transfer). + + When pulling files from an rsync older than 3.0.0, you may need to use this + option if the sending side has a symlink in the path you request and you + wish the implied directories to be transferred as normal directories. + +0. `--backup`, `-b` + + With this option, preexisting destination files are renamed as each file is + transferred or deleted. You can control where the backup file goes and + what (if any) suffix gets appended using the `--backup-dir` and `--suffix` + options. + + Note that if you don't specify `--backup-dir`, (1) the `--omit-dir-times` + option will be forced on, and (2) if `--delete` is also in effect (without + `--delete-excluded`), rsync will add a "protect" filter-rule for the backup + suffix to the end of all your existing excludes (e.g. `-f "P *~"`). This + will prevent previously backed-up files from being deleted. Note that if + you are supplying your own filter rules, you may need to manually insert + your own exclude/protect rule somewhere higher up in the list so that it + has a high enough priority to be effective (e.g., if your rules specify a + trailing inclusion/exclusion of `*`, the auto-added rule would never be + reached). + +0. `--backup-dir=DIR` + + In combination with the `--backup` option, this tells rsync to store all + backups in the specified directory on the receiving side. This can be used + for incremental backups. You can additionally specify a backup suffix + using the `--suffix` option (otherwise the files backed up in the specified + directory will keep their original filenames). + + Note that if you specify a relative path, the backup directory will be + relative to the destination directory, so you probably want to specify + either an absolute path or a path that starts with "../". If an rsync + daemon is the receiver, the backup dir cannot go outside the module's path + hierarchy, so take extra care not to delete it or copy into it. + +0. `--suffix=SUFFIX` + + This option allows you to override the default backup suffix used with the + `--backup` (`-b`) option. The default suffix is a `~` if no `--backup-dir` + was specified, otherwise it is an empty string. + +0. `--update`, `-u` + + This forces rsync to skip any files which exist on the destination and have + a modified time that is newer than the source file. (If an existing + destination file has a modification time equal to the source file's, it + will be updated if the sizes are different.) + + Note that this does not affect the copying of dirs, symlinks, or other + special files. Also, a difference of file format between the sender and + receiver is always considered to be important enough for an update, no + matter what date is on the objects. In other words, if the source has a + directory where the destination has a file, the transfer would occur + regardless of the timestamps. + + This option is a transfer rule, not an exclude, so it doesn't affect the + data that goes into the file-lists, and thus it doesn't affect deletions. + It just limits the files that the receiver requests to be transferred. + +0. `--inplace` + + This option changes how rsync transfers a file when its data needs to be + updated: instead of the default method of creating a new copy of the file + and moving it into place when it is complete, rsync instead writes the + updated data directly to the destination file. + + This has several effects: + + - Hard links are not broken. This means the new data will be visible + through other hard links to the destination file. Moreover, attempts to + copy differing source files onto a multiply-linked destination file will + result in a "tug of war" with the destination data changing back and + forth. + - In-use binaries cannot be updated (either the OS will prevent this from + happening, or binaries that attempt to swap-in their data will misbehave + or crash). + - The file's data will be in an inconsistent state during the transfer and + will be left that way if the transfer is interrupted or if an update + fails. + - A file that rsync cannot write to cannot be updated. While a super user + can update any file, a normal user needs to be granted write permission + for the open of the file for writing to be successful. + - The efficiency of rsync's delta-transfer algorithm may be reduced if some + data in the destination file is overwritten before it can be copied to a + position later in the file. This does not apply if you use `--backup`, + since rsync is smart enough to use the backup file as the basis file for + the transfer. + + WARNING: you should not use this option to update files that are being + accessed by others, so be careful when choosing to use this for a copy. + + This option is useful for transferring large files with block-based changes + or appended data, and also on systems that are disk bound, not network + bound. It can also help keep a copy-on-write filesystem snapshot from + diverging the entire contents of a file that only has minor changes. + + The option implies `--partial` (since an interrupted transfer does not + delete the file), but conflicts with `--partial-dir` and `--delay-updates`. + Prior to rsync 2.6.4 `--inplace` was also incompatible with + `--compare-dest` and `--link-dest`. + +0. `--append` + + This causes rsync to update a file by appending data onto the end of the + file, which presumes that the data that already exists on the receiving + side is identical with the start of the file on the sending side. If a + file needs to be transferred and its size on the receiver is the same or + longer than the size on the sender, the file is skipped. This does not + interfere with the updating of a file's non-content attributes (e.g. + permissions, ownership, etc.) when the file does not need to be + transferred, nor does it affect the updating of any non-regular files. + Implies `--inplace`. + + The use of `--append` can be dangerous if you aren't 100% sure that the + files that are longer have only grown by the appending of data onto the + end. You should thus use include/exclude/filter rules to ensure that such + a transfer is only affecting files that you know to be growing via appended + data. + +0. `--append-verify` + + This works just like the `--append` option, but the existing data on the + receiving side is included in the full-file checksum verification step, + which will cause a file to be resent if the final verification step fails + (rsync uses a normal, non-appending `--inplace` transfer for the resend). + It otherwise has the exact same caveats for files that have not grown + larger, so don't use this for a general copy. + + Note: prior to rsync 3.0.0, the `--append` option worked like + `--append-verify`, so if you are interacting with an older rsync (or the + transfer is using a protocol prior to 30), specifying either append option + will initiate an `--append-verify` transfer. + +0. `--dirs`, `-d` + + Tell the sending side to include any directories that are encountered. + Unlike `--recursive`, a directory's contents are not copied unless the + directory name specified is "." or ends with a trailing slash (e.g. ".", + "dir/.", "dir/", etc.). Without this option or the `--recursive` option, + rsync will skip all directories it encounters (and output a message to that + effect for each one). If you specify both `--dirs` and `--recursive`, + `--recursive` takes precedence. + + The `--dirs` option is implied by the `--files-from` option or the + `--list-only` option (including an implied `--list-only` usage) if + `--recursive` wasn't specified (so that directories are seen in the + listing). Specify `--no-dirs` (or `--no-d`) if you want to turn this off. + + There is also a backward-compatibility helper option, `--old-dirs` (or + `--old-d`) that tells rsync to use a hack of `-r --exclude='/*/*'` to get + an older rsync to list a single directory without recursing. + +0. `--links`, `-l` + + When symlinks are encountered, recreate the symlink on the destination. + +0. `--copy-links`, `-L` + + When symlinks are encountered, the item that they point to (the referent) + is copied, rather than the symlink. In older versions of rsync, this + option also had the side-effect of telling the receiving side to follow + symlinks, such as symlinks to directories. In a modern rsync such as this + one, you'll need to specify `--keep-dirlinks` (`-K`) to get this extra + behavior. The only exception is when sending files to an rsync that is too + old to understand `-K` -- in that case, the `-L` option will still have the + side-effect of `-K` on that older receiving rsync. + +0. `--copy-unsafe-links` + + This tells rsync to copy the referent of symbolic links that point outside + the copied tree. Absolute symlinks are also treated like ordinary files, + and so are any symlinks in the source path itself when `--relative` is + used. This option has no additional effect if `--copy-links` was also + specified. + + Note that the cut-off point is the top of the transfer, which is the part + of the path that rsync isn't mentioning in the verbose output. If you copy + "/src/subdir" to "/dest/" then the "subdir" directory is a name inside the + transfer tree, not the top of the transfer (which is /src) so it is legal + for created relative symlinks to refer to other names inside the /src and + /dest directories. If you instead copy "/src/subdir/" (with a trailing + slash) to "/dest/subdir" that would not allow symlinks to any files outside + of "subdir". + +0. `--safe-links` + + This tells rsync to ignore any symbolic links which point outside the + copied tree. All absolute symlinks are also ignored. Using this option in + conjunction with `--relative` may give unexpected results. + +0. `--munge-links` + + This option tells rsync to (1) modify all symlinks on the receiving side in + a way that makes them unusable but recoverable (see below), or (2) to + unmunge symlinks on the sending side that had been stored in a munged + state. This is useful if you don't quite trust the source of the data to + not try to slip in a symlink to a unexpected place. + + The way rsync disables the use of symlinks is to prefix each one with the + string "/rsyncd-munged/". This prevents the links from being used as long + as that directory does not exist. When this option is enabled, rsync will + refuse to run if that path is a directory or a symlink to a directory. + + The option only affects the client side of the transfer, so if you need it + to affect the server, specify it via `--remote-option`. (Note that in a + local transfer, the client side is the sender.) + + This option has no affect on a daemon, since the daemon configures whether + it wants munged symlinks via its "`munge symlinks`" parameter. See also the + "munge-symlinks" perl script in the support directory of the source code. + +0. `--copy-dirlinks`, `-k` + + This option causes the sending side to treat a symlink to a directory as + though it were a real directory. This is useful if you don't want symlinks + to non-directories to be affected, as they would be using `--copy-links`. + + Without this option, if the sending side has replaced a directory with a + symlink to a directory, the receiving side will delete anything that is in + the way of the new symlink, including a directory hierarchy (as long as + `--force` or `--delete` is in effect). + + See also `--keep-dirlinks` for an analogous option for the receiving side. + + `--copy-dirlinks` applies to all symlinks to directories in the source. If + you want to follow only a few specified symlinks, a trick you can use is to + pass them as additional source args with a trailing slash, using + `--relative` to make the paths match up right. For example: + + > rsync -r --relative src/./ src/./follow-me/ dest/ + + This works because rsync calls **lstat**(2) on the source arg as given, and + the trailing slash makes **lstat**(2) follow the symlink, giving rise to a + directory in the file-list which overrides the symlink found during the + scan of "src/./". + +0. `--keep-dirlinks`, `-K` + + This option causes the receiving side to treat a symlink to a directory as + though it were a real directory, but only if it matches a real directory + from the sender. Without this option, the receiver's symlink would be + deleted and replaced with a real directory. + + For example, suppose you transfer a directory "foo" that contains a file + "file", but "foo" is a symlink to directory "bar" on the receiver. Without + `--keep-dirlinks`, the receiver deletes symlink "foo", recreates it as a + directory, and receives the file into the new directory. With + `--keep-dirlinks`, the receiver keeps the symlink and "file" ends up in + "bar". + + One note of caution: if you use `--keep-dirlinks`, you must trust all the + symlinks in the copy! If it is possible for an untrusted user to create + their own symlink to any directory, the user could then (on a subsequent + copy) replace the symlink with a real directory and affect the content of + whatever directory the symlink references. For backup copies, you are + better off using something like a bind mount instead of a symlink to modify + your receiving hierarchy. + + See also `--copy-dirlinks` for an analogous option for the sending side. + +0. `--hard-links`, `-H` + + This tells rsync to look for hard-linked files in the source and link + together the corresponding files on the destination. Without this option, + hard-linked files in the source are treated as though they were separate + files. + + This option does NOT necessarily ensure that the pattern of hard links on + the destination exactly matches that on the source. Cases in which the + destination may end up with extra hard links include the following: + + - If the destination contains extraneous hard-links (more linking than what + is present in the source file list), the copying algorithm will not break + them explicitly. However, if one or more of the paths have content + differences, the normal file-update process will break those extra links + (unless you are using the `--inplace` option). + - If you specify a `--link-dest` directory that contains hard links, the + linking of the destination files against the `--link-dest` files can + cause some paths in the destination to become linked together due to the + `--link-dest` associations. + + Note that rsync can only detect hard links between files that are inside + the transfer set. If rsync updates a file that has extra hard-link + connections to files outside the transfer, that linkage will be broken. If + you are tempted to use the `--inplace` option to avoid this breakage, be + very careful that you know how your files are being updated so that you are + certain that no unintended changes happen due to lingering hard links (and + see the `--inplace` option for more caveats). + + If incremental recursion is active (see `--recursive`), rsync may transfer + a missing hard-linked file before it finds that another link for that + contents exists elsewhere in the hierarchy. This does not affect the + accuracy of the transfer (i.e. which files are hard-linked together), just + its efficiency (i.e. copying the data for a new, early copy of a + hard-linked file that could have been found later in the transfer in + another member of the hard-linked set of files). One way to avoid this + inefficiency is to disable incremental recursion using the + `--no-inc-recursive` option. + +0. `--perms`, `-p` + + This option causes the receiving rsync to set the destination permissions + to be the same as the source permissions. (See also the `--chmod` option + for a way to modify what rsync considers to be the source permissions.) + + When this option is _off_, permissions are set as follows: + + - Existing files (including updated files) retain their existing + permissions, though the `--executability` option might change just the + execute permission for the file. + - New files get their "normal" permission bits set to the source file's + permissions masked with the receiving directory's default permissions + (either the receiving process's umask, or the permissions specified via + the destination directory's default ACL), and their special permission + bits disabled except in the case where a new directory inherits a setgid + bit from its parent directory. + + Thus, when `--perms` and `--executability` are both disabled, rsync's + behavior is the same as that of other file-copy utilities, such as **cp**(1) + and **tar**(1). + + In summary: to give destination files (both old and new) the source + permissions, use `--perms`. To give new files the destination-default + permissions (while leaving existing files unchanged), make sure that the + `--perms` option is off and use `--chmod=ugo=rwX` (which ensures that all + non-masked bits get enabled). If you'd care to make this latter behavior + easier to type, you could define a popt alias for it, such as putting this + line in the file `~/.popt` (the following defines the `-Z` option, and + includes `--no-g` to use the default group of the destination dir): + + > rsync alias -Z --no-p --no-g --chmod=ugo=rwX + + You could then use this new option in a command such as this one: + + > rsync -avZ src/ dest/ + + (Caveat: make sure that `-a` does not follow `-Z`, or it will re-enable the + two `--no-*` options mentioned above.) + + The preservation of the destination's setgid bit on newly-created + directories when `--perms` is off was added in rsync 2.6.7. Older rsync + versions erroneously preserved the three special permission bits for + newly-created files when `--perms` was off, while overriding the + destination's setgid bit setting on a newly-created directory. Default ACL + observance was added to the ACL patch for rsync 2.6.7, so older (or + non-ACL-enabled) rsyncs use the umask even if default ACLs are present. + (Keep in mind that it is the version of the receiving rsync that affects + these behaviors.) + +0. `--executability`, `-E` + + This option causes rsync to preserve the executability (or + non-executability) of regular files when `--perms` is not enabled. A + regular file is considered to be executable if at least one 'x' is turned + on in its permissions. When an existing destination file's executability + differs from that of the corresponding source file, rsync modifies the + destination file's permissions as follows: + + - To make a file non-executable, rsync turns off all its 'x' permissions. + - To make a file executable, rsync turns on each 'x' permission that has a + corresponding 'r' permission enabled. + + If `--perms` is enabled, this option is ignored. + +0. `--acls`, `-A` + + This option causes rsync to update the destination ACLs to be the same as + the source ACLs. The option also implies `--perms`. + + The source and destination systems must have compatible ACL entries for + this option to work properly. See the `--fake-super` option for a way to + backup and restore ACLs that are not compatible. + +0. `--xattrs`, `-X` + + This option causes rsync to update the destination extended attributes to + be the same as the source ones. + + For systems that support extended-attribute namespaces, a copy being done + by a super-user copies all namespaces except system.\*. A normal user only + copies the user.\* namespace. To be able to backup and restore non-user + namespaces as a normal user, see the `--fake-super` option. + + The above name filtering can be overridden by using one or more filter + options with the **x** modifier. When you specify an xattr-affecting + filter rule, rsync requires that you do your own system/user filtering, as + well as any additional filtering for what xattr names are copied and what + names are allowed to be deleted. For example, to skip the system + namespace, you could specify: + + > --filter='-x system.*' + + To skip all namespaces except the user namespace, you could specify a + negated-user match: + + > --filter='-x! user.*' + + To prevent any attributes from being deleted, you could specify a + receiver-only rule that excludes all names: + + > --filter='-xr *' + + Note that the `-X` option does not copy rsync's special xattr values (e.g. + those used by `--fake-super`) unless you repeat the option (e.g. `-XX`). + This "copy all xattrs" mode cannot be used with `--fake-super`. + +0. `--chmod` + + This option tells rsync to apply one or more comma-separated "chmod" modes + to the permission of the files in the transfer. The resulting value is + treated as though it were the permissions that the sending side supplied + for the file, which means that this option can seem to have no effect on + existing files if `--perms` is not enabled. + + In addition to the normal parsing rules specified in the **chmod**(1) + manpage, you can specify an item that should only apply to a directory by + prefixing it with a 'D', or specify an item that should only apply to a + file by prefixing it with a 'F'. For example, the following will ensure + that all directories get marked set-gid, that no files are other-writable, + that both are user-writable and group-writable, and that both have + consistent executability across all bits: + + > --chmod=Dg+s,ug+w,Fo-w,+X + + Using octal mode numbers is also allowed: + + > --chmod=D2775,F664 + + It is also legal to specify multiple `--chmod` options, as each additional + option is just appended to the list of changes to make. + + See the `--perms` and `--executability` options for how the resulting + permission value can be applied to the files in the transfer. + +0. `--owner`, `-o` + + This option causes rsync to set the owner of the destination file to be the + same as the source file, but only if the receiving rsync is being run as + the super-user (see also the `--super` and `--fake-super` options). Without + this option, the owner of new and/or transferred files are set to the + invoking user on the receiving side. + + The preservation of ownership will associate matching names by default, but + may fall back to using the ID number in some circumstances (see also the + `--numeric-ids` option for a full discussion). + +0. `--group`, `-g` + + This option causes rsync to set the group of the destination file to be the + same as the source file. If the receiving program is not running as the + super-user (or if `--no-super` was specified), only groups that the + invoking user on the receiving side is a member of will be preserved. + Without this option, the group is set to the default group of the invoking + user on the receiving side. + + The preservation of group information will associate matching names by + default, but may fall back to using the ID number in some circumstances + (see also the `--numeric-ids` option for a full discussion). + +0. `--devices` + + This option causes rsync to transfer character and block device files to + the remote system to recreate these devices. This option has no effect if + the receiving rsync is not run as the super-user (see also the `--super` + and `--fake-super` options). + +0. `--specials` + + This option causes rsync to transfer special files such as named sockets + and fifos. + +0. `-D` + + The `-D` option is equivalent to `--devices --specials`. + +0. `--write-devices` + + This tells rsync to treat a device on the receiving side as a regular file, + allowing the writing of file data into a device. + + This option implies the `--inplace` option. + + Be careful using this, as you should know what devices are present on the + receiving side of the transfer, especially if running rsync as root. + + This option is refused by an rsync daemon. + +0. `--times`, `-t` + + This tells rsync to transfer modification times along with the files and + update them on the remote system. Note that if this option is not used, + the optimization that excludes files that have not been modified cannot be + effective; in other words, a missing `-t` or `-a` will cause the next + transfer to behave as if it used `-I`, causing all files to be updated + (though rsync's delta-transfer algorithm will make the update fairly + efficient if the files haven't actually changed, you're much better off + using `-t`). + +0. `--atimes`, `-U` + + This tells rsync to set the access (use) times of the destination files to + the same value as the source files. + + If repeated, it also sets the `--open-noatime` option, which can help you + to make the sending and receiving systems have the same access times on the + transferred files without needing to run rsync an extra time after a file + is transferred. + + Note that some older rsync versions (prior to 3.2.0) may have been built + with a pre-release `--atimes` patch that does not imply `--open-noatime` + when this option is repeated. + +0. `--open-noatime` + + This tells rsync to open files with the O_NOATIME flag (on systems that + support it) to avoid changing the access time of the files that are being + transferred. If your OS does not support the O_NOATIME flag then rsync + will silently ignore this option. Note also that some filesystems are + mounted to avoid updating the atime on read access even without the + O_NOATIME flag being set. + +0. `--omit-dir-times`, `-O` + + This tells rsync to omit directories when it is preserving modification + times (see `--times`). If NFS is sharing the directories on the receiving + side, it is a good idea to use `-O`. This option is inferred if you use + `--backup` without `--backup-dir`. + + This option also has the side-effect of avoiding early creation of + directories in incremental recursion copies. The default `--inc-recursive` + copying normally does an early-create pass of all the sub-directories in a + parent directory in order for it to be able to then set the modify time of + the parent directory right away (without having to delay that until a bunch + of recursive copying has finished). This early-create idiom is not + necessary if directory modify times are not being preserved, so it is + skipped. Since early-create directories don't have accurate mode, mtime, + or ownership, the use of this option can help when someone wants to avoid + these partially-finished directories. + +0. `--omit-link-times`, `-J` + + This tells rsync to omit symlinks when it is preserving modification times + (see `--times`). + +0. `--super` + + This tells the receiving side to attempt super-user activities even if the + receiving rsync wasn't run by the super-user. These activities include: + preserving users via the `--owner` option, preserving all groups (not just + the current user's groups) via the `--groups` option, and copying devices + via the `--devices` option. This is useful for systems that allow such + activities without being the super-user, and also for ensuring that you + will get errors if the receiving side isn't being run as the super-user. + To turn off super-user activities, the super-user can use `--no-super`. + +0. `--fake-super` + + When this option is enabled, rsync simulates super-user activities by + saving/restoring the privileged attributes via special extended attributes + that are attached to each file (as needed). This includes the file's owner + and group (if it is not the default), the file's device info (device & + special files are created as empty text files), and any permission bits + that we won't allow to be set on the real file (e.g. the real file gets + u-s,g-s,o-t for safety) or that would limit the owner's access (since the + real super-user can always access/change a file, the files we create can + always be accessed/changed by the creating user). This option also handles + ACLs (if `--acls` was specified) and non-user extended attributes (if + `--xattrs` was specified). + + This is a good way to backup data without using a super-user, and to store + ACLs from incompatible systems. + + The `--fake-super` option only affects the side where the option is used. + To affect the remote side of a remote-shell connection, use the + `--remote-option` (`-M`) option: + + > rsync -av -M--fake-super /src/ host:/dest/ + + For a local copy, this option affects both the source and the destination. + If you wish a local copy to enable this option just for the destination + files, specify `-M--fake-super`. If you wish a local copy to enable this + option just for the source files, combine `--fake-super` with `-M--super`. + + This option is overridden by both `--super` and `--no-super`. + + See also the "`fake super`" setting in the daemon's rsyncd.conf file. + +0. `--sparse`, `-S` + + Try to handle sparse files efficiently so they take up less space on the + destination. If combined with `--inplace` the file created might not end + up with sparse blocks with some combinations of kernel version and/or + filesystem type. If `--whole-file` is in effect (e.g. for a local copy) + then it will always work because rsync truncates the file prior to writing + out the updated version. + + Note that versions of rsync older than 3.1.3 will reject the combination of + `--sparse` and `--inplace`. + +0. `--preallocate` + + This tells the receiver to allocate each destination file to its eventual + size before writing data to the file. Rsync will only use the real + filesystem-level preallocation support provided by Linux's **fallocate**(2) + system call or Cygwin's **posix_fallocate**(3), not the slow glibc + implementation that writes a null byte into each block. + + Without this option, larger files may not be entirely contiguous on the + filesystem, but with this option rsync will probably copy more slowly. If + the destination is not an extent-supporting filesystem (such as ext4, xfs, + NTFS, etc.), this option may have no positive effect at all. + + If combined with `--sparse`, the file will only have sparse blocks (as + opposed to allocated sequences of null bytes) if the kernel version and + filesystem type support creating holes in the allocated data. + +0. `--dry-run`, `-n` + + This makes rsync perform a trial run that doesn't make any changes (and + produces mostly the same output as a real run). It is most commonly used + in combination with the `--verbose`, `-v` and/or `--itemize-changes`, `-i` + options to see what an rsync command is going to do before one actually + runs it. + + The output of `--itemize-changes` is supposed to be exactly the same on a + dry run and a subsequent real run (barring intentional trickery and system + call failures); if it isn't, that's a bug. Other output should be mostly + unchanged, but may differ in some areas. Notably, a dry run does not send + the actual data for file transfers, so `--progress` has no effect, the + "bytes sent", "bytes received", "literal data", and "matched data" + statistics are too small, and the "speedup" value is equivalent to a run + where no file transfers were needed. + +0. `--whole-file`, `-W` + + This option disables rsync's delta-transfer algorithm, which causes all + transferred files to be sent whole. The transfer may be faster if this + option is used when the bandwidth between the source and destination + machines is higher than the bandwidth to disk (especially when the "disk" + is actually a networked filesystem). This is the default when both the + source and destination are specified as local paths, but only if no + batch-writing option is in effect. + +0. `--checksum-choice=STR`, `--cc=STR` + + This option overrides the checksum algorithms. If one algorithm name is + specified, it is used for both the transfer checksums and (assuming + `--checksum` is specified) the pre-transfer checksums. If two + comma-separated names are supplied, the first name affects the transfer + checksums, and the second name affects the pre-transfer checksums (`-c`). + + The checksum options that you may be able to use are: + + - `auto` (the default) + - `xxh64` (aka xxhash) + - `md5` + - `md4` + - `none` + + Run `rsync -V` to see the default checksum list compiled into your version. + + If "none" is specified for the first (or only) name, the `--whole-file` + option is forced on and no checksum verification is performed on the + transferred data. If "none" is specified for the second (or only) name, + the `--checksum` option cannot be used. + + The "auto" option is the default, where rsync bases its algorithm choice on + a negotiation between the client and the server as follows: + + If both the client and the server are at least version 3.2.0, they will + exchange a list of checksum names and choose the first one in the list that + they have in common. This typically means that they will choose xxh64 if + they both support it and fall back to MD5. If one side of the transfer is + not new enough to support this checksum negotiation, then a value is chosen + based on the protocol version (which chooses between MD5 and various + flavors of MD4 based on protocol age). + + You can also override the checksum using the RSYNC_CHECKSUM_LIST + environment variable by setting it to a space-separated list of checksum + names that you consider acceptable. If no common checksum is found, the + client exits with an error. This method does not allow you to specify the + transfer checksum separately from the pre-transfer checksum, and it ignores + "auto" and all unknown checksum names. If the remote rsync is not new + enough to handle a checksum negotiation list, the list is silently ignored + unless it contains the string "FAIL". + + The use of the `--checksum-choice` option overrides this environment list. + +0. `--one-file-system`, `-x` + + This tells rsync to avoid crossing a filesystem boundary when recursing. + This does not limit the user's ability to specify items to copy from + multiple filesystems, just rsync's recursion through the hierarchy of each + directory that the user specified, and also the analogous recursion on the + receiving side during deletion. Also keep in mind that rsync treats a + "bind" mount to the same device as being on the same filesystem. + + If this option is repeated, rsync omits all mount-point directories from + the copy. Otherwise, it includes an empty directory at each mount-point it + encounters (using the attributes of the mounted directory because those of + the underlying mount-point directory are inaccessible). + + If rsync has been told to collapse symlinks (via `--copy-links` or + `--copy-unsafe-links`), a symlink to a directory on another device is + treated like a mount-point. Symlinks to non-directories are unaffected by + this option. + +0. `--existing`, `--ignore-non-existing` + + This tells rsync to skip creating files (including directories) that do not + exist yet on the destination. If this option is combined with the + `--ignore-existing` option, no files will be updated (which can be useful + if all you want to do is delete extraneous files). + + This option is a transfer rule, not an exclude, so it doesn't affect the + data that goes into the file-lists, and thus it doesn't affect deletions. + It just limits the files that the receiver requests to be transferred. + +0. `--ignore-existing` + + This tells rsync to skip updating files that already exist on the + destination (this does _not_ ignore existing directories, or nothing would + get done). See also `--existing`. + + This option is a transfer rule, not an exclude, so it doesn't affect the + data that goes into the file-lists, and thus it doesn't affect deletions. + It just limits the files that the receiver requests to be transferred. + + This option can be useful for those doing backups using the `--link-dest` + option when they need to continue a backup run that got interrupted. Since + a `--link-dest` run is copied into a new directory hierarchy (when it is + used properly), using `--ignore-existing` will ensure that the + already-handled files don't get tweaked (which avoids a change in + permissions on the hard-linked files). This does mean that this option is + only looking at the existing files in the destination hierarchy itself. + +0. `--remove-source-files` + + This tells rsync to remove from the sending side the files (meaning + non-directories) that are a part of the transfer and have been successfully + duplicated on the receiving side. + + Note that you should only use this option on source files that are + quiescent. If you are using this to move files that show up in a + particular directory over to another host, make sure that the finished + files get renamed into the source directory, not directly written into it, + so that rsync can't possibly transfer a file that is not yet fully written. + If you can't first write the files into a different directory, you should + use a naming idiom that lets rsync avoid transferring files that are not + yet finished (e.g. name the file "foo.new" when it is written, rename it to + "foo" when it is done, and then use the option `--exclude='*.new'` for the + rsync transfer). + + Starting with 3.1.0, rsync will skip the sender-side removal (and output an + error) if the file's size or modify time has not stayed unchanged. + +0. `--delete` + + This tells rsync to delete extraneous files from the receiving side (ones + that aren't on the sending side), but only for the directories that are + being synchronized. You must have asked rsync to send the whole directory + (e.g. "`dir`" or "`dir/`") without using a wildcard for the directory's + contents (e.g. "`dir/*`") since the wildcard is expanded by the shell and + rsync thus gets a request to transfer individual files, not the files' + parent directory. Files that are excluded from the transfer are also + excluded from being deleted unless you use the `--delete-excluded` option + or mark the rules as only matching on the sending side (see the + include/exclude modifiers in the FILTER RULES section). + + Prior to rsync 2.6.7, this option would have no effect unless `--recursive` + was enabled. Beginning with 2.6.7, deletions will also occur when `--dirs` + (`-d`) is enabled, but only for directories whose contents are being + copied. + + This option can be dangerous if used incorrectly! It is a very good idea to + first try a run using the `--dry-run` option (`-n`) to see what files are + going to be deleted. + + If the sending side detects any I/O errors, then the deletion of any files + at the destination will be automatically disabled. This is to prevent + temporary filesystem failures (such as NFS errors) on the sending side from + causing a massive deletion of files on the destination. You can override + this with the `--ignore-errors` option. + + The `--delete` option may be combined with one of the --delete-WHEN options + without conflict, as well as `--delete-excluded`. However, if none of the + `--delete-WHEN` options are specified, rsync will choose the + `--delete-during` algorithm when talking to rsync 3.0.0 or newer, and the + `--delete-before` algorithm when talking to an older rsync. See also + `--delete-delay` and `--delete-after`. + +0. `--delete-before` + + Request that the file-deletions on the receiving side be done before the + transfer starts. See `--delete` (which is implied) for more details on + file-deletion. + + Deleting before the transfer is helpful if the filesystem is tight for + space and removing extraneous files would help to make the transfer + possible. However, it does introduce a delay before the start of the + transfer, and this delay might cause the transfer to timeout (if + `--timeout` was specified). It also forces rsync to use the old, + non-incremental recursion algorithm that requires rsync to scan all the + files in the transfer into memory at once (see `--recursive`). + +0. `--delete-during`, `--del` + + Request that the file-deletions on the receiving side be done incrementally + as the transfer happens. The per-directory delete scan is done right + before each directory is checked for updates, so it behaves like a more + efficient `--delete-before`, including doing the deletions prior to any + per-directory filter files being updated. This option was first added in + rsync version 2.6.4. See `--delete` (which is implied) for more details on + file-deletion. + +0. `--delete-delay` + + Request that the file-deletions on the receiving side be computed during + the transfer (like `--delete-during`), and then removed after the transfer + completes. This is useful when combined with `--delay-updates` and/or + `--fuzzy`, and is more efficient than using `--delete-after` (but can + behave differently, since `--delete-after` computes the deletions in a + separate pass after all updates are done). If the number of removed files + overflows an internal buffer, a temporary file will be created on the + receiving side to hold the names (it is removed while open, so you + shouldn't see it during the transfer). If the creation of the temporary + file fails, rsync will try to fall back to using `--delete-after` (which it + cannot do if `--recursive` is doing an incremental scan). See `--delete` + (which is implied) for more details on file-deletion. + +0. `--delete-after` + + Request that the file-deletions on the receiving side be done after the + transfer has completed. This is useful if you are sending new + per-directory merge files as a part of the transfer and you want their + exclusions to take effect for the delete phase of the current transfer. It + also forces rsync to use the old, non-incremental recursion algorithm that + requires rsync to scan all the files in the transfer into memory at once + (see `--recursive`). See `--delete` (which is implied) for more details on + file-deletion. + +0. `--delete-excluded` + + In addition to deleting the files on the receiving side that are not on the + sending side, this tells rsync to also delete any files on the receiving + side that are excluded (see `--exclude`). See the FILTER RULES section for + a way to make individual exclusions behave this way on the receiver, and + for a way to protect files from `--delete-excluded`. See `--delete` (which + is implied) for more details on file-deletion. + +0. `--ignore-missing-args` + + When rsync is first processing the explicitly requested source files (e.g. + command-line arguments or `--files-from` entries), it is normally an error + if the file cannot be found. This option suppresses that error, and does + not try to transfer the file. This does not affect subsequent + vanished-file errors if a file was initially found to be present and later + is no longer there. + +0. `--delete-missing-args` + + This option takes the behavior of (the implied) `--ignore-missing-args` + option a step farther: each missing arg will become a deletion request of + the corresponding destination file on the receiving side (should it exist). + If the destination file is a non-empty directory, it will only be + successfully deleted if `--force` or `--delete` are in effect. Other than + that, this option is independent of any other type of delete processing. + + The missing source files are represented by special file-list entries which + display as a "`*missing`" entry in the `--list-only` output. + +0. `--ignore-errors` + + Tells `--delete` to go ahead and delete files even when there are I/O + errors. + +0. `--force` + + This option tells rsync to delete a non-empty directory when it is to be + replaced by a non-directory. This is only relevant if deletions are not + active (see `--delete` for details). + + Note for older rsync versions: `--force` used to still be required when + using `--delete-after`, and it used to be non-functional unless the + `--recursive` option was also enabled. + +0. `--max-delete=NUM` + + This tells rsync not to delete more than NUM files or directories. If that + limit is exceeded, all further deletions are skipped through the end of the + transfer. At the end, rsync outputs a warning (including a count of the + skipped deletions) and exits with an error code of 25 (unless some more + important error condition also occurred). + + Beginning with version 3.0.0, you may specify `--max-delete=0` to be warned + about any extraneous files in the destination without removing any of them. + Older clients interpreted this as "unlimited", so if you don't know what + version the client is, you can use the less obvious `--max-delete=-1` as a + backward-compatible way to specify that no deletions be allowed (though + really old versions didn't warn when the limit was exceeded). + +0. `--max-size=SIZE` + + This tells rsync to avoid transferring any file that is larger than the + specified SIZE. The SIZE value can be suffixed with a string to indicate a + size multiplier, and may be a fractional value (e.g. `--max-size=1.5m`). + + This option is a transfer rule, not an exclude, so it doesn't affect the + data that goes into the file-lists, and thus it doesn't affect deletions. + It just limits the files that the receiver requests to be transferred. + + The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or + "MiB") is a mebibyte (1024\*1024), and "G" (or "GiB") is a gibibyte + (1024\*1024\*1024). If you want the multiplier to be 1000 instead of 1024, + use "KB", "MB", or "GB". (Note: lower-case is also accepted for all + values.) Finally, if the suffix ends in either "+1" or "-1", the value will + be offset by one byte in the indicated direction. + + Examples: `--max-size=1.5mb-1` is 1499999 bytes, and `--max-size=2g+1` is + 2147483649 bytes. + + Note that rsync versions prior to 3.1.0 did not allow `--max-size=0`. + +0. `--min-size=SIZE` + + This tells rsync to avoid transferring any file that is smaller than the + specified SIZE, which can help in not transferring small, junk files. See + the `--max-size` option for a description of SIZE and other information. + + Note that rsync versions prior to 3.1.0 did not allow `--min-size=0`. + +0. `--block-size=BLOCKSIZE`, `-B` + + This forces the block size used in rsync's delta-transfer algorithm to a + fixed value. It is normally selected based on the size of each file being + updated. See the technical report for details. + +0. `--rsh=COMMAND`, `-e` + + This option allows you to choose an alternative remote shell program to use + for communication between the local and remote copies of rsync. Typically, + rsync is configured to use ssh by default, but you may prefer to use rsh on + a local network. + + If this option is used with `[user@]host::module/path`, then the remote + shell _COMMAND_ will be used to run an rsync daemon on the remote host, and + all data will be transmitted through that remote shell connection, rather + than through a direct socket connection to a running rsync daemon on the + remote host. See the section "USING RSYNC-DAEMON FEATURES VIA A + REMOTE-SHELL CONNECTION" above. + + Beginning with rsync 3.2.0, the RSYNC_PORT environment variable will be set + when a daemon connection is being made via a remote-shell connection. It + is set to 0 if the default daemon port is being assumed, or it is set to + the value of the rsync port that was specified via either the `--port` + option or a non-empty port value in an rsync:// URL. This allows the + script to discern if a non-default port is being requested, allowing for + things such as an SSL or stunnel helper script to connect to a default or + alternate port. + + Command-line arguments are permitted in COMMAND provided that COMMAND is + presented to rsync as a single argument. You must use spaces (not tabs or + other whitespace) to separate the command and args from each other, and you + can use single- and/or double-quotes to preserve spaces in an argument (but + not backslashes). Note that doubling a single-quote inside a single-quoted + string gives you a single-quote; likewise for double-quotes (though you + need to pay attention to which quotes your shell is parsing and which + quotes rsync is parsing). Some examples: + + > -e 'ssh -p 2234' + > -e 'ssh -o "ProxyCommand nohup ssh firewall nc -w1 %h %p"' + + (Note that ssh users can alternately customize site-specific connect + options in their .ssh/config file.) + + You can also choose the remote shell program using the RSYNC_RSH + environment variable, which accepts the same range of values as `-e`. + + See also the `--blocking-io` option which is affected by this option. + +0. `--rsync-path=PROGRAM` + + Use this to specify what program is to be run on the remote machine to + start-up rsync. Often used when rsync is not in the default remote-shell's + path (e.g. `--rsync-path=/usr/local/bin/rsync`). Note that PROGRAM is run + with the help of a shell, so it can be any program, script, or command + sequence you'd care to run, so long as it does not corrupt the standard-in + & standard-out that rsync is using to communicate. + + One tricky example is to set a different default directory on the remote + machine for use with the `--relative` option. For instance: + + > rsync -avR --rsync-path="cd /a/b && rsync" host:c/d /e/ + +0. `--remote-option=OPTION`, `-M` + + This option is used for more advanced situations where you want certain + effects to be limited to one side of the transfer only. For instance, if + you want to pass `--log-file=FILE` and `--fake-super` to the remote system, + specify it like this: + + > rsync -av -M --log-file=foo -M--fake-super src/ dest/ + + If you want to have an option affect only the local side of a transfer when + it normally affects both sides, send its negation to the remote side. Like + this: + + > rsync -av -x -M--no-x src/ dest/ + + Be cautious using this, as it is possible to toggle an option that will + cause rsync to have a different idea about what data to expect next over + the socket, and that will make it fail in a cryptic fashion. + + Note that it is best to use a separate `--remote-option` for each option + you want to pass. This makes your usage compatible with the + `--protect-args` option. If that option is off, any spaces in your remote + options will be split by the remote shell unless you take steps to protect + them. + + When performing a local transfer, the "local" side is the sender and the + "remote" side is the receiver. + + Note some versions of the popt option-parsing library have a bug in them + that prevents you from using an adjacent arg with an equal in it next to a + short option letter (e.g. `-M--log-file=/tmp/foo`). If this bug affects + your version of popt, you can use the version of popt that is included with + rsync. + +0. `--cvs-exclude`, `-C` + + This is a useful shorthand for excluding a broad range of files that you + often don't want to transfer between systems. It uses a similar algorithm + to CVS to determine if a file should be ignored. + + The exclude list is initialized to exclude the following items (these + initial items are marked as perishable -- see the FILTER RULES section): + + [comment]: # (This list gets used for the default-cvsignore.h file.) + + > `RCS` + > `SCCS` + > `CVS` + > `CVS.adm` + > `RCSLOG` + > `cvslog.*` + > `tags` + > `TAGS` + > `.make.state` + > `.nse_depinfo` + > `*~` + > `#*` + > `.#*` + > `,*` + > `_$*` + > `*$` + > `*.old` + > `*.bak` + > `*.BAK` + > `*.orig` + > `*.rej` + > `.del-*` + > `*.a` + > `*.olb` + > `*.o` + > `*.obj` + > `*.so` + > `*.exe` + > `*.Z` + > `*.elc` + > `*.ln` + > `core` + > `.svn/` + > `.git/` + > `.hg/` + > `.bzr/` + + then, files listed in a $HOME/.cvsignore are added to the list and any + files listed in the CVSIGNORE environment variable (all cvsignore names are + delimited by whitespace). + + Finally, any file is ignored if it is in the same directory as a .cvsignore + file and matches one of the patterns listed therein. Unlike rsync's + filter/exclude files, these patterns are split on whitespace. See the + **cvs**(1) manual for more information. + + If you're combining `-C` with your own `--filter` rules, you should note + that these CVS excludes are appended at the end of your own rules, + regardless of where the `-C` was placed on the command-line. This makes + them a lower priority than any rules you specified explicitly. If you want + to control where these CVS excludes get inserted into your filter rules, + you should omit the `-C` as a command-line option and use a combination of + `--filter=:C` and `--filter=-C` (either on your command-line or by putting + the ":C" and "-C" rules into a filter file with your other rules). The + first option turns on the per-directory scanning for the .cvsignore file. + The second option does a one-time import of the CVS excludes mentioned + above. + +0. `--filter=RULE`, `-f` + + This option allows you to add rules to selectively exclude certain files + from the list of files to be transferred. This is most useful in + combination with a recursive transfer. + + You may use as many `--filter` options on the command line as you like to + build up the list of files to exclude. If the filter contains whitespace, + be sure to quote it so that the shell gives the rule to rsync as a single + argument. The text below also mentions that you can use an underscore to + replace the space that separates a rule from its arg. + + See the FILTER RULES section for detailed information on this option. + +0. `-F` + + The `-F` option is a shorthand for adding two `--filter` rules to your + command. The first time it is used is a shorthand for this rule: + + > --filter='dir-merge /.rsync-filter' + + This tells rsync to look for per-directory .rsync-filter files that have + been sprinkled through the hierarchy and use their rules to filter the + files in the transfer. If `-F` is repeated, it is a shorthand for this + rule: + + > --filter='exclude .rsync-filter' + + This filters out the .rsync-filter files themselves from the transfer. + + See the FILTER RULES section for detailed information on how these options + work. + +0. `--exclude=PATTERN` + + This option is a simplified form of the `--filter` option that defaults to + an exclude rule and does not allow the full rule-parsing syntax of normal + filter rules. + + See the FILTER RULES section for detailed information on this option. + +0. `--exclude-from=FILE` + + This option is related to the `--exclude` option, but it specifies a FILE + that contains exclude patterns (one per line). Blank lines in the file and + lines starting with '`;`' or '`#`' are ignored. If _FILE_ is '`-`', the + list will be read from standard input. + +0. `--include=PATTERN` + + This option is a simplified form of the `--filter` option that defaults to + an include rule and does not allow the full rule-parsing syntax of normal + filter rules. + + See the FILTER RULES section for detailed information on this option. + +0. `--include-from=FILE` + + This option is related to the `--include` option, but it specifies a FILE + that contains include patterns (one per line). Blank lines in the file and + lines starting with '`;`' or '`#`' are ignored. If _FILE_ is '`-`', the + list will be read from standard input. + +0. `--files-from=FILE` + + Using this option allows you to specify the exact list of files to transfer + (as read from the specified FILE or '`-`' for standard input). It also + tweaks the default behavior of rsync to make transferring just the + specified files and directories easier: + + - The `--relative` (`-R`) option is implied, which preserves the path + information that is specified for each item in the file (use + `--no-relative` or `--no-R` if you want to turn that off). + - The `--dirs` (`-d`) option is implied, which will create directories + specified in the list on the destination rather than noisily skipping + them (use `--no-dirs` or `--no-d` if you want to turn that off). + - The `--archive` (`-a`) option's behavior does not imply `--recursive` + (`-r`), so specify it explicitly, if you want it. + - These side-effects change the default state of rsync, so the position of + the `--files-from` option on the command-line has no bearing on how other + options are parsed (e.g. `-a` works the same before or after + `--files-from`, as does `--no-R` and all other options). + + The filenames that are read from the FILE are all relative to the source + dir -- any leading slashes are removed and no ".." references are allowed + to go higher than the source dir. For example, take this command: + + > rsync -a --files-from=/tmp/foo /usr remote:/backup + + If /tmp/foo contains the string "bin" (or even "/bin"), the /usr/bin + directory will be created as /backup/bin on the remote host. If it + contains "bin/" (note the trailing slash), the immediate contents of the + directory would also be sent (without needing to be explicitly mentioned in + the file -- this began in version 2.6.4). In both cases, if the `-r` + option was enabled, that dir's entire hierarchy would also be transferred + (keep in mind that `-r` needs to be specified explicitly with + `--files-from`, since it is not implied by `-a`). Also note that the + effect of the (enabled by default) `--relative` option is to duplicate only + the path info that is read from the file -- it does not force the + duplication of the source-spec path (/usr in this case). + + In addition, the `--files-from` file can be read from the remote host + instead of the local host if you specify a "host:" in front of the file + (the host must match one end of the transfer). As a short-cut, you can + specify just a prefix of ":" to mean "use the remote end of the transfer". + For example: + + > rsync -a --files-from=:/path/file-list src:/ /tmp/copy + + This would copy all the files specified in the /path/file-list file that + was located on the remote "src" host. + + If the `--iconv` and `--protect-args` options are specified and the + `--files-from` filenames are being sent from one host to another, the + filenames will be translated from the sending host's charset to the + receiving host's charset. + + NOTE: sorting the list of files in the `--files-from` input helps rsync to + be more efficient, as it will avoid re-visiting the path elements that are + shared between adjacent entries. If the input is not sorted, some path + elements (implied directories) may end up being scanned multiple times, and + rsync will eventually unduplicate them after they get turned into file-list + elements. + +0. `--from0`, `-0` + + This tells rsync that the rules/filenames it reads from a file are + terminated by a null ('\\0') character, not a NL, CR, or CR+LF. This + affects `--exclude-from`, `--include-from`, `--files-from`, and any merged + files specified in a `--filter` rule. It does not affect `--cvs-exclude` + (since all names read from a .cvsignore file are split on whitespace). + +0. `--protect-args`, `-s` + + This option sends all filenames and most options to the remote rsync + without allowing the remote shell to interpret them. This means that + spaces are not split in names, and any non-wildcard special characters are + not translated (such as `~`, `$`, `;`, `&`, etc.). Wildcards are expanded + on the remote host by rsync (instead of the shell doing it). + + If you use this option with `--iconv`, the args related to the remote side + will also be translated from the local to the remote character-set. The + translation happens before wild-cards are expanded. See also the + `--files-from` option. + + You may also control this option via the RSYNC_PROTECT_ARGS environment + variable. If this variable has a non-zero value, this option will be + enabled by default, otherwise it will be disabled by default. Either state + is overridden by a manually specified positive or negative version of this + option (note that `--no-s` and `--no-protect-args` are the negative + versions). Since this option was first introduced in 3.0.0, you'll need to + make sure it's disabled if you ever need to interact with a remote rsync + that is older than that. + + Rsync can also be configured (at build time) to have this option enabled by + default (with is overridden by both the environment and the command-line). + This option will eventually become a new default setting at some + as-yet-undetermined point in the future. + +0. `--copy-as=USER[:GROUP]` + + This option instructs rsync to use the USER and (if specified after a + colon) the GROUP for the copy operations. This only works if the user that + is running rsync has the ability to change users. If the group is not + specified then the user's default groups are used. + + This option can help to reduce the risk of an rsync being run as root into + or out of a directory that might have live changes happening to it and you + want to make sure that root-level read or write actions of system files are + not possible. While you could alternatively run all of rsync as the + specified user, sometimes you need the root-level host-access credentials + to be used, so this allows rsync to drop root for the copying part of the + operation after the remote-shell or daemon connection is established. + + The option only affects one side of the transfer unless the transfer is + local, in which case it affects both sides. Use the `--remote-option` to + affect the remote side, such as `-M--copy-as=joe`. For a local transfer, + the lsh (or lsh.sh) support file provides a local-shell helper script that + can be used to allow a "localhost:" or "lh:" host-spec to be specified + without needing to setup any remote shells, allowing you to specify remote + options that affect the side of the transfer that is using the host-spec + (and using hostname "lh" avoids the overriding of the remote directory to + the user's home dir). + + For example, the following rsync writes the local files as user "joe": + + > sudo rsync -aiv --copy-as=joe host1:backups/joe/ /home/joe/ + + This makes all files owned by user "joe", limits the groups to those that + are available to that user, and makes it impossible for the joe user to do + a timed exploit of the path to induce a change to a file that the joe user + has no permissions to change. + + The following command does a local copy into the "dest/" dir as user "joe" + (assumimg you've installed support/lsh into a dir on your $PATH): + + > sudo rsync -aive lsh -M--copy-as=joe src/ lh:dest/ + +0. `--temp-dir=DIR`, `-T` + + This option instructs rsync to use DIR as a scratch directory when creating + temporary copies of the files transferred on the receiving side. The + default behavior is to create each temporary file in the same directory as + the associated destination file. Beginning with rsync 3.1.1, the temp-file + names inside the specified DIR will not be prefixed with an extra dot + (though they will still have a random suffix added). + + This option is most often used when the receiving disk partition does not + have enough free space to hold a copy of the largest file in the transfer. + In this case (i.e. when the scratch directory is on a different disk + partition), rsync will not be able to rename each received temporary file + over the top of the associated destination file, but instead must copy it + into place. Rsync does this by copying the file over the top of the + destination file, which means that the destination file will contain + truncated data during this copy. If this were not done this way (even if + the destination file were first removed, the data locally copied to a + temporary file in the destination directory, and then renamed into place) + it would be possible for the old file to continue taking up disk space (if + someone had it open), and thus there might not be enough room to fit the + new version on the disk at the same time. + + If you are using this option for reasons other than a shortage of disk + space, you may wish to combine it with the `--delay-updates` option, which + will ensure that all copied files get put into subdirectories in the + destination hierarchy, awaiting the end of the transfer. If you don't have + enough room to duplicate all the arriving files on the destination + partition, another way to tell rsync that you aren't overly concerned about + disk space is to use the `--partial-dir` option with a relative path; + because this tells rsync that it is OK to stash off a copy of a single file + in a subdir in the destination hierarchy, rsync will use the partial-dir as + a staging area to bring over the copied file, and then rename it into place + from there. (Specifying a `--partial-dir` with an absolute path does not + have this side-effect.) + +0. `--fuzzy`, `-y` + + This option tells rsync that it should look for a basis file for any + destination file that is missing. The current algorithm looks in the same + directory as the destination file for either a file that has an identical + size and modified-time, or a similarly-named file. If found, rsync uses + the fuzzy basis file to try to speed up the transfer. + + If the option is repeated, the fuzzy scan will also be done in any matching + alternate destination directories that are specified via `--compare-dest`, + `--copy-dest`, or `--link-dest`. + + Note that the use of the `--delete` option might get rid of any potential + fuzzy-match files, so either use `--delete-after` or specify some filename + exclusions if you need to prevent this. + +0. `--compare-dest=DIR` + + This option instructs rsync to use _DIR_ on the destination machine as an + additional hierarchy to compare destination files against doing transfers + (if the files are missing in the destination directory). If a file is + found in _DIR_ that is identical to the sender's file, the file will NOT be + transferred to the destination directory. This is useful for creating a + sparse backup of just files that have changed from an earlier backup. This + option is typically used to copy into an empty (or newly created) + directory. + + Beginning in version 2.6.4, multiple `--compare-dest` directories may be + provided, which will cause rsync to search the list in the order specified + for an exact match. If a match is found that differs only in attributes, a + local copy is made and the attributes updated. If a match is not found, a + basis file from one of the _DIRs_ will be selected to try to speed up the + transfer. + + If _DIR_ is a relative path, it is relative to the destination directory. + See also `--copy-dest` and `--link-dest`. + + NOTE: beginning with version 3.1.0, rsync will remove a file from a + non-empty destination hierarchy if an exact match is found in one of the + compare-dest hierarchies (making the end result more closely match a fresh + copy). + +0. `--copy-dest=DIR` + + This option behaves like `--compare-dest`, but rsync will also copy + unchanged files found in _DIR_ to the destination directory using a local + copy. This is useful for doing transfers to a new destination while + leaving existing files intact, and then doing a flash-cutover when all + files have been successfully transferred. + + Multiple `--copy-dest` directories may be provided, which will cause rsync + to search the list in the order specified for an unchanged file. If a + match is not found, a basis file from one of the _DIRs_ will be selected to + try to speed up the transfer. + + If _DIR_ is a relative path, it is relative to the destination directory. + See also `--compare-dest` and `--link-dest`. + +0. `--link-dest=DIR` + + This option behaves like `--copy-dest`, but unchanged files are hard linked + from _DIR_ to the destination directory. The files must be identical in + all preserved attributes (e.g. permissions, possibly ownership) in order + for the files to be linked together. An example: + + > rsync -av --link-dest=$PWD/prior_dir host:src_dir/ new_dir/ + + If file's aren't linking, double-check their attributes. Also check if + some attributes are getting forced outside of rsync's control, such a mount + option that squishes root to a single user, or mounts a removable drive + with generic ownership (such as OS X's "Ignore ownership on this volume" + option). + + Beginning in version 2.6.4, multiple `--link-dest` directories may be + provided, which will cause rsync to search the list in the order specified + for an exact match (there is a limit of 20 such directories). If a match + is found that differs only in attributes, a local copy is made and the + attributes updated. If a match is not found, a basis file from one of the + _DIRs_ will be selected to try to speed up the transfer. + + This option works best when copying into an empty destination hierarchy, as + existing files may get their attributes tweaked, and that can affect + alternate destination files via hard-links. Also, itemizing of changes can + get a bit muddled. Note that prior to version 3.1.0, an + alternate-directory exact match would never be found (nor linked into the + destination) when a destination file already exists. + + Note that if you combine this option with `--ignore-times`, rsync will not + link any files together because it only links identical files together as a + substitute for transferring the file, never as an additional check after + the file is updated. + + If _DIR_ is a relative path, it is relative to the destination directory. + See also `--compare-dest` and `--copy-dest`. + + Note that rsync versions prior to 2.6.1 had a bug that could prevent + `--link-dest` from working properly for a non-super-user when `-o` was + specified (or implied by `-a`). You can work-around this bug by avoiding + the `-o` option when sending to an old rsync. + +0. `--compress`, `-z` + + With this option, rsync compresses the file data as it is sent to the + destination machine, which reduces the amount of data being transmitted -- + something that is useful over a slow connection. + + The "zlib" compression method typically achieves better compression ratios + than can be achieved by using a compressing remote shell or a compressing + transport because it takes advantage of the implicit information in the + matching data blocks that are not explicitly sent over the connection. + This matching-data compression comes at a cost of CPU, though, and can be + disabled by using the "zlibx" compresson method instead. This can be + selected by repeating the `-z` option or specifying + `--compress-choice=zlibx`, but it only works if both sides of the transfer + are at least version 3.1.1. + + Note that if you see an error about an option named `--old-compress` or + `--new-compress`, this is rsync trying to send the `--compress-choice=zlib` + or `--compress-choice=zlibx` option in a backward-compatible manner that + more rsync versions understand. This error indicates that the older rsync + version will not allow you to force the compression type. + + See the `--skip-compress` option for the default list of file suffixes that + will not be compressed. + +0. `--compress-choice=STR`, `--zc=STR` + + This option can be used to override the automatic selection of the + compression algorithm that is the default when `--compress` is used. + + The compression options that you may be able to use are: + + - `zstd` + - `lz4` + - `zlibx` + - `zlib` + - `none` + + Run `rsync -V` to see the compress list compiled into your version. + + The "zlibx" algorithm is given preference over "zlib" if both sides of the + transfer are at least version 3.2.0, otherwise it will choose "zlib" unless + you override it via something like `-zz`. These 2 algorithms are the stame + except that "zlibx" does not try to include matched data that was not + transferred in the compression computations. + + If "none" is specified, that is equivalent to using `--no-compress`. + + This option implies `--compress` unless "none" was specified. + + You can also override the compression negotiation using the + RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated + list of compression names that you consider acceptable. If no common + compress choice is found, the client exits with an error. It ignores + "auto" and all unknown compression names. If the remote rsync is not new + enough to handle a compression negotiation list, the list is silently + ignored unless it contains the string "FAIL". + +0. `--compress-level=NUM` + + Explicitly set the compression level to use (see `--compress`) instead of + letting it default. If NUM is non-zero, the `--compress` option is + implied. + +0. `--skip-compress=LIST` + + Override the list of file suffixes that will be compressed as little as + possible. Rsync sets the compression level on a per-file basis based on + the file's suffix. If the compression algorithm has an "off" level (such + as zlib/zlibx) then no compression occurs for those files. Other + algorithms have the level minimized to reduces the CPU usage as much as + possible. + + The **LIST** should be one or more file suffixes (without the dot) separated + by slashes (`/`). You may specify an empty string to indicate that no files + should be skipped. + + Simple character-class matching is supported: each must consist of a list + of letters inside the square brackets (e.g. no special classes, such as + "[:alpha:]", are supported, and '-' has no special meaning). + + The characters asterisk (`*`) and question-mark (`?`) have no special meaning. + + Here's an example that specifies 6 suffixes to skip (since 1 of the 5 rules + matches 2 suffixes): + + > --skip-compress=gz/jpg/mp[34]/7z/bz2 + + The default file suffixes in the skip-compress list in this version of + rsync are: + + [comment]: # (This list gets used for the default-dont-compress.h file.) + + > 7z + > ace + > apk + > avi + > bz2 + > deb + > flac + > gpg + > gz + > iso + > jar + > jpeg + > jpg + > lz + > lz4 + > lzma + > lzo + > mkv + > mov + > mp3 + > mp4 + > odb + > odf + > odg + > odi + > odm + > odp + > ods + > odt + > ogg + > ogv + > opus + > otg + > oth + > otp + > ots + > ott + > oxt + > png + > rar + > rpm + > rz + > rzip + > squashfs + > sxc + > sxd + > sxg + > sxm + > sxw + > tbz + > tgz + > tlz + > txz + > tzo + > webm + > webp + > xz + > z + > zip + > zst + + This list will be replaced by your `--skip-compress` list in all but one + situation: a copy from a daemon rsync will add your skipped suffixes to its + list of non-compressing files (and its list may be configured to a + different default). + +0. `--numeric-ids` + + With this option rsync will transfer numeric group and user IDs rather than + using user and group names and mapping them at both ends. + + By default rsync will use the username and groupname to determine what + ownership to give files. The special uid 0 and the special group 0 are + never mapped via user/group names even if the `--numeric-ids` option is not + specified. + + If a user or group has no name on the source system or it has no match on + the destination system, then the numeric ID from the source system is used + instead. See also the comments on the "`use chroot`" setting in the + rsyncd.conf manpage for information on how the chroot setting affects + rsync's ability to look up the names of the users and groups and what you + can do about it. + +0. `--usermap=STRING`, `--groupmap=STRING` + + These options allow you to specify users and groups that should be mapped + to other values by the receiving side. The **STRING** is one or more + **FROM**:**TO** pairs of values separated by commas. Any matching **FROM** + value from the sender is replaced with a **TO** value from the receiver. + You may specify usernames or user IDs for the **FROM** and **TO** values, + and the **FROM** value may also be a wild-card string, which will be + matched against the sender's names (wild-cards do NOT match against ID + numbers, though see below for why a '`*`' matches everything). You may + instead specify a range of ID numbers via an inclusive range: LOW-HIGH. + For example: + + > --usermap=0-99:nobody,wayne:admin,*:normal --groupmap=usr:1,1:usr + + The first match in the list is the one that is used. You should specify + all your user mappings using a single `--usermap` option, and/or all your + group mappings using a single `--groupmap` option. + + Note that the sender's name for the 0 user and group are not transmitted to + the receiver, so you should either match these values using a 0, or use the + names in effect on the receiving side (typically "root"). All other + **FROM** names match those in use on the sending side. All **TO** names + match those in use on the receiving side. + + Any IDs that do not have a name on the sending side are treated as having + an empty name for the purpose of matching. This allows them to be matched + via a "`*`" or using an empty name. For instance: + + > --usermap=:nobody --groupmap=*:nobody + + When the `--numeric-ids` option is used, the sender does not send any + names, so all the IDs are treated as having an empty name. This means that + you will need to specify numeric **FROM** values if you want to map these + nameless IDs to different values. + + For the `--usermap` option to have any effect, the `-o` (`--owner`) option + must be used (or implied), and the receiver will need to be running as a + super-user (see also the `--fake-super` option). For the `--groupmap` + option to have any effect, the `-g` (`--groups`) option must be used (or + implied), and the receiver will need to have permissions to set that group. + +0. `--chown=USER:GROUP` + + This option forces all files to be owned by USER with group GROUP. This is + a simpler interface than using `--usermap` and `--groupmap` directly, but + it is implemented using those options internally, so you cannot mix them. + If either the USER or GROUP is empty, no mapping for the omitted user/group + will occur. If GROUP is empty, the trailing colon may be omitted, but if + USER is empty, a leading colon must be supplied. + + If you specify "`--chown=foo:bar`", this is exactly the same as specifying + "`--usermap=*:foo --groupmap=*:bar`", only easier. + +0. `--timeout=TIMEOUT` + + This option allows you to set a maximum I/O timeout in seconds. If no data + is transferred for the specified time then rsync will exit. The default is + 0, which means no timeout. + +0. `--contimeout` + + This option allows you to set the amount of time that rsync will wait for + its connection to an rsync daemon to succeed. If the timeout is reached, + rsync exits with an error. + +0. `--address` + + By default rsync will bind to the wildcard address when connecting to an + rsync daemon. The `--address` option allows you to specify a specific IP + address (or hostname) to bind to. See also this option in the `--daemon` + mode section. + +0. `--port=PORT` + + This specifies an alternate TCP port number to use rather than the default + of 873. This is only needed if you are using the double-colon (::) syntax + to connect with an rsync daemon (since the URL syntax has a way to specify + the port as a part of the URL). See also this option in the `--daemon` + mode section. + +0. `--sockopts` + + This option can provide endless fun for people who like to tune their + systems to the utmost degree. You can set all sorts of socket options + which may make transfers faster (or slower!). Read the man page for the + `setsockopt()` system call for details on some of the options you may be + able to set. By default no special socket options are set. This only + affects direct socket connections to a remote rsync daemon. + + This option also exists in the `--daemon` mode section. + +0. `--blocking-io` + + This tells rsync to use blocking I/O when launching a remote shell + transport. If the remote shell is either rsh or remsh, rsync defaults to + using blocking I/O, otherwise it defaults to using non-blocking I/O. (Note + that ssh prefers non-blocking I/O.) + +0. `--outbuf=MODE` + + This sets the output buffering mode. The mode can be None (aka + Unbuffered), Line, or Block (aka Full). You may specify as little as a + single letter for the mode, and use upper or lower case. + + The main use of this option is to change Full buffering to Line buffering + when rsync's output is going to a file or pipe. + +0. `--itemize-changes`, `-i` + + Requests a simple itemized list of the changes that are being made to each + file, including attribute changes. This is exactly the same as specifying + `--out-format='%i %n%L'`. If you repeat the option, unchanged files will + also be output, but only if the receiving rsync is at least version 2.6.7 + (you can use `-vv` with older versions of rsync, but that also turns on the + output of other verbose messages). + + The "%i" escape has a cryptic output that is 11 letters long. The general + format is like the string `YXcstpoguax`, where **Y** is replaced by the type + of update being done, **X** is replaced by the file-type, and the other + letters represent attributes that may be output if they are being modified. + + The update types that replace the **Y** are as follows: + + - A `<` means that a file is being transferred to the remote host (sent). + - A `>` means that a file is being transferred to the local host + (received). + - A `c` means that a local change/creation is occurring for the item (such + as the creation of a directory or the changing of a symlink, etc.). + - A `h` means that the item is a hard link to another item (requires + `--hard-links`). + - A `.` means that the item is not being updated (though it might have + attributes that are being modified). + - A `*` means that the rest of the itemized-output area contains a message + (e.g. "deleting"). + + The file-types that replace the **X** are: `f` for a file, a `d` for a + directory, an `L` for a symlink, a `D` for a device, and a `S` for a + special file (e.g. named sockets and fifos). + + The other letters in the string above are the actual letters that will be + output if the associated attribute for the item is being updated or a "." + for no change. Three exceptions to this are: (1) a newly created item + replaces each letter with a "+", (2) an identical item replaces the dots + with spaces, and (3) an unknown attribute replaces each letter with a "?" + (this can happen when talking to an older rsync). + + The attribute that is associated with each letter is as follows: + + - A `c` means either that a regular file has a different checksum (requires + `--checksum`) or that a symlink, device, or special file has a changed + value. Note that if you are sending files to an rsync prior to 3.0.1, + this change flag will be present only for checksum-differing regular + files. + - A `s` means the size of a regular file is different and will be updated + by the file transfer. + - A `t` means the modification time is different and is being updated to + the sender's value (requires `--times`). An alternate value of `T` means + that the modification time will be set to the transfer time, which + happens when a file/symlink/device is updated without `--times` and when + a symlink is changed and the receiver can't set its time. (Note: when + using an rsync 3.0.0 client, you might see the `s` flag combined with `t` + instead of the proper `T` flag for this time-setting failure.) + - A `p` means the permissions are different and are being updated to the + sender's value (requires `--perms`). + - An `o` means the owner is different and is being updated to the sender's + value (requires `--owner` and super-user privileges). + - A `g` means the group is different and is being updated to the sender's + value (requires `--group` and the authority to set the group). + - A `u` means the access (use) time is different and is being updated to + the sender's value (requires `--atimes`). An alternate value of `U` + means that the access time will be set to the transfer time, which + happens when a symlink or directory is updated. + - The `a` means that the ACL information changed. + - The `x` means that the extended attribute information changed. + + One other output is possible: when deleting files, the "%i" will output the + string "`*deleting`" for each item that is being removed (assuming that you + are talking to a recent enough rsync that it logs deletions instead of + outputting them as a verbose message). + +0. `--out-format=FORMAT` + + This allows you to specify exactly what the rsync client outputs to the + user on a per-update basis. The format is a text string containing + embedded single-character escape sequences prefixed with a percent (%) + character. A default format of "%n%L" is assumed if either `--info=name` + or `-v` is specified (this tells you just the name of the file and, if the + item is a link, where it points). For a full list of the possible escape + characters, see the "`log format`" setting in the rsyncd.conf manpage. + + Specifying the `--out-format` option implies the `--info=name` option, + which will mention each file, dir, etc. that gets updated in a significant + way (a transferred file, a recreated symlink/device, or a touched + directory). In addition, if the itemize-changes escape (%i) is included in + the string (e.g. if the `--itemize-changes` option was used), the logging + of names increases to mention any item that is changed in any way (as long + as the receiving side is at least 2.6.4). See the `--itemize-changes` + option for a description of the output of "%i". + + Rsync will output the out-format string prior to a file's transfer unless + one of the transfer-statistic escapes is requested, in which case the + logging is done at the end of the file's transfer. When this late logging + is in effect and `--progress` is also specified, rsync will also output the + name of the file being transferred prior to its progress information + (followed, of course, by the out-format output). + +0. `--log-file=FILE` + + This option causes rsync to log what it is doing to a file. This is + similar to the logging that a daemon does, but can be requested for the + client side and/or the server side of a non-daemon transfer. If specified + as a client option, transfer logging will be enabled with a default format + of "%i %n%L". See the `--log-file-format` option if you wish to override + this. + + Here's a example command that requests the remote side to log what is + happening: + + > rsync -av --remote-option=--log-file=/tmp/rlog src/ dest/ + + This is very useful if you need to debug why a connection is closing + unexpectedly. + +0. `--log-file-format=FORMAT` + + This allows you to specify exactly what per-update logging is put into the + file specified by the `--log-file` option (which must also be specified for + this option to have any effect). If you specify an empty string, updated + files will not be mentioned in the log file. For a list of the possible + escape characters, see the "`log format`" setting in the rsyncd.conf manpage. + + The default FORMAT used if `--log-file` is specified and this option is not + is '%i %n%L'. + +0. `--stats` + + This tells rsync to print a verbose set of statistics on the file transfer, + allowing you to tell how effective rsync's delta-transfer algorithm is for + your data. This option is equivalent to `--info=stats2` if combined with 0 + or 1 `-v` options, or `--info=stats3` if combined with 2 or more `-v` + options. + + The current statistics are as follows: + + - `Number of files` is the count of all "files" (in the generic sense), + which includes directories, symlinks, etc. The total count will be + followed by a list of counts by filetype (if the total is non-zero). For + example: "(reg: 5, dir: 3, link: 2, dev: 1, special: 1)" lists the totals + for regular files, directories, symlinks, devices, and special files. If + any of value is 0, it is completely omitted from the list. + - `Number of created files` is the count of how many "files" (generic + sense) were created (as opposed to updated). The total count will be + followed by a list of counts by filetype (if the total is non-zero). + - `Number of deleted files` is the count of how many "files" (generic + sense) were created (as opposed to updated). The total count will be + followed by a list of counts by filetype (if the total is non-zero). + Note that this line is only output if deletions are in effect, and only + if protocol 31 is being used (the default for rsync 3.1.x). + - `Number of regular files transferred` is the count of normal files that + were updated via rsync's delta-transfer algorithm, which does not include + dirs, symlinks, etc. Note that rsync 3.1.0 added the word "regular" into + this heading. + - `Total file size` is the total sum of all file sizes in the transfer. + This does not count any size for directories or special files, but does + include the size of symlinks. + - `Total transferred file size` is the total sum of all files sizes for + just the transferred files. + - `Literal data` is how much unmatched file-update data we had to send to + the receiver for it to recreate the updated files. + - `Matched data` is how much data the receiver got locally when recreating + the updated files. + - `File list size` is how big the file-list data was when the sender sent + it to the receiver. This is smaller than the in-memory size for the file + list due to some compressing of duplicated data when rsync sends the + list. + - `File list generation time` is the number of seconds that the sender + spent creating the file list. This requires a modern rsync on the + sending side for this to be present. + - `File list transfer time` is the number of seconds that the sender spent + sending the file list to the receiver. + - `Total bytes sent` is the count of all the bytes that rsync sent from the + client side to the server side. + - `Total bytes received` is the count of all non-message bytes that rsync + received by the client side from the server side. "Non-message" bytes + means that we don't count the bytes for a verbose message that the server + sent to us, which makes the stats more consistent. + +0. `--8-bit-output`, `-8` + + This tells rsync to leave all high-bit characters unescaped in the output + instead of trying to test them to see if they're valid in the current + locale and escaping the invalid ones. All control characters (but never + tabs) are always escaped, regardless of this option's setting. + + The escape idiom that started in 2.6.7 is to output a literal backslash + (`\`) and a hash (`#`), followed by exactly 3 octal digits. For example, a + newline would output as "`\#012`". A literal backslash that is in a + filename is not escaped unless it is followed by a hash and 3 digits (0-9). + +0. `--human-readable`, `-h` + + Output numbers in a more human-readable format. There are 3 possible + levels: (1) output numbers with a separator between each set of 3 digits + (either a comma or a period, depending on if the decimal point is + represented by a period or a comma); (2) output numbers in units of 1000 + (with a character suffix for larger units -- see below); (3) output + numbers in units of 1024. + + The default is human-readable level 1. Each `-h` option increases the + level by one. You can take the level down to 0 (to output numbers as pure + digits) by specifying the `--no-human-readable` (`--no-h`) option. + + The unit letters that are appended in levels 2 and 3 are: K (kilo), M + (mega), G (giga), or T (tera). For example, a 1234567-byte file would + output as 1.23M in level-2 (assuming that a period is your local decimal + point). + + Backward compatibility note: versions of rsync prior to 3.1.0 do not + support human-readable level 1, and they default to level 0. Thus, + specifying one or two `-h` options will behave in a comparable manner in + old and new versions as long as you didn't specify a `--no-h` option prior + to one or more `-h` options. See the `--list-only` option for one + difference. + +0. `--partial` + + By default, rsync will delete any partially transferred file if the + transfer is interrupted. In some circumstances it is more desirable to + keep partially transferred files. Using the `--partial` option tells rsync + to keep the partial file which should make a subsequent transfer of the + rest of the file much faster. + +0. `--partial-dir=DIR` + + A better way to keep partial files than the `--partial` option is to + specify a _DIR_ that will be used to hold the partial data (instead of + writing it out to the destination file). On the next transfer, rsync will + use a file found in this dir as data to speed up the resumption of the + transfer and then delete it after it has served its purpose. + + Note that if `--whole-file` is specified (or implied), any partial-dir file + that is found for a file that is being updated will simply be removed + (since rsync is sending files without using rsync's delta-transfer + algorithm). + + Rsync will create the _DIR_ if it is missing (just the last dir -- not the + whole path). This makes it easy to use a relative path (such as + "`--partial-dir=.rsync-partial`") to have rsync create the + partial-directory in the destination file's directory when needed, and then + remove it again when the partial file is deleted. Note that the directory + is only removed if it is a relative pathname, as it is expected that an + absolute path is to a directory that is reserved for partial-dir work. + + If the partial-dir value is not an absolute path, rsync will add an exclude + rule at the end of all your existing excludes. This will prevent the + sending of any partial-dir files that may exist on the sending side, and + will also prevent the untimely deletion of partial-dir items on the + receiving side. An example: the above `--partial-dir` option would add the + equivalent of "`-f '-p .rsync-partial/'`" at the end of any other filter + rules. + + If you are supplying your own exclude rules, you may need to add your own + exclude/hide/protect rule for the partial-dir because (1) the auto-added + rule may be ineffective at the end of your other rules, or (2) you may wish + to override rsync's exclude choice. For instance, if you want to make + rsync clean-up any left-over partial-dirs that may be lying around, you + should specify `--delete-after` and add a "risk" filter rule, e.g. + `-f 'R .rsync-partial/'`. (Avoid using `--delete-before` or + `--delete-during` unless you don't need rsync to use any of the left-over + partial-dir data during the current run.) + + IMPORTANT: the `--partial-dir` should not be writable by other users or it + is a security risk. E.g. AVOID "/tmp". + + You can also set the partial-dir value the RSYNC_PARTIAL_DIR environment + variable. Setting this in the environment does not force `--partial` to be + enabled, but rather it affects where partial files go when `--partial` is + specified. For instance, instead of using `--partial-dir=.rsync-tmp` along + with `--progress`, you could set RSYNC_PARTIAL_DIR=.rsync-tmp in your + environment and then just use the `-P` option to turn on the use of the + .rsync-tmp dir for partial transfers. The only times that the `--partial` + option does not look for this environment value are (1) when `--inplace` + was specified (since `--inplace` conflicts with `--partial-dir`), and (2) + when `--delay-updates` was specified (see below). + + When a modern rsync resumes the transfer of a file in the partial-dir, that + partial file is now updated in-place instead of creating yet another + tmp-file copy (so it maxes out at dest + tmp instead of dest + partial + + tmp). This requires both ends of the transfer to be at least version + 3.2.0. + + For the purposes of the daemon-config's "`refuse options`" setting, + `--partial-dir` does _not_ imply `--partial`. This is so that a refusal of + the `--partial` option can be used to disallow the overwriting of + destination files with a partial transfer, while still allowing the safer + idiom provided by `--partial-dir`. + +0. `--delay-updates` + + This option puts the temporary file from each updated file into a holding + directory until the end of the transfer, at which time all the files are + renamed into place in rapid succession. This attempts to make the updating + of the files a little more atomic. By default the files are placed into a + directory named `.~tmp~` in each file's destination directory, but if + you've specified the `--partial-dir` option, that directory will be used + instead. See the comments in the `--partial-dir` section for a discussion + of how this `.~tmp~` dir will be excluded from the transfer, and what you + can do if you want rsync to cleanup old `.~tmp~` dirs that might be lying + around. Conflicts with `--inplace` and `--append`. + + This option uses more memory on the receiving side (one bit per file + transferred) and also requires enough free disk space on the receiving side + to hold an additional copy of all the updated files. Note also that you + should not use an absolute path to `--partial-dir` unless (1) there is no + chance of any of the files in the transfer having the same name (since all + the updated files will be put into a single directory if the path is + absolute) and (2) there are no mount points in the hierarchy (since the + delayed updates will fail if they can't be renamed into place). + + See also the "atomic-rsync" perl script in the "support" subdir for an + update algorithm that is even more atomic (it uses `--link-dest` and a + parallel hierarchy of files). + +0. `--prune-empty-dirs`, `-m` + + This option tells the receiving rsync to get rid of empty directories from + the file-list, including nested directories that have no non-directory + children. This is useful for avoiding the creation of a bunch of useless + directories when the sending rsync is recursively scanning a hierarchy of + files using include/exclude/filter rules. + + Note that the use of transfer rules, such as the `--min-size` option, does + not affect what goes into the file list, and thus does not leave + directories empty, even if none of the files in a directory match the + transfer rule. + + Because the file-list is actually being pruned, this option also affects + what directories get deleted when a delete is active. However, keep in + mind that excluded files and directories can prevent existing items from + being deleted due to an exclude both hiding source files and protecting + destination files. See the perishable filter-rule option for how to avoid + this. + + You can prevent the pruning of certain empty directories from the file-list + by using a global "protect" filter. For instance, this option would ensure + that the directory "emptydir" was kept in the file-list: + + > --filter 'protect emptydir/' + + Here's an example that copies all .pdf files in a hierarchy, only creating + the necessary destination directories to hold the .pdf files, and ensures + that any superfluous files and directories in the destination are removed + (note the hide filter of non-directories being used instead of an exclude): + + > rsync -avm --del --include='*.pdf' -f 'hide,! */' src/ dest + + If you didn't want to remove superfluous destination files, the more + time-honored options of `--include='*/' --exclude='*'` would work + fine in place of the hide-filter (if that is more natural to you). + +0. `--progress` + + This option tells rsync to print information showing the progress of the + transfer. This gives a bored user something to watch. With a modern rsync + this is the same as specifying `--info=flist2,name,progress`, but any + user-supplied settings for those info flags takes precedence (e.g. + "`--info=flist0 --progress`"). + + While rsync is transferring a regular file, it updates a progress line that + looks like this: + + > 782448 63% 110.64kB/s 0:00:04 + + In this example, the receiver has reconstructed 782448 bytes or 63% of the + sender's file, which is being reconstructed at a rate of 110.64 kilobytes + per second, and the transfer will finish in 4 seconds if the current rate + is maintained until the end. + + These statistics can be misleading if rsync's delta-transfer algorithm is + in use. For example, if the sender's file consists of the basis file + followed by additional data, the reported rate will probably drop + dramatically when the receiver gets to the literal data, and the transfer + will probably take much longer to finish than the receiver estimated as it + was finishing the matched part of the file. + + When the file transfer finishes, rsync replaces the progress line with a + summary line that looks like this: + + > 1,238,099 100% 146.38kB/s 0:00:08 (xfr#5, to-chk=169/396) + + In this example, the file was 1,238,099 bytes long in total, the average + rate of transfer for the whole file was 146.38 kilobytes per second over + the 8 seconds that it took to complete, it was the 5th transfer of a + regular file during the current rsync session, and there are 169 more files + for the receiver to check (to see if they are up-to-date or not) remaining + out of the 396 total files in the file-list. + + In an incremental recursion scan, rsync won't know the total number of + files in the file-list until it reaches the ends of the scan, but since it + starts to transfer files during the scan, it will display a line with the + text "ir-chk" (for incremental recursion check) instead of "to-chk" until + the point that it knows the full size of the list, at which point it will + switch to using "to-chk". Thus, seeing "ir-chk" lets you know that the + total count of files in the file list is still going to increase (and each + time it does, the count of files left to check will increase by the number + of the files added to the list). + +0. `-P` + + The `-P` option is equivalent to `--partial --progress`. Its purpose is + to make it much easier to specify these two options for a long transfer + that may be interrupted. + + There is also a `--info=progress2` option that outputs statistics based on + the whole transfer, rather than individual files. Use this flag without + outputting a filename (e.g. avoid `-v` or specify `--info=name0`) if you + want to see how the transfer is doing without scrolling the screen with a + lot of names. (You don't need to specify the `--progress` option in order + to use `--info=progress2`.) + + Finally, you can get an instant progress report by sending rsync a signal + of either SIGINFO or SIGVTALRM. On BSD systems, a SIGINFO is generated by + typing a Ctrl+T (Linux doesn't currently support a SIGINFO signal). When + the client-side process receives one of those signals, it sets a flag to + output a single progress report which is output when the current file + transfer finishes (so it may take a little time if a big file is being + handled when the signal arrives). A filename is output (if needed) + followed by the `--info=progress2` format of progress info. If you don't + know which of the 3 rsync processes is the client process, it's OK to + signal all of them (since the non-client processes ignore the signal). + + CAUTION: sending SIGVTALRM to an older rsync (pre-3.2.0) will kill it. + +0. `--password-file=FILE` + + This option allows you to provide a password for accessing an rsync daemon + via a file or via standard input if **FILE** is `-`. The file should + contain just the password on the first line (all other lines are ignored). + Rsync will exit with an error if **FILE** is world readable or if a + root-run rsync command finds a non-root-owned file. + + This option does not supply a password to a remote shell transport such as + ssh; to learn how to do that, consult the remote shell's documentation. + When accessing an rsync daemon using a remote shell as the transport, this + option only comes into effect after the remote shell finishes its + authentication (i.e. if you have also specified a password in the daemon's + config file). + +0. `--list-only` + + This option will cause the source files to be listed instead of + transferred. This option is inferred if there is a single source arg and + no destination specified, so its main uses are: (1) to turn a copy command + that includes a destination arg into a file-listing command, or (2) to be + able to specify more than one source arg (note: be sure to include the + destination). Caution: keep in mind that a source arg with a wild-card is + expanded by the shell into multiple args, so it is never safe to try to + list such an arg without using this option. For example: + + > rsync -av --list-only foo* dest/ + + Starting with rsync 3.1.0, the sizes output by `--list-only` are affected + by the `--human-readable` option. By default they will contain digit + separators, but higher levels of readability will output the sizes with + unit suffixes. Note also that the column width for the size output has + increased from 11 to 14 characters for all human-readable levels. Use + `--no-h` if you want just digits in the sizes, and the old column width of + 11 characters. + + Compatibility note: when requesting a remote listing of files from an rsync + that is version 2.6.3 or older, you may encounter an error if you ask for a + non-recursive listing. This is because a file listing implies the `--dirs` + option w/o `--recursive`, and older rsyncs don't have that option. To + avoid this problem, either specify the `--no-dirs` option (if you don't + need to expand a directory's content), or turn on recursion and exclude the + content of subdirectories: `-r --exclude='/*/*'`. + +0. `--bwlimit=RATE` + + This option allows you to specify the maximum transfer rate for the data + sent over the socket, specified in units per second. The RATE value can be + suffixed with a string to indicate a size multiplier, and may be a + fractional value (e.g. "`--bwlimit=1.5m`"). If no suffix is specified, the + value will be assumed to be in units of 1024 bytes (as if "K" or "KiB" had + been appended). See the `--max-size` option for a description of all the + available suffixes. A value of zero specifies no limit. + + For backward-compatibility reasons, the rate limit will be rounded to the + nearest KiB unit, so no rate smaller than 1024 bytes per second is + possible. + + Rsync writes data over the socket in blocks, and this option both limits + the size of the blocks that rsync writes, and tries to keep the average + transfer rate at the requested limit. Some burstiness may be seen where + rsync writes out a block of data and then sleeps to bring the average rate + into compliance. + + Due to the internal buffering of data, the `--progress` option may not be + an accurate reflection on how fast the data is being sent. This is because + some files can show up as being rapidly sent when the data is quickly + buffered, while other can show up as very slow when the flushing of the + output buffer occurs. This may be fixed in a future version. + +0. `--write-batch=FILE` + + Record a file that can later be applied to another identical destination + with `--read-batch`. See the "BATCH MODE" section for details, and also + the `--only-write-batch` option. + +0. `--only-write-batch=FILE` + + Works like `--write-batch`, except that no updates are made on the + destination system when creating the batch. This lets you transport the + changes to the destination system via some other means and then apply the + changes via `--read-batch`. + + Note that you can feel free to write the batch directly to some portable + media: if this media fills to capacity before the end of the transfer, you + can just apply that partial transfer to the destination and repeat the + whole process to get the rest of the changes (as long as you don't mind a + partially updated destination system while the multi-update cycle is + happening). + + Also note that you only save bandwidth when pushing changes to a remote + system because this allows the batched data to be diverted from the sender + into the batch file without having to flow over the wire to the receiver + (when pulling, the sender is remote, and thus can't write the batch). + +0. `--read-batch=FILE` + + Apply all of the changes stored in FILE, a file previously generated by + `--write-batch`. If _FILE_ is `-`, the batch data will be read from + standard input. See the "BATCH MODE" section for details. + +0. `--protocol=NUM` + + Force an older protocol version to be used. This is useful for creating a + batch file that is compatible with an older version of rsync. For + instance, if rsync 2.6.4 is being used with the `--write-batch` option, but + rsync 2.6.3 is what will be used to run the `--read-batch` option, you + should use "--protocol=28" when creating the batch file to force the older + protocol version to be used in the batch file (assuming you can't upgrade + the rsync on the reading system). + +0. `--iconv=CONVERT_SPEC` + + Rsync can convert filenames between character sets using this option. + Using a CONVERT_SPEC of "." tells rsync to look up the default + character-set via the locale setting. Alternately, you can fully specify + what conversion to do by giving a local and a remote charset separated by a + comma in the order `--iconv=LOCAL,REMOTE`, e.g. `--iconv=utf8,iso88591`. + This order ensures that the option will stay the same whether you're + pushing or pulling files. Finally, you can specify either `--no-iconv` or + a CONVERT_SPEC of "-" to turn off any conversion. The default setting of + this option is site-specific, and can also be affected via the RSYNC_ICONV + environment variable. + + For a list of what charset names your local iconv library supports, you can + run "`iconv --list`". + + If you specify the `--protect-args` option (`-s`), rsync will translate the + filenames you specify on the command-line that are being sent to the remote + host. See also the `--files-from` option. + + Note that rsync does not do any conversion of names in filter files + (including include/exclude files). It is up to you to ensure that you're + specifying matching rules that can match on both sides of the transfer. + For instance, you can specify extra include/exclude rules if there are + filename differences on the two sides that need to be accounted for. + + When you pass an `--iconv` option to an rsync daemon that allows it, the + daemon uses the charset specified in its "charset" configuration parameter + regardless of the remote charset you actually pass. Thus, you may feel + free to specify just the local charset for a daemon transfer (e.g. + `--iconv=utf8`). + +0. `--ipv4`, `-4` or `--ipv6`, `-6` + + Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh. This + affects sockets that rsync has direct control over, such as the outgoing + socket when directly contacting an rsync daemon, as well as the forwarding + of the `-4` or `-6` option to ssh when rsync can deduce that ssh is being + used as the remote shell. For other remote shells you'll need to specify + the "`--rsh SHELL -4`" option directly (or whatever ipv4/ipv6 hint options + it uses). + + These options also exist in the `--daemon` mode section. + + If rsync was complied without support for IPv6, the `--ipv6` option will + have no effect. The `rsync -V` output will contain "`no IPv6`" if is the + case. + +0. `--checksum-seed=NUM` + + Set the checksum seed to the integer NUM. This 4 byte checksum seed is + included in each block and MD4 file checksum calculation (the more modern + MD5 file checksums don't use a seed). By default the checksum seed is + generated by the server and defaults to the current **time**(). This + option is used to set a specific checksum seed, which is useful for + applications that want repeatable block checksums, or in the case where the + user wants a more random checksum seed. Setting NUM to 0 causes rsync to + use the default of **time**() for checksum seed. + +# DAEMON OPTIONS + +The options allowed when starting an rsync daemon are as follows: + +0. `--daemon` + + This tells rsync that it is to run as a daemon. The daemon you start + running may be accessed using an rsync client using the `host::module` or + `rsync://host/module/` syntax. + + If standard input is a socket then rsync will assume that it is being run + via inetd, otherwise it will detach from the current terminal and become a + background daemon. The daemon will read the config file (rsyncd.conf) on + each connect made by a client and respond to requests accordingly. See the + **rsyncd.conf**(5) man page for more details. + +0. `--address` + + By default rsync will bind to the wildcard address when run as a daemon + with the `--daemon` option. The `--address` option allows you to specify a + specific IP address (or hostname) to bind to. This makes virtual hosting + possible in conjunction with the `--config` option. See also the "address" + global option in the rsyncd.conf manpage. + +0. `--bwlimit=RATE` + + This option allows you to specify the maximum transfer rate for the data + the daemon sends over the socket. The client can still specify a smaller + `--bwlimit` value, but no larger value will be allowed. See the client + version of this option (above) for some extra details. + +0. `--config=FILE` + + This specifies an alternate config file than the default. This is only + relevant when `--daemon` is specified. The default is /etc/rsyncd.conf + unless the daemon is running over a remote shell program and the remote + user is not the super-user; in that case the default is rsyncd.conf in the + current directory (typically $HOME). + +0. `--dparam=OVERRIDE`, `-M` + + This option can be used to set a daemon-config parameter when starting up + rsync in daemon mode. It is equivalent to adding the parameter at the end + of the global settings prior to the first module's definition. The + parameter names can be specified without spaces, if you so desire. For + instance: + + > rsync --daemon -M pidfile=/path/rsync.pid + +0. `--no-detach` + + When running as a daemon, this option instructs rsync to not detach itself + and become a background process. This option is required when running as a + service on Cygwin, and may also be useful when rsync is supervised by a + program such as `daemontools` or AIX's `System Resource Controller`. + `--no-detach` is also recommended when rsync is run under a debugger. This + option has no effect if rsync is run from inetd or sshd. + +0. `--port=PORT` + + This specifies an alternate TCP port number for the daemon to listen on + rather than the default of 873. See also the "port" global option in the + rsyncd.conf manpage. + +0. `--log-file=FILE` + + This option tells the rsync daemon to use the given log-file name instead + of using the "`log file`" setting in the config file. + +0. `--log-file-format=FORMAT` + + This option tells the rsync daemon to use the given FORMAT string instead + of using the "`log format`" setting in the config file. It also enables + "`transfer logging`" unless the string is empty, in which case transfer + logging is turned off. + +0. `--sockopts` + + This overrides the `socket options` setting in the rsyncd.conf file and has + the same syntax. + +0. `--verbose`, `-v` + + This option increases the amount of information the daemon logs during its + startup phase. After the client connects, the daemon's verbosity level + will be controlled by the options that the client used and the + "`max verbosity`" setting in the module's config section. + +0. `--ipv4`, `-4` or `--ipv6`, `-6` + + Tells rsync to prefer IPv4/IPv6 when creating the incoming sockets that the + rsync daemon will use to listen for connections. One of these options may + be required in older versions of Linux to work around an IPv6 bug in the + kernel (if you see an "address already in use" error when nothing else is + using the port, try specifying `--ipv6` or `--ipv4` when starting the + daemon). + + These options also exist in the regular rsync options section. + + If rsync was complied without support for IPv6, the `--ipv6` option will + have no effect. The `rsync -V` output will contain "`no IPv6`" if is the + case. + +0. `--help`, `-h` + + When specified after `--daemon`, print a short help page describing the + options available for starting an rsync daemon. + +# FILTER RULES + +The filter rules allow for flexible selection of which files to transfer +(include) and which files to skip (exclude). The rules either directly specify +include/exclude patterns or they specify a way to acquire more include/exclude +patterns (e.g. to read them from a file). + +As the list of files/directories to transfer is built, rsync checks each name +to be transferred against the list of include/exclude patterns in turn, and the +first matching pattern is acted on: if it is an exclude pattern, then that file +is skipped; if it is an include pattern then that filename is not skipped; if +no matching pattern is found, then the filename is not skipped. + +Rsync builds an ordered list of filter rules as specified on the command-line. +Filter rules have the following syntax: + +> RULE [PATTERN_OR_FILENAME] +> RULE,MODIFIERS [PATTERN_OR_FILENAME] + +You have your choice of using either short or long RULE names, as described +below. If you use a short-named rule, the ',' separating the RULE from the +MODIFIERS is optional. The PATTERN or FILENAME that follows (when present) +must come after either a single space or an underscore (\_). Here are the +available rule prefixes: + +0. `exclude, '-'` specifies an exclude pattern. +0. `include, '+'` specifies an include pattern. +0. `merge, '.'` specifies a merge-file to read for more rules. +0. `dir-merge, ':'` specifies a per-directory merge-file. +0. `hide, 'H'` specifies a pattern for hiding files from the transfer. +0. `show, 'S'` files that match the pattern are not hidden. +0. `protect, 'P'` specifies a pattern for protecting files from deletion. +0. `risk, 'R'` files that match the pattern are not protected. +0. `clear, '!'` clears the current include/exclude list (takes no arg) + +When rules are being read from a file, empty lines are ignored, as are comment +lines that start with a "#". + +[comment]: # (Remember that markdown strips spaces from start/end of ` ... ` sequences!) +[comment]: # (Thus, the `x ` sequences below use a literal non-breakable space!) + +Note that the `--include` & `--exclude` command-line options do not allow the +full range of rule parsing as described above -- they only allow the +specification of include / exclude patterns plus a "`!`" token to clear the +list (and the normal comment parsing when rules are read from a file). If a +pattern does not begin with "`-Â `" (dash, space) or "`+Â `" (plus, space), then +the rule will be interpreted as if "`+Â `" (for an include option) or "`-Â `" +(for an exclude option) were prefixed to the string. A `--filter` option, on +the other hand, must always contain either a short or long rule name at the +start of the rule. + +Note also that the `--filter`, `--include`, and `--exclude` options take one +rule/pattern each. To add multiple ones, you can repeat the options on the +command-line, use the merge-file syntax of the `--filter` option, or the +`--include-from` / `--exclude-from` options. + +# INCLUDE/EXCLUDE PATTERN RULES + +You can include and exclude files by specifying patterns using the "+", "-", +etc. filter rules (as introduced in the FILTER RULES section above). The +include/exclude rules each specify a pattern that is matched against the names +of the files that are going to be transferred. These patterns can take several +forms: + +- if the pattern starts with a `/` then it is anchored to a particular spot in + the hierarchy of files, otherwise it is matched against the end of the + pathname. This is similar to a leading `^` in regular expressions. Thus + `/foo` would match a name of "foo" at either the "root of the transfer" (for + a global rule) or in the merge-file's directory (for a per-directory rule). + An unqualified `foo` would match a name of "foo" anywhere in the tree because + the algorithm is applied recursively from the top down; it behaves as if each + path component gets a turn at being the end of the filename. Even the + unanchored "sub/foo" would match at any point in the hierarchy where a "foo" + was found within a directory named "sub". See the section on ANCHORING + INCLUDE/EXCLUDE PATTERNS for a full discussion of how to specify a pattern + that matches at the root of the transfer. +- if the pattern ends with a `/` then it will only match a directory, not a + regular file, symlink, or device. +- rsync chooses between doing a simple string match and wildcard matching by + checking if the pattern contains one of these three wildcard characters: + '`*`', '`?`', and '`[`' . +- a '`*`' matches any path component, but it stops at slashes. +- use '`**`' to match anything, including slashes. +- a '`?`' matches any character except a slash (`/`). +- a '`[`' introduces a character class, such as `[a-z]` or `[[:alpha:]]`. +- in a wildcard pattern, a backslash can be used to escape a wildcard + character, but it is matched literally when no wildcards are present. This + means that there is an extra level of backslash removal when a pattern + contains wildcard characters compared to a pattern that has none. e.g. if + you add a wildcard to "`foo\bar`" (which matches the backslash) you would + need to use "`foo\\bar*`" to avoid the "`\b`" becoming just "b". +- if the pattern contains a `/` (not counting a trailing /) or a "`**`", then it + is matched against the full pathname, including any leading directories. If + the pattern doesn't contain a `/` or a "`**`", then it is matched only against + the final component of the filename. (Remember that the algorithm is applied + recursively so "full filename" can actually be any portion of a path from the + starting directory on down.) +- a trailing "`dir_name/***`" will match both the directory (as if "dir_name/" + had been specified) and everything in the directory (as if "`dir_name/**`" + had been specified). This behavior was added in version 2.6.7. + +Note that, when using the `--recursive` (`-r`) option (which is implied by +`-a`), every subdir component of every path is visited left to right, with each +directory having a chance for exclusion before its content. In this way +include/exclude patterns are applied recursively to the pathname of each node +in the filesystem's tree (those inside the transfer). The exclude patterns +short-circuit the directory traversal stage as rsync finds the files to send. + +For instance, to include "`/foo/bar/baz`", the directories "`/foo`" and "`/foo/bar`" +must not be excluded. Excluding one of those parent directories prevents the +examination of its content, cutting off rsync's recursion into those paths and +rendering the include for "`/foo/bar/baz`" ineffectual (since rsync can't match +something it never sees in the cut-off section of the directory hierarchy). + +The concept path exclusion is particularly important when using a trailing '`*`' +rule. For instance, this won't work: + +> + /some/path/this-file-will-not-be-found +> + /file-is-included +> - * + +This fails because the parent directory "some" is excluded by the '`*`' rule, so +rsync never visits any of the files in the "some" or "some/path" directories. +One solution is to ask for all directories in the hierarchy to be included by +using a single rule: "`+ */`" (put it somewhere before the "`- *`" rule), and +perhaps use the `--prune-empty-dirs` option. Another solution is to add +specific include rules for all the parent dirs that need to be visited. For +instance, this set of rules works fine: + +> + /some/ +> + /some/path/ +> + /some/path/this-file-is-found +> + /file-also-included +> - * + +Here are some examples of exclude/include matching: + +- "`- *.o`" would exclude all names matching `*.o` +- "`- /foo`" would exclude a file (or directory) named foo in the transfer-root + directory +- "`- foo/`" would exclude any directory named foo +- "`- /foo/*/bar`" would exclude any file named bar which is at two levels + below a directory named foo in the transfer-root directory +- "`- /foo/**/bar`" would exclude any file named bar two or more levels below a + directory named foo in the transfer-root directory +- The combination of "`+ */`", "`+ *.c`", and "`- *`" would include all + directories and C source files but nothing else (see also the + `--prune-empty-dirs` option) +- The combination of "`+ foo/`", "`+ foo/bar.c`", and "`- *`" would include + only the foo directory and foo/bar.c (the foo directory must be explicitly + included or it would be excluded by the "`*`") + +The following modifiers are accepted after a "`+`" or "`-`": + +- A `/` specifies that the include/exclude rule should be matched against the + absolute pathname of the current item. For example, "`-/ /etc/passwd`" would + exclude the passwd file any time the transfer was sending files from the + "/etc" directory, and "-/ subdir/foo" would always exclude "foo" when it is + in a dir named "subdir", even if "foo" is at the root of the current + transfer. +- A `!` specifies that the include/exclude should take effect if the pattern + fails to match. For instance, "`-! */`" would exclude all non-directories. +- A `C` is used to indicate that all the global CVS-exclude rules should be + inserted as excludes in place of the "-C". No arg should follow. +- An `s` is used to indicate that the rule applies to the sending side. When a + rule affects the sending side, it prevents files from being transferred. The + default is for a rule to affect both sides unless `--delete-excluded` was + specified, in which case default rules become sender-side only. See also the + hide (H) and show (S) rules, which are an alternate way to specify + sending-side includes/excludes. +- An `r` is used to indicate that the rule applies to the receiving side. When + a rule affects the receiving side, it prevents files from being deleted. See + the `s` modifier for more info. See also the protect (P) and risk (R) rules, + which are an alternate way to specify receiver-side includes/excludes. +- A `p` indicates that a rule is perishable, meaning that it is ignored in + directories that are being deleted. For instance, the `-C` option's default + rules that exclude things like "CVS" and "`*.o`" are marked as perishable, + and will not prevent a directory that was removed on the source from being + deleted on the destination. +- An `x` indicates that a rule affects xattr names in xattr copy/delete + operations (and is thus ignored when matching file/dir names). If no + xattr-matching rules are specified, a default xattr filtering rule is used + (see the `--xattrs` option). + +# MERGE-FILE FILTER RULES + +You can merge whole files into your filter rules by specifying either a merge +(.) or a dir-merge (:) filter rule (as introduced in the FILTER RULES section +above). + +There are two kinds of merged files -- single-instance ('.') and per-directory +(':'). A single-instance merge file is read one time, and its rules are +incorporated into the filter list in the place of the "." rule. For +per-directory merge files, rsync will scan every directory that it traverses +for the named file, merging its contents when the file exists into the current +list of inherited rules. These per-directory rule files must be created on the +sending side because it is the sending side that is being scanned for the +available files to transfer. These rule files may also need to be transferred +to the receiving side if you want them to affect what files don't get deleted +(see PER-DIRECTORY RULES AND DELETE below). + +Some examples: + +> merge /etc/rsync/default.rules +> . /etc/rsync/default.rules +> dir-merge .per-dir-filter +> dir-merge,n- .non-inherited-per-dir-excludes +> :n- .non-inherited-per-dir-excludes + +The following modifiers are accepted after a merge or dir-merge rule: + +- A `-` specifies that the file should consist of only exclude patterns, with + no other rule-parsing except for in-file comments. +- A `+` specifies that the file should consist of only include patterns, with + no other rule-parsing except for in-file comments. +- A `C` is a way to specify that the file should be read in a CVS-compatible + manner. This turns on 'n', 'w', and '-', but also allows the list-clearing + token (!) to be specified. If no filename is provided, ".cvsignore" is + assumed. +- A `e` will exclude the merge-file name from the transfer; e.g. "dir-merge,e + .rules" is like "dir-merge .rules" and "- .rules". +- An `n` specifies that the rules are not inherited by subdirectories. +- A `w` specifies that the rules are word-split on whitespace instead of the + normal line-splitting. This also turns off comments. Note: the space that + separates the prefix from the rule is treated specially, so "- foo + bar" is + parsed as two rules (assuming that prefix-parsing wasn't also disabled). +- You may also specify any of the modifiers for the "+" or "-" rules (above) in + order to have the rules that are read in from the file default to having that + modifier set (except for the `!` modifier, which would not be useful). For + instance, "merge,-/ .excl" would treat the contents of .excl as absolute-path + excludes, while "dir-merge,s .filt" and ":sC" would each make all their + per-directory rules apply only on the sending side. If the merge rule + specifies sides to affect (via the `s` or `r` modifier or both), then the + rules in the file must not specify sides (via a modifier or a rule prefix + such as `hide`). + +Per-directory rules are inherited in all subdirectories of the directory where +the merge-file was found unless the 'n' modifier was used. Each subdirectory's +rules are prefixed to the inherited per-directory rules from its parents, which +gives the newest rules a higher priority than the inherited rules. The entire +set of dir-merge rules are grouped together in the spot where the merge-file +was specified, so it is possible to override dir-merge rules via a rule that +got specified earlier in the list of global rules. When the list-clearing rule +("!") is read from a per-directory file, it only clears the inherited rules for +the current merge file. + +Another way to prevent a single rule from a dir-merge file from being inherited +is to anchor it with a leading slash. Anchored rules in a per-directory +merge-file are relative to the merge-file's directory, so a pattern "/foo" +would only match the file "foo" in the directory where the dir-merge filter +file was found. + +Here's an example filter file which you'd specify via `--filter=". file":` + +> merge /home/user/.global-filter +> - *.gz +> dir-merge .rules +> + *.[ch] +> - *.o +> - foo* + +This will merge the contents of the /home/user/.global-filter file at the start +of the list and also turns the ".rules" filename into a per-directory filter +file. All rules read in prior to the start of the directory scan follow the +global anchoring rules (i.e. a leading slash matches at the root of the +transfer). + +If a per-directory merge-file is specified with a path that is a parent +directory of the first transfer directory, rsync will scan all the parent dirs +from that starting point to the transfer directory for the indicated +per-directory file. For instance, here is a common filter (see `-F`): + +> --filter=': /.rsync-filter' + +That rule tells rsync to scan for the file .rsync-filter in all directories +from the root down through the parent directory of the transfer prior to the +start of the normal directory scan of the file in the directories that are sent +as a part of the transfer. (Note: for an rsync daemon, the root is always the +same as the module's "path".) + +Some examples of this pre-scanning for per-directory files: + +> rsync -avF /src/path/ /dest/dir +> rsync -av --filter=': ../../.rsync-filter' /src/path/ /dest/dir +> rsync -av --filter=': .rsync-filter' /src/path/ /dest/dir + +The first two commands above will look for ".rsync-filter" in "/" and "/src" +before the normal scan begins looking for the file in "/src/path" and its +subdirectories. The last command avoids the parent-dir scan and only looks for +the ".rsync-filter" files in each directory that is a part of the transfer. + +If you want to include the contents of a ".cvsignore" in your patterns, you +should use the rule ":C", which creates a dir-merge of the .cvsignore file, but +parsed in a CVS-compatible manner. You can use this to affect where the +`--cvs-exclude` (`-C`) option's inclusion of the per-directory .cvsignore file +gets placed into your rules by putting the ":C" wherever you like in your +filter rules. Without this, rsync would add the dir-merge rule for the +.cvsignore file at the end of all your other rules (giving it a lower priority +than your command-line rules). For example: + +> ``` +> cat <<EOT | rsync -avC --filter='. -' a/ b +> + foo.o +> :C +> - *.old +> EOT +> rsync -avC --include=foo.o -f :C --exclude='*.old' a/ b +> ``` + +Both of the above rsync commands are identical. Each one will merge all the +per-directory .cvsignore rules in the middle of the list rather than at the +end. This allows their dir-specific rules to supersede the rules that follow +the :C instead of being subservient to all your rules. To affect the other CVS +exclude rules (i.e. the default list of exclusions, the contents of +$HOME/.cvsignore, and the value of $CVSIGNORE) you should omit the `-C` +command-line option and instead insert a "-C" rule into your filter rules; e.g. +"`--filter=-C`". + +# LIST-CLEARING FILTER RULE + +You can clear the current include/exclude list by using the "!" filter rule (as +introduced in the FILTER RULES section above). The "current" list is either +the global list of rules (if the rule is encountered while parsing the filter +options) or a set of per-directory rules (which are inherited in their own +sub-list, so a subdirectory can use this to clear out the parent's rules). + +# ANCHORING INCLUDE/EXCLUDE PATTERNS + +As mentioned earlier, global include/exclude patterns are anchored at the "root +of the transfer" (as opposed to per-directory patterns, which are anchored at +the merge-file's directory). If you think of the transfer as a subtree of +names that are being sent from sender to receiver, the transfer-root is where +the tree starts to be duplicated in the destination directory. This root +governs where patterns that start with a / match. + +Because the matching is relative to the transfer-root, changing the trailing +slash on a source path or changing your use of the `--relative` option affects +the path you need to use in your matching (in addition to changing how much of +the file tree is duplicated on the destination host). The following examples +demonstrate this. + +Let's say that we want to match two source files, one with an absolute +path of "/home/me/foo/bar", and one with a path of "/home/you/bar/baz". +Here is how the various command choices differ for a 2-source transfer: + +> ``` +> Example cmd: rsync -a /home/me /home/you /dest +> +/- pattern: /me/foo/bar +> +/- pattern: /you/bar/baz +> Target file: /dest/me/foo/bar +> Target file: /dest/you/bar/baz +> ``` + +> ``` +> Example cmd: rsync -a /home/me/ /home/you/ /dest +> +/- pattern: /foo/bar (note missing "me") +> +/- pattern: /bar/baz (note missing "you") +> Target file: /dest/foo/bar +> Target file: /dest/bar/baz +> ``` + +> ``` +> Example cmd: rsync -a --relative /home/me/ /home/you /dest +> +/- pattern: /home/me/foo/bar (note full path) +> +/- pattern: /home/you/bar/baz (ditto) +> Target file: /dest/home/me/foo/bar +> Target file: /dest/home/you/bar/baz +> ``` + +> ``` +> Example cmd: cd /home; rsync -a --relative me/foo you/ /dest +> +/- pattern: /me/foo/bar (starts at specified path) +> +/- pattern: /you/bar/baz (ditto) +> Target file: /dest/me/foo/bar +> Target file: /dest/you/bar/baz +> ``` + +The easiest way to see what name you should filter is to just +look at the output when using `--verbose` and put a / in front of the name +(use the `--dry-run` option if you're not yet ready to copy any files). + +# PER-DIRECTORY RULES AND DELETE + +Without a delete option, per-directory rules are only relevant on the sending +side, so you can feel free to exclude the merge files themselves without +affecting the transfer. To make this easy, the 'e' modifier adds this exclude +for you, as seen in these two equivalent commands: + +> rsync -av --filter=': .excl' --exclude=.excl host:src/dir /dest +> rsync -av --filter=':e .excl' host:src/dir /dest + +However, if you want to do a delete on the receiving side AND you want some +files to be excluded from being deleted, you'll need to be sure that the +receiving side knows what files to exclude. The easiest way is to include the +per-directory merge files in the transfer and use `--delete-after`, because +this ensures that the receiving side gets all the same exclude rules as the +sending side before it tries to delete anything: + +> rsync -avF --delete-after host:src/dir /dest + +However, if the merge files are not a part of the transfer, you'll need to +either specify some global exclude rules (i.e. specified on the command line), +or you'll need to maintain your own per-directory merge files on the receiving +side. An example of the first is this (assume that the remote .rules files +exclude themselves): + +> rsync -av --filter=': .rules' --filter='. /my/extra.rules' +> --delete host:src/dir /dest + +In the above example the extra.rules file can affect both sides of the +transfer, but (on the sending side) the rules are subservient to the rules +merged from the .rules files because they were specified after the +per-directory merge rule. + +In one final example, the remote side is excluding the .rsync-filter files from +the transfer, but we want to use our own .rsync-filter files to control what +gets deleted on the receiving side. To do this we must specifically exclude +the per-directory merge files (so that they don't get deleted) and then put +rules into the local files to control what else should not get deleted. Like +one of these commands: + +> ``` +> rsync -av --filter=':e /.rsync-filter' --delete \ +> host:src/dir /dest +> rsync -avFF --delete host:src/dir /dest +> ``` + +# BATCH MODE + +Batch mode can be used to apply the same set of updates to many identical +systems. Suppose one has a tree which is replicated on a number of hosts. Now +suppose some changes have been made to this source tree and those changes need +to be propagated to the other hosts. In order to do this using batch mode, +rsync is run with the write-batch option to apply the changes made to the +source tree to one of the destination trees. The write-batch option causes the +rsync client to store in a "batch file" all the information needed to repeat +this operation against other, identical destination trees. + +Generating the batch file once saves having to perform the file status, +checksum, and data block generation more than once when updating multiple +destination trees. Multicast transport protocols can be used to transfer the +batch update files in parallel to many hosts at once, instead of sending the +same data to every host individually. + +To apply the recorded changes to another destination tree, run rsync with the +read-batch option, specifying the name of the same batch file, and the +destination tree. Rsync updates the destination tree using the information +stored in the batch file. + +For your convenience, a script file is also created when the write-batch option +is used: it will be named the same as the batch file with ".sh" appended. This +script file contains a command-line suitable for updating a destination tree +using the associated batch file. It can be executed using a Bourne (or +Bourne-like) shell, optionally passing in an alternate destination tree +pathname which is then used instead of the original destination path. This is +useful when the destination tree path on the current host differs from the one +used to create the batch file. + +Examples: + +> $ rsync --write-batch=foo -a host:/source/dir/ /adest/dir/ +> $ scp foo* remote: +> $ ssh remote ./foo.sh /bdest/dir/ + +> $ rsync --write-batch=foo -a /source/dir/ /adest/dir/ +> $ ssh remote rsync --read-batch=- -a /bdest/dir/ <foo + +In these examples, rsync is used to update /adest/dir/ from /source/dir/ and +the information to repeat this operation is stored in "foo" and "foo.sh". The +host "remote" is then updated with the batched data going into the directory +/bdest/dir. The differences between the two examples reveals some of the +flexibility you have in how you deal with batches: + +- The first example shows that the initial copy doesn't have to be local -- you + can push or pull data to/from a remote host using either the remote-shell + syntax or rsync daemon syntax, as desired. +- The first example uses the created "foo.sh" file to get the right rsync + options when running the read-batch command on the remote host. +- The second example reads the batch data via standard input so that the batch + file doesn't need to be copied to the remote machine first. This example + avoids the foo.sh script because it needed to use a modified `--read-batch` + option, but you could edit the script file if you wished to make use of it + (just be sure that no other option is trying to use standard input, such as + the "`--exclude-from=-`" option). + +Caveats: + +The read-batch option expects the destination tree that it is updating to be +identical to the destination tree that was used to create the batch update +fileset. When a difference between the destination trees is encountered the +update might be discarded with a warning (if the file appears to be up-to-date +already) or the file-update may be attempted and then, if the file fails to +verify, the update discarded with an error. This means that it should be safe +to re-run a read-batch operation if the command got interrupted. If you wish +to force the batched-update to always be attempted regardless of the file's +size and date, use the `-I` option (when reading the batch). If an error +occurs, the destination tree will probably be in a partially updated state. In +that case, rsync can be used in its regular (non-batch) mode of operation to +fix up the destination tree. + +The rsync version used on all destinations must be at least as new as the one +used to generate the batch file. Rsync will die with an error if the protocol +version in the batch file is too new for the batch-reading rsync to handle. +See also the `--protocol` option for a way to have the creating rsync generate +a batch file that an older rsync can understand. (Note that batch files +changed format in version 2.6.3, so mixing versions older than that with newer +versions will not work.) + +When reading a batch file, rsync will force the value of certain options to +match the data in the batch file if you didn't set them to the same as the +batch-writing command. Other options can (and should) be changed. For +instance `--write-batch` changes to `--read-batch`, `--files-from` is dropped, +and the `--filter` / `--include` / `--exclude` options are not needed unless +one of the `--delete` options is specified. + +The code that creates the BATCH.sh file transforms any filter/include/exclude +options into a single list that is appended as a "here" document to the shell +script file. An advanced user can use this to modify the exclude list if a +change in what gets deleted by `--delete` is desired. A normal user can ignore +this detail and just use the shell script as an easy way to run the appropriate +`--read-batch` command for the batched data. + +The original batch mode in rsync was based on "rsync+", but the latest +version uses a new implementation. + +# SYMBOLIC LINKS + +Three basic behaviors are possible when rsync encounters a symbolic +link in the source directory. + +By default, symbolic links are not transferred at all. A message "skipping +non-regular" file is emitted for any symlinks that exist. + +If `--links` is specified, then symlinks are recreated with the same target on +the destination. Note that `--archive` implies `--links`. + +If `--copy-links` is specified, then symlinks are "collapsed" by +copying their referent, rather than the symlink. + +Rsync can also distinguish "safe" and "unsafe" symbolic links. An example +where this might be used is a web site mirror that wishes to ensure that the +rsync module that is copied does not include symbolic links to `/etc/passwd` in +the public section of the site. Using `--copy-unsafe-links` will cause any +links to be copied as the file they point to on the destination. Using +`--safe-links` will cause unsafe links to be omitted altogether. (Note that you +must specify `--links` for `--safe-links` to have any effect.) + +Symbolic links are considered unsafe if they are absolute symlinks +(start with `/`), empty, or if they contain enough ".." +components to ascend from the directory being copied. + +Here's a summary of how the symlink options are interpreted. The list is in +order of precedence, so if your combination of options isn't mentioned, use the +first line that is a complete subset of your options: + +0. `--copy-links` Turn all symlinks into normal files (leaving no symlinks for + any other options to affect). +0. `--links --copy-unsafe-links` Turn all unsafe symlinks into files and + duplicate all safe symlinks. +0. `--copy-unsafe-links` Turn all unsafe symlinks into files, noisily skip all + safe symlinks. +0. `--links --safe-links` Duplicate safe symlinks and skip unsafe ones. +0. `--links` Duplicate all symlinks. + +# DIAGNOSTICS + +rsync occasionally produces error messages that may seem a little cryptic. The +one that seems to cause the most confusion is "protocol version mismatch -- is +your shell clean?". + +This message is usually caused by your startup scripts or remote shell facility +producing unwanted garbage on the stream that rsync is using for its transport. +The way to diagnose this problem is to run your remote shell like this: + +> ssh remotehost /bin/true > out.dat + +then look at out.dat. If everything is working correctly then out.dat should +be a zero length file. If you are getting the above error from rsync then you +will probably find that out.dat contains some text or data. Look at the +contents and try to work out what is producing it. The most common cause is +incorrectly configured shell startup scripts (such as .cshrc or .profile) that +contain output statements for non-interactive logins. + +If you are having trouble debugging filter patterns, then try specifying the +`-vv` option. At this level of verbosity rsync will show why each individual +file is included or excluded. + +# EXIT VALUES + +0. **0** Success +0. **1** Syntax or usage error +0. **2** Protocol incompatibility +0. **3** Errors selecting input/output files, dirs +0. **4** Requested action not supported: an attempt was made to manipulate + 64-bit files on a platform that cannot support them; or an option was + specified that is supported by the client and not by the server. +0. **5** Error starting client-server protocol +0. **6** Daemon unable to append to log-file +0. **10** Error in socket I/O +0. **11** Error in file I/O +0. **12** Error in rsync protocol data stream +0. **13** Errors with program diagnostics +0. **14** Error in IPC code +0. **20** Received SIGUSR1 or SIGINT +0. **21** Some error returned by **waitpid()** +0. **22** Error allocating core memory buffers +0. **23** Partial transfer due to error +0. **24** Partial transfer due to vanished source files +0. **25** The --max-delete limit stopped deletions +0. **30** Timeout in data send/receive +0. **35** Timeout waiting for daemon connection + +# ENVIRONMENT VARIABLES + +0. `CVSIGNORE` + + The CVSIGNORE environment variable supplements any ignore patterns in + .cvsignore files. See the `--cvs-exclude` option for more details. + +0. `RSYNC_ICONV` + + Specify a default `--iconv` setting using this environment variable. (First + supported in 3.0.0.) + +0. `RSYNC_PROTECT_ARGS` + + Specify a non-zero numeric value if you want the `--protect-args` option to + be enabled by default, or a zero value to make sure that it is disabled by + default. (First supported in 3.1.0.) + +0. `RSYNC_RSH` + + The RSYNC_RSH environment variable allows you to override the default shell + used as the transport for rsync. Command line options are permitted after + the command name, just as in the `-e` option. + +0. `RSYNC_PROXY` + + The RSYNC_PROXY environment variable allows you to redirect your rsync + client to use a web proxy when connecting to a rsync daemon. You should + set RSYNC_PROXY to a hostname:port pair. + +0. `RSYNC_PASSWORD` + + Setting RSYNC_PASSWORD to the required password allows you to run + authenticated rsync connections to an rsync daemon without user + intervention. Note that this does not supply a password to a remote shell + transport such as ssh; to learn how to do that, consult the remote shell's + documentation. + +0. `USER` or `LOGNAME` + + The USER or LOGNAME environment variables are used to determine the default + username sent to an rsync daemon. If neither is set, the username defaults + to "nobody". + +0. `HOME` + + The HOME environment variable is used to find the user's default .cvsignore + file. + +# FILES + +/etc/rsyncd.conf or rsyncd.conf + +# SEE ALSO + +**rsync-ssl**(1), **rsyncd.conf**(5) + +# BUGS + +times are transferred as \*nix time_t values + +When transferring to FAT filesystems rsync may re-sync +unmodified files. +See the comments on the `--modify-window` option. + +file permissions, devices, etc. are transferred as native numerical +values + +see also the comments on the `--delete` option + +Please report bugs! See the web site at <http://rsync.samba.org/>. + +# VERSION + +This man page is current for version @VERSION@ of rsync. + +# INTERNAL OPTIONS + +The options `--server` and `--sender` are used internally by rsync, and should +never be typed by a user under normal circumstances. Some awareness of these +options may be needed in certain scenarios, such as when setting up a login +that can only run an rsync command. For instance, the support directory of the +rsync distribution has an example script named rrsync (for restricted rsync) +that can be used with a restricted ssh login. + +# CREDITS + +rsync is distributed under the GNU General Public License. See the file +COPYING for details. + +A web site is available at <http://rsync.samba.org/>. The site includes an +FAQ-O-Matic which may cover questions unanswered by this manual page. + +The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync> + +We would be delighted to hear from you if you like this program. Please +contact the mailing-list at <rsync@lists.samba.org>. + +This program uses the excellent zlib compression library written by Jean-loup +Gailly and Mark Adler. + +# THANKS + +Special thanks go out to: John Van Essen, Matt McCutchen, Wesley W. Terpstra, +David Dykstra, Jos Backus, Sebastian Krahmer, Martin Pool, and our +gone-but-not-forgotten compadre, J.W. Schultz. + +Thanks also to Richard Brent, Brendan Mackay, Bill Waite, Stephen Rothwell and +David Bell. I've probably missed some people, my apologies if I have. + +# AUTHOR + +rsync was originally written by Andrew Tridgell and Paul Mackerras. Many +people have later contributed to it. It is currently maintained by Wayne +Davison. + +Mailing lists for support and development are available at +<http://lists.samba.org/>. @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -43,6 +43,7 @@ extern int am_starting_up; extern int allow_8bit_chars; extern int protocol_version; extern int got_kill_signal; +extern int called_from_signal_handler; extern int inc_recurse; extern int inplace; extern int flist_eof; @@ -56,6 +57,15 @@ extern struct chmod_mode_struct *daemon_chmod_modes; extern char *iconv_opt; #endif +#define UPDATED_OWNER (1<<0) +#define UPDATED_GROUP (1<<1) +#define UPDATED_MTIME (1<<2) +#define UPDATED_ATIME (1<<3) +#define UPDATED_ACLS (1<<4) +#define UPDATED_MODE (1<<5) + +#define UPDATED_TIMES (UPDATED_MTIME|UPDATED_ATIME) + #ifdef ICONV_CONST iconv_t ic_chck = (iconv_t)-1; # ifdef ICONV_OPTION @@ -308,8 +318,7 @@ void send_protected_args(int fd, char *args[]) #endif } -int read_ndx_and_attrs(int f_in, int f_out, int *iflag_ptr, uchar *type_ptr, - char *buf, int *len_ptr) +int read_ndx_and_attrs(int f_in, int f_out, int *iflag_ptr, uchar *type_ptr, char *buf, int *len_ptr) { int len, iflags = 0; struct file_list *flist; @@ -458,6 +467,21 @@ mode_t dest_mode(mode_t flist_mode, mode_t stat_mode, int dflt_perms, return new_mode; } +static int same_mtime(struct file_struct *file, STRUCT_STAT *st, int extra_accuracy) +{ +#ifdef ST_MTIME_NSEC + uint32 f1_nsec = F_MOD_NSEC_or_0(file); + uint32 f2_nsec = (uint32)st->ST_MTIME_NSEC; +#else + uint32 f1_nsec = 0, f2_nsec = 0; +#endif + + if (extra_accuracy) /* ignore modify_window when setting the time after a transfer or checksum check */ + return file->modtime == st->st_mtime && f1_nsec == f2_nsec; + + return same_time(file->modtime, f1_nsec, st->st_mtime , f2_nsec); +} + int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, const char *fnamecmp, int flags) { @@ -523,8 +547,8 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, /* We shouldn't have attempted to change uid * or gid unless have the privilege. */ rsyserr(FERROR_XFER, errno, "%s %s failed", - change_uid ? "chown" : "chgrp", - full_fname(fname)); + change_uid ? "chown" : "chgrp", + full_fname(fname)); goto cleanup; } if (uid == (uid_t)-1 && sxp->st.st_uid != (uid_t)-1) @@ -539,7 +563,10 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, keep_dirlinks && S_ISDIR(sxp->st.st_mode)); } } - updated = 1; + if (change_uid) + updated |= UPDATED_OWNER; + if (change_gid) + updated |= UPDATED_GROUP; } #ifdef SUPPORT_XATTRS @@ -552,23 +579,39 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, if (!preserve_times || (!(preserve_times & PRESERVE_DIR_TIMES) && S_ISDIR(sxp->st.st_mode)) || (!(preserve_times & PRESERVE_LINK_TIMES) && S_ISLNK(sxp->st.st_mode))) - flags |= ATTRS_SKIP_MTIME; - if (!(flags & ATTRS_SKIP_MTIME) - && (sxp->st.st_mtime != file->modtime + flags |= ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME; + else if (sxp != &sx2) + memcpy(&sx2.st, &sxp->st, sizeof (sx2.st)); + if (!atimes_ndx || S_ISDIR(sxp->st.st_mode)) + flags |= ATTRS_SKIP_ATIME; + if (!(flags & ATTRS_SKIP_MTIME) && !same_mtime(file, &sxp->st, flags & ATTRS_ACCURATE_TIME)) { + sx2.st.st_mtime = file->modtime; #ifdef ST_MTIME_NSEC - || (flags & ATTRS_SET_NANO && NSEC_BUMP(file) && (uint32)sxp->st.ST_MTIME_NSEC != F_MOD_NSEC(file)) + sx2.st.ST_MTIME_NSEC = F_MOD_NSEC_or_0(file); #endif - )) { - int ret = set_modtime(fname, file->modtime, F_MOD_NSEC(file), sxp->st.st_mode); + updated |= UPDATED_MTIME; + } + if (!(flags & ATTRS_SKIP_ATIME)) { + time_t file_atime = F_ATIME(file); + if (flags & ATTRS_ACCURATE_TIME || !same_time(sxp->st.st_atime, 0, file_atime, 0)) { + sx2.st.st_atime = file_atime; +#ifdef ST_ATIME_NSEC + sx2.st.ST_ATIME_NSEC = 0; +#endif + updated |= UPDATED_ATIME; + } + } + if (updated & UPDATED_TIMES) { + int ret = set_times(fname, &sx2.st); if (ret < 0) { rsyserr(FERROR_XFER, errno, "failed to set times on %s", full_fname(fname)); goto cleanup; } - if (ret == 0) /* ret == 1 if symlink could not be set */ - updated = 1; - else + if (ret > 0) { /* ret == 1 if symlink could not be set */ + updated &= ~UPDATED_TIMES; file->flags |= FLAG_TIME_FAILED; + } } #ifdef SUPPORT_ACLS @@ -580,7 +623,7 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, * need to chmod(). */ if (preserve_acls && !S_ISLNK(new_mode)) { if (set_acl(fname, file, sxp, new_mode) > 0) - updated = 1; + updated |= UPDATED_ACLS; } #endif @@ -594,7 +637,7 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, goto cleanup; } if (ret == 0) /* ret == 1 if symlink could not be set */ - updated = 1; + updated |= UPDATED_MODE; } #endif @@ -613,6 +656,8 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, /* This is only called for SIGINT, SIGHUP, and SIGTERM. */ void sig_int(int sig_num) { + called_from_signal_handler = 1; + /* KLUGE: if the user hits Ctrl-C while ssh is prompting * for a password, then our cleanup's sending of a SIGUSR1 * signal to all our children may kill ssh before it has a @@ -636,6 +681,7 @@ void sig_int(int sig_num) * we didn't already set the flag. */ if (!got_kill_signal && (am_server || am_receiver)) { got_kill_signal = sig_num; + called_from_signal_handler = 0; return; } @@ -646,7 +692,7 @@ void sig_int(int sig_num) * attributes (e.g. permissions, ownership, etc.). If the robust_rename() * call is forced to copy the temp file and partialptr is both non-NULL and * not an absolute path, we stage the file into the partial-dir and then - * rename it into place. This returns 1 on succcess or 0 on failure. */ + * rename it into place. This returns 1 on success or 0 on failure. */ int finish_transfer(const char *fname, const char *fnametmp, const char *fnamecmp, const char *partialptr, struct file_struct *file, int ok_to_set_time, @@ -672,7 +718,7 @@ int finish_transfer(const char *fname, const char *fnametmp, /* Change permissions before putting the file into place. */ set_file_attrs(fnametmp, file, NULL, fnamecmp, - ok_to_set_time ? ATTRS_SET_NANO : ATTRS_SKIP_MTIME); + ok_to_set_time ? ATTRS_ACCURATE_TIME : ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME); /* move tmp file over real file */ if (DEBUG_GTE(RECV, 1)) @@ -697,7 +743,7 @@ int finish_transfer(const char *fname, const char *fnametmp, do_set_file_attrs: set_file_attrs(fnametmp, file, NULL, fnamecmp, - ok_to_set_time ? ATTRS_SET_NANO : ATTRS_SKIP_MTIME); + ok_to_set_time ? ATTRS_ACCURATE_TIME : ATTRS_SKIP_MTIME | ATTRS_SKIP_ATIME); if (temp_copy_name) { if (do_rename(fnametmp, fname) < 0) { @@ -2,7 +2,7 @@ * Copyright (C) 1996, 2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -52,16 +52,24 @@ #define XMIT_SAME_NAME (1<<5) #define XMIT_LONG_NAME (1<<6) #define XMIT_SAME_TIME (1<<7) + #define XMIT_SAME_RDEV_MAJOR (1<<8) /* protocols 28 - now (devices only) */ #define XMIT_NO_CONTENT_DIR (1<<8) /* protocols 30 - now (dirs only) */ -#define XMIT_HLINKED (1<<9) /* protocols 28 - now */ +#define XMIT_HLINKED (1<<9) /* protocols 28 - now (non-dirs) */ #define XMIT_SAME_DEV_pre30 (1<<10) /* protocols 28 - 29 */ #define XMIT_USER_NAME_FOLLOWS (1<<10) /* protocols 30 - now */ -#define XMIT_RDEV_MINOR_8_pre30 (1<<11) /* protocols 28 - 29 */ +#define XMIT_RDEV_MINOR_8_pre30 (1<<11) /* protocols 28 - 29 */ #define XMIT_GROUP_NAME_FOLLOWS (1<<11) /* protocols 30 - now */ #define XMIT_HLINK_FIRST (1<<12) /* protocols 30 - now (HLINKED files only) */ #define XMIT_IO_ERROR_ENDLIST (1<<12) /* protocols 31*- now (w/XMIT_EXTENDED_FLAGS) (also protocol 30 w/'f' compat flag) */ #define XMIT_MOD_NSEC (1<<13) /* protocols 31 - now */ +#define XMIT_SAME_ATIME (1<<14) /* any protocol - restricted by command-line option */ +#define XMIT_UNUSED_15 (1<<15) /* unused flag bit */ + +/* The following XMIT flags require an rsync that uses a varint for the flag values */ + +#define XMIT_RESERVED_16 (1<<16) /* reserved for future fileflags use */ +#define XMIT_RESERVED_17 (1<<17) /* reserved for future crtimes use */ /* These flags are used in the live flist data. */ @@ -87,9 +95,11 @@ /* These flags are passed to functions but not stored. */ #define FLAG_DIVERT_DIRS (1<<16) /* sender, but must be unique */ +#define FLAG_PERHAPS_DIR (1<<17) /* generator */ /* These flags are for get_dirlist(). */ #define GDL_IGNORE_FILTER_RULES (1<<0) +#define GDL_PERHAPS_DIR (1<<1) /* Some helper macros for matching bits. */ #define BITS_SET(val,bits) (((val) & (bits)) == (bits)) @@ -151,6 +161,10 @@ #define MAX_BASIS_DIRS 20 #define MAX_SERVER_ARGS (MAX_BASIS_DIRS*2 + 100) +#define COMPARE_DEST 1 +#define COPY_DEST 2 +#define LINK_DEST 3 + #define MPLEX_BASE 7 #define NO_FILTERS 0 @@ -165,8 +179,10 @@ #define ATTRS_REPORT (1<<0) #define ATTRS_SKIP_MTIME (1<<1) -#define ATTRS_SET_NANO (1<<2) +#define ATTRS_ACCURATE_TIME (1<<2) +#define ATTRS_SKIP_ATIME (1<<3) +#define MSG_FLUSH 2 #define FULL_FLUSH 1 #define NORMAL_FLUSH 0 @@ -392,10 +408,13 @@ enum delret { #ifdef CAN_SET_NSEC #ifdef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC #define ST_MTIME_NSEC st_mtim.tv_nsec +#define ST_ATIME_NSEC st_atim.tv_nsec #elif defined(HAVE_STRUCT_STAT_ST_MTIMENSEC) #define ST_MTIME_NSEC st_mtimensec +#define ST_ATIME_NSEC st_atimensec #elif defined(HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC) #define ST_MTIME_NSEC st_mtimespec.tv_nsec +#define ST_ATIME_NSEC st_atimespec.tv_nsec #endif #endif @@ -616,6 +635,9 @@ typedef unsigned int size_t; # define SIZEOF_INT64 SIZEOF_OFF_T #endif +#define HT_KEY32 0 +#define HT_KEY64 1 + struct hashtable { void *nodes; int32 size, entries; @@ -698,9 +720,29 @@ struct ht_int64_node { #endif #endif +#if SIZEOF_CHARP == 4 +# define PTRS_ARE_32 1 +# define PTR_EXTRA_CNT 1 +#elif SIZEOF_CHARP == 8 +# define PTRS_ARE_64 1 +# define PTR_EXTRA_CNT EXTRA64_CNT +#else +# error Character pointers are not 4 or 8 bytes. +#endif + union file_extras { int32 num; uint32 unum; +#ifdef PTRS_ARE_32 + const char* ptr; +#endif +}; + +union file_extras64 { + int64 num; +#ifdef PTRS_ARE_64 + const char* ptr; +#endif }; struct file_struct { @@ -714,6 +756,9 @@ struct file_struct { extern int file_extra_cnt; extern int inc_recurse; +extern int atimes_ndx; +extern int pathname_ndx; +extern int depth_ndx; extern int uid_ndx; extern int gid_ndx; extern int acls_ndx; @@ -721,14 +766,18 @@ extern int xattrs_ndx; #define FILE_STRUCT_LEN (offsetof(struct file_struct, basename)) #define EXTRA_LEN (sizeof (union file_extras)) -#define PTR_EXTRA_CNT ((sizeof (char *) + EXTRA_LEN - 1) / EXTRA_LEN) #define DEV_EXTRA_CNT 2 #define DIRNODE_EXTRA_CNT 3 +#define EXTRA64_CNT ((sizeof (union file_extras64) + EXTRA_LEN - 1) / EXTRA_LEN) #define SUM_EXTRA_CNT ((MAX_DIGEST_LEN + EXTRA_LEN - 1) / EXTRA_LEN) #define REQ_EXTRA(f,ndx) ((union file_extras*)(f) - (ndx)) #define OPT_EXTRA(f,bump) ((union file_extras*)(f) - file_extra_cnt - 1 - (bump)) +/* These are guaranteed to be allocated first in the array so that they + * are aligned for direct int64-pointer access. */ +#define REQ_EXTRA64(f,ndx) ((union file_extras64*)REQ_EXTRA(f,ndx)) + #define NSEC_BUMP(f) ((f)->flags & FLAG_MOD_NSEC ? 1 : 0) #define LEN64_BUMP(f) ((f)->flags & FLAG_LENGTH64 ? 1 : 0) #define START_BUMP(f) (NSEC_BUMP(f) + LEN64_BUMP(f)) @@ -739,20 +788,25 @@ extern int xattrs_ndx; #if SIZEOF_INT64 < 8 #define F_LENGTH(f) ((int64)(f)->len32) #else -#define F_LENGTH(f) ((int64)(f)->len32 + ((f)->flags & FLAG_LENGTH64 \ - ? (int64)OPT_EXTRA(f, NSEC_BUMP(f))->unum << 32 : 0)) +#define F_HIGH_LEN(f) (OPT_EXTRA(f, NSEC_BUMP(f))->unum) +#define F_LENGTH(f) ((int64)(f)->len32 + ((f)->flags & FLAG_LENGTH64 ? (int64)F_HIGH_LEN(f) << 32 : 0)) #endif -#define F_MOD_NSEC(f) ((f)->flags & FLAG_MOD_NSEC ? OPT_EXTRA(f, 0)->unum : 0) +#define F_MOD_NSEC(f) OPT_EXTRA(f, 0)->unum +#define F_MOD_NSEC_or_0(f) ((f)->flags & FLAG_MOD_NSEC ? F_MOD_NSEC(f) : 0) /* If there is a symlink string, it is always right after the basename */ #define F_SYMLINK(f) ((f)->basename + strlen((f)->basename) + 1) /* The sending side always has this available: */ -#define F_PATHNAME(f) (*(const char**)REQ_EXTRA(f, PTR_EXTRA_CNT)) +#ifdef PTRS_ARE_32 +#define F_PATHNAME(f) REQ_EXTRA(f, pathname_ndx)->ptr +#else +#define F_PATHNAME(f) REQ_EXTRA64(f, pathname_ndx)->ptr +#endif /* The receiving side always has this available: */ -#define F_DEPTH(f) REQ_EXTRA(f, 1)->num +#define F_DEPTH(f) REQ_EXTRA(f, depth_ndx)->num /* When the associated option is on, all entries will have these present: */ #define F_OWNER(f) REQ_EXTRA(f, uid_ndx)->unum @@ -760,6 +814,7 @@ extern int xattrs_ndx; #define F_ACL(f) REQ_EXTRA(f, acls_ndx)->num #define F_XATTR(f) REQ_EXTRA(f, xattrs_ndx)->num #define F_NDX(f) REQ_EXTRA(f, unsort_ndx)->num +#define F_ATIME(f) REQ_EXTRA64(f, atimes_ndx)->num /* These items are per-entry optional: */ #define F_HL_GNUM(f) OPT_EXTRA(f, START_BUMP(f))->num /* non-dirs */ @@ -1014,7 +1069,32 @@ typedef struct { #define ACL_READY(sx) ((sx).acc_acl != NULL) #define XATTR_READY(sx) ((sx).xattr != NULL) +#define CLVL_NOT_SPECIFIED INT_MIN + +#define CPRES_AUTO (-1) +#define CPRES_NONE 0 +#define CPRES_ZLIB 1 +#define CPRES_ZLIBX 2 +#define CPRES_LZ4 3 +#define CPRES_ZSTD 4 + +struct name_num_item { + int num; + const char *name, *main_name; +}; + +struct name_num_obj { + const char *type; + const char *negotiated_name; + uchar *saw; + int saw_len; + int negotiated_num; + struct name_num_item list[]; +}; + +#ifndef __cplusplus #include "proto.h" +#endif #ifndef SUPPORT_XATTRS #define x_stat(fn,fst,xst) do_stat(fn,fst) @@ -1041,7 +1121,6 @@ int vsnprintf(char *str, size_t count, const char *fmt, va_list args); int snprintf(char *str, size_t count, const char *fmt,...); #endif - #ifndef HAVE_STRERROR extern char *sys_errlist[]; #define strerror(i) sys_errlist[i] @@ -1280,7 +1359,8 @@ extern short info_levels[], debug_levels[]; #define DEBUG_HLINK (DEBUG_HASH+1) #define DEBUG_ICONV (DEBUG_HLINK+1) #define DEBUG_IO (DEBUG_ICONV+1) -#define DEBUG_OWN (DEBUG_IO+1) +#define DEBUG_NSTR (DEBUG_IO+1) +#define DEBUG_OWN (DEBUG_NSTR+1) #define DEBUG_PROTO (DEBUG_OWN+1) #define DEBUG_RECV (DEBUG_PROTO+1) #define DEBUG_SEND (DEBUG_RECV+1) diff --git a/rsync.yo b/rsync.yo deleted file mode 100644 index 48d5da1d..00000000 --- a/rsync.yo +++ /dev/null @@ -1,3539 +0,0 @@ -mailto(rsync-bugs@samba.org) -manpage(rsync)(1)(28 Jan 2018)()() -manpagename(rsync)(a fast, versatile, remote (and local) file-copying tool) -manpagesynopsis() - -verb(Local: rsync [OPTION...] SRC... [DEST] - -Access via remote shell: - Pull: rsync [OPTION...] [USER@]HOST:SRC... [DEST] - Push: rsync [OPTION...] SRC... [USER@]HOST:DEST - -Access via rsync daemon: - Pull: rsync [OPTION...] [USER@]HOST::SRC... [DEST] - rsync [OPTION...] rsync://[USER@]HOST[:PORT]/SRC... [DEST] - Push: rsync [OPTION...] SRC... [USER@]HOST::DEST - rsync [OPTION...] SRC... rsync://[USER@]HOST[:PORT]/DEST) - -Usages with just one SRC arg and no DEST arg will list the source files -instead of copying. - -manpagedescription() - -Rsync is a fast and extraordinarily versatile file copying tool. It can -copy locally, to/from another host over any remote shell, or to/from a -remote rsync daemon. It offers a large number of options that control -every aspect of its behavior and permit very flexible specification of the -set of files to be copied. It is famous for its delta-transfer algorithm, -which reduces the amount of data sent over the network by sending only the -differences between the source files and the existing files in the -destination. Rsync is widely used for backups and mirroring and as an -improved copy command for everyday use. - -Rsync finds files that need to be transferred using a "quick check" -algorithm (by default) that looks for files that have changed in size or -in last-modified time. Any changes in the other preserved attributes (as -requested by options) are made on the destination file directly when the -quick check indicates that the file's data does not need to be updated. - -Some of the additional features of rsync are: - -itemization( - it() support for copying links, devices, owners, groups, and permissions - it() exclude and exclude-from options similar to GNU tar - it() a CVS exclude mode for ignoring the same files that CVS would ignore - it() can use any transparent remote shell, including ssh or rsh - it() does not require super-user privileges - it() pipelining of file transfers to minimize latency costs - it() support for anonymous or authenticated rsync daemons (ideal for - mirroring) -) - -manpagesection(GENERAL) - -Rsync copies files either to or from a remote host, or locally on the -current host (it does not support copying files between two remote hosts). - -There are two different ways for rsync to contact a remote system: using a -remote-shell program as the transport (such as ssh or rsh) or contacting an -rsync daemon directly via TCP. The remote-shell transport is used whenever -the source or destination path contains a single colon (:) separator after -a host specification. Contacting an rsync daemon directly happens when the -source or destination path contains a double colon (::) separator after a -host specification, OR when an rsync:// URL is specified (see also the -"USING RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION" section for -an exception to this latter rule). - -As a special case, if a single source arg is specified without a -destination, the files are listed in an output format similar to "ls -l". - -As expected, if neither the source or destination path specify a remote -host, the copy occurs locally (see also the bf(--list-only) option). - -Rsync refers to the local side as the "client" and the remote side as the -"server". Don't confuse "server" with an rsync daemon -- a daemon is always a -server, but a server can be either a daemon or a remote-shell spawned process. - -manpagesection(SETUP) - -See the file README for installation instructions. - -Once installed, you can use rsync to any machine that you can access via -a remote shell (as well as some that you can access using the rsync -daemon-mode protocol). For remote transfers, a modern rsync uses ssh -for its communications, but it may have been configured to use a -different remote shell by default, such as rsh or remsh. - -You can also specify any remote shell you like, either by using the bf(-e) -command line option, or by setting the RSYNC_RSH environment variable. - -Note that rsync must be installed on both the source and destination -machines. - -manpagesection(USAGE) - -You use rsync in the same way you use rcp. You must specify a source -and a destination, one of which may be remote. - -Perhaps the best way to explain the syntax is with some examples: - -quote(tt(rsync -t *.c foo:src/)) - -This would transfer all files matching the pattern *.c from the -current directory to the directory src on the machine foo. If any of -the files already exist on the remote system then the rsync -remote-update protocol is used to update the file by sending only the -differences in the data. Note that the expansion of wildcards on the -commandline (*.c) into a list of files is handled by the shell before -it runs rsync and not by rsync itself (exactly the same as all other -posix-style programs). - -quote(tt(rsync -avz foo:src/bar /data/tmp)) - -This would recursively transfer all files from the directory src/bar on the -machine foo into the /data/tmp/bar directory on the local machine. The -files are transferred in "archive" mode, which ensures that symbolic -links, devices, attributes, permissions, ownerships, etc. are preserved -in the transfer. Additionally, compression will be used to reduce the -size of data portions of the transfer. - -quote(tt(rsync -avz foo:src/bar/ /data/tmp)) - -A trailing slash on the source changes this behavior to avoid creating an -additional directory level at the destination. You can think of a trailing -/ on a source as meaning "copy the contents of this directory" as opposed -to "copy the directory by name", but in both cases the attributes of the -containing directory are transferred to the containing directory on the -destination. In other words, each of the following commands copies the -files in the same way, including their setting of the attributes of -/dest/foo: - -quote( -tt(rsync -av /src/foo /dest)nl() -tt(rsync -av /src/foo/ /dest/foo)nl() -) - -Note also that host and module references don't require a trailing slash to -copy the contents of the default directory. For example, both of these -copy the remote directory's contents into "/dest": - -quote( -tt(rsync -av host: /dest)nl() -tt(rsync -av host::module /dest)nl() -) - -You can also use rsync in local-only mode, where both the source and -destination don't have a ':' in the name. In this case it behaves like -an improved copy command. - -Finally, you can list all the (listable) modules available from a -particular rsync daemon by leaving off the module name: - -quote(tt(rsync somehost.mydomain.com::)) - -See the following section for more details. - -manpagesection(ADVANCED USAGE) - -The syntax for requesting multiple files from a remote host is done by -specifying additional remote-host args in the same style as the first, -or with the hostname omitted. For instance, all these work: - -quote(tt(rsync -av host:file1 :file2 host:file{3,4} /dest/)nl() -tt(rsync -av host::modname/file{1,2} host::modname/file3 /dest/)nl() -tt(rsync -av host::modname/file1 ::modname/file{3,4})) - -Older versions of rsync required using quoted spaces in the SRC, like these -examples: - -quote(tt(rsync -av host:'dir1/file1 dir2/file2' /dest)nl() -tt(rsync host::'modname/dir1/file1 modname/dir2/file2' /dest)) - -This word-splitting still works (by default) in the latest rsync, but is -not as easy to use as the first method. - -If you need to transfer a filename that contains whitespace, you can either -specify the bf(--protect-args) (bf(-s)) option, or you'll need to escape -the whitespace in a way that the remote shell will understand. For -instance: - -quote(tt(rsync -av host:'file\ name\ with\ spaces' /dest)) - -manpagesection(CONNECTING TO AN RSYNC DAEMON) - -It is also possible to use rsync without a remote shell as the transport. -In this case you will directly connect to a remote rsync daemon, typically -using TCP port 873. (This obviously requires the daemon to be running on -the remote system, so refer to the STARTING AN RSYNC DAEMON TO ACCEPT -CONNECTIONS section below for information on that.) - -Using rsync in this way is the same as using it with a remote shell except -that: - -itemization( - it() you either use a double colon :: instead of a single colon to - separate the hostname from the path, or you use an rsync:// URL. - it() the first word of the "path" is actually a module name. - it() the remote daemon may print a message of the day when you - connect. - it() if you specify no path name on the remote daemon then the - list of accessible paths on the daemon will be shown. - it() if you specify no local destination then a listing of the - specified files on the remote daemon is provided. - it() you must not specify the bf(--rsh) (bf(-e)) option. -) - -An example that copies all the files in a remote module named "src": - -verb( rsync -av host::src /dest) - -Some modules on the remote daemon may require authentication. If so, -you will receive a password prompt when you connect. You can avoid the -password prompt by setting the environment variable RSYNC_PASSWORD to -the password you want to use or using the bf(--password-file) option. This -may be useful when scripting rsync. - -WARNING: On some systems environment variables are visible to all -users. On those systems using bf(--password-file) is recommended. - -You may establish the connection via a web proxy by setting the -environment variable RSYNC_PROXY to a hostname:port pair pointing to -your web proxy. Note that your web proxy's configuration must support -proxy connections to port 873. - -You may also establish a daemon connection using a program as a proxy by -setting the environment variable RSYNC_CONNECT_PROG to the commands you -wish to run in place of making a direct socket connection. The string may -contain the escape "%H" to represent the hostname specified in the rsync -command (so use "%%" if you need a single "%" in your string). For -example: - -verb( export RSYNC_CONNECT_PROG='ssh proxyhost nc %H 873' - rsync -av targethost1::module/src/ /dest/ - rsync -av rsync:://targethost2/module/src/ /dest/ ) - -The command specified above uses ssh to run nc (netcat) on a proxyhost, -which forwards all data to port 873 (the rsync daemon) on the targethost -(%H). - -manpagesection(USING RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION) - -It is sometimes useful to use various features of an rsync daemon (such as -named modules) without actually allowing any new socket connections into a -system (other than what is already required to allow remote-shell access). -Rsync supports connecting to a host using a remote shell and then spawning -a single-use "daemon" server that expects to read its config file in the -home dir of the remote user. This can be useful if you want to encrypt a -daemon-style transfer's data, but since the daemon is started up fresh by -the remote user, you may not be able to use features such as chroot or -change the uid used by the daemon. (For another way to encrypt a daemon -transfer, consider using ssh to tunnel a local port to a remote machine and -configure a normal rsync daemon on that remote host to only allow -connections from "localhost".) - -From the user's perspective, a daemon transfer via a remote-shell -connection uses nearly the same command-line syntax as a normal -rsync-daemon transfer, with the only exception being that you must -explicitly set the remote shell program on the command-line with the -bf(--rsh=COMMAND) option. (Setting the RSYNC_RSH in the environment -will not turn on this functionality.) For example: - -verb( rsync -av --rsh=ssh host::module /dest) - -If you need to specify a different remote-shell user, keep in mind that the -user@ prefix in front of the host is specifying the rsync-user value (for a -module that requires user-based authentication). This means that you must -give the '-l user' option to ssh when specifying the remote-shell, as in -this example that uses the short version of the bf(--rsh) option: - -verb( rsync -av -e "ssh -l ssh-user" rsync-user@host::module /dest) - -The "ssh-user" will be used at the ssh level; the "rsync-user" will be -used to log-in to the "module". - -manpagesection(STARTING AN RSYNC DAEMON TO ACCEPT CONNECTIONS) - -In order to connect to an rsync daemon, the remote system needs to have a -daemon already running (or it needs to have configured something like inetd -to spawn an rsync daemon for incoming connections on a particular port). -For full information on how to start a daemon that will handling incoming -socket connections, see the bf(rsyncd.conf)(5) man page -- that is the config -file for the daemon, and it contains the full details for how to run the -daemon (including stand-alone and inetd configurations). - -If you're using one of the remote-shell transports for the transfer, there is -no need to manually start an rsync daemon. - -manpagesection(SORTED TRANSFER ORDER) - -Rsync always sorts the specified filenames into its internal transfer list. -This handles the merging together of the contents of identically named -directories, makes it easy to remove duplicate filenames, and may confuse -someone when the files are transferred in a different order than what was -given on the command-line. - -If you need a particular file to be transferred prior to another, either -separate the files into different rsync calls, or consider using -bf(--delay-updates) (which doesn't affect the sorted transfer order, but -does make the final file-updating phase happen much more rapidly). - -manpagesection(EXAMPLES) - -Here are some examples of how I use rsync. - -To backup my wife's home directory, which consists of large MS Word -files and mail folders, I use a cron job that runs - -quote(tt(rsync -Cavz . arvidsjaur:backup)) - -each night over a PPP connection to a duplicate directory on my machine -"arvidsjaur". - -To synchronize my samba source trees I use the following Makefile -targets: - -verb( get: - rsync -avuzb --exclude '*~' samba:samba/ . - put: - rsync -Cavuzb . samba:samba/ - sync: get put) - -this allows me to sync with a CVS directory at the other end of the -connection. I then do CVS operations on the remote machine, which saves a -lot of time as the remote CVS protocol isn't very efficient. - -I mirror a directory between my "old" and "new" ftp sites with the -command: - -tt(rsync -az -e ssh --delete ~ftp/pub/samba nimbus:"~ftp/pub/tridge") - -This is launched from cron every few hours. - -manpagesection(OPTIONS SUMMARY) - -Here is a short summary of the options available in rsync. Please refer -to the detailed description below for a complete description. verb( - -v, --verbose increase verbosity - --info=FLAGS fine-grained informational verbosity - --debug=FLAGS fine-grained debug verbosity - --msgs2stderr special output handling for debugging - -q, --quiet suppress non-error messages - --no-motd suppress daemon-mode MOTD (see caveat) - -c, --checksum skip based on checksum, not mod-time & size - -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) - --no-OPTION turn off an implied OPTION (e.g. --no-D) - -r, --recursive recurse into directories - -R, --relative use relative path names - --no-implied-dirs don't send implied dirs with --relative - -b, --backup make backups (see --suffix & --backup-dir) - --backup-dir=DIR make backups into hierarchy based in DIR - --suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) - -u, --update skip files that are newer on the receiver - --inplace update destination files in-place - --append append data onto shorter files - --append-verify --append w/old data in file checksum - -d, --dirs transfer directories without recursing - -l, --links copy symlinks as symlinks - -L, --copy-links transform symlink into referent file/dir - --copy-unsafe-links only "unsafe" symlinks are transformed - --safe-links ignore symlinks that point outside the tree - --munge-links munge symlinks to make them safer - -k, --copy-dirlinks transform symlink to dir into referent dir - -K, --keep-dirlinks treat symlinked dir on receiver as dir - -H, --hard-links preserve hard links - -p, --perms preserve permissions - -E, --executability preserve executability - --chmod=CHMOD affect file and/or directory permissions - -A, --acls preserve ACLs (implies -p) - -X, --xattrs preserve extended attributes - -o, --owner preserve owner (super-user only) - -g, --group preserve group - --devices preserve device files (super-user only) - --specials preserve special files - -D same as --devices --specials - -t, --times preserve modification times - -O, --omit-dir-times omit directories from --times - -J, --omit-link-times omit symlinks from --times - --super receiver attempts super-user activities - --fake-super store/recover privileged attrs using xattrs - -S, --sparse turn sequences of nulls into sparse blocks - --preallocate allocate dest files before writing - -n, --dry-run perform a trial run with no changes made - -W, --whole-file copy files whole (w/o delta-xfer algorithm) - --checksum-choice=STR choose the checksum algorithms - -x, --one-file-system don't cross filesystem boundaries - -B, --block-size=SIZE force a fixed checksum block-size - -e, --rsh=COMMAND specify the remote shell to use - --rsync-path=PROGRAM specify the rsync to run on remote machine - --existing skip creating new files on receiver - --ignore-existing skip updating files that exist on receiver - --remove-source-files sender removes synchronized files (non-dir) - --del an alias for --delete-during - --delete delete extraneous files from dest dirs - --delete-before receiver deletes before xfer, not during - --delete-during receiver deletes during the transfer - --delete-delay find deletions during, delete after - --delete-after receiver deletes after transfer, not during - --delete-excluded also delete excluded files from dest dirs - --ignore-missing-args ignore missing source args without error - --delete-missing-args delete missing source args from destination - --ignore-errors delete even if there are I/O errors - --force force deletion of dirs even if not empty - --max-delete=NUM don't delete more than NUM files - --max-size=SIZE don't transfer any file larger than SIZE - --min-size=SIZE don't transfer any file smaller than SIZE - --partial keep partially transferred files - --partial-dir=DIR put a partially transferred file into DIR - --delay-updates put all updated files into place at end - -m, --prune-empty-dirs prune empty directory chains from file-list - --numeric-ids don't map uid/gid values by user/group name - --usermap=STRING custom username mapping - --groupmap=STRING custom groupname mapping - --chown=USER:GROUP simple username/groupname mapping - --timeout=SECONDS set I/O timeout in seconds - --contimeout=SECONDS set daemon connection timeout in seconds - -I, --ignore-times don't skip files that match size and time - --size-only skip files that match in size - -@, --modify-window=NUM set the accuracy for mod-time comparisons - -T, --temp-dir=DIR create temporary files in directory DIR - -y, --fuzzy find similar file for basis if no dest file - --compare-dest=DIR also compare received files relative to DIR - --copy-dest=DIR ... and include copies of unchanged files - --link-dest=DIR hardlink to files in DIR when unchanged - -z, --compress compress file data during the transfer - --compress-level=NUM explicitly set compression level - --skip-compress=LIST skip compressing files with suffix in LIST - -C, --cvs-exclude auto-ignore files in the same way CVS does - -f, --filter=RULE add a file-filtering RULE - -F same as --filter='dir-merge /.rsync-filter' - repeated: --filter='- .rsync-filter' - --exclude=PATTERN exclude files matching PATTERN - --exclude-from=FILE read exclude patterns from FILE - --include=PATTERN don't exclude files matching PATTERN - --include-from=FILE read include patterns from FILE - --files-from=FILE read list of source-file names from FILE - -0, --from0 all *from/filter files are delimited by 0s - -s, --protect-args no space-splitting; wildcard chars only - --address=ADDRESS bind address for outgoing socket to daemon - --port=PORT specify double-colon alternate port number - --sockopts=OPTIONS specify custom TCP options - --blocking-io use blocking I/O for the remote shell - --outbuf=N|L|B set out buffering to None, Line, or Block - --stats give some file-transfer stats - -8, --8-bit-output leave high-bit chars unescaped in output - -h, --human-readable output numbers in a human-readable format - --progress show progress during transfer - -P same as --partial --progress - -i, --itemize-changes output a change-summary for all updates - -M, --remote-option=OPTION send OPTION to the remote side only - --out-format=FORMAT output updates using the specified FORMAT - --log-file=FILE log what we're doing to the specified FILE - --log-file-format=FMT log updates using the specified FMT - --password-file=FILE read daemon-access password from FILE - --list-only list the files instead of copying them - --bwlimit=RATE limit socket I/O bandwidth - --write-batch=FILE write a batched update to FILE - --only-write-batch=FILE like --write-batch but w/o updating dest - --read-batch=FILE read a batched update from FILE - --protocol=NUM force an older protocol version to be used - --iconv=CONVERT_SPEC request charset conversion of filenames - --checksum-seed=NUM set block/file checksum seed (advanced) - -4, --ipv4 prefer IPv4 - -6, --ipv6 prefer IPv6 - --version print version number -(-h) --help show this help (see below for -h comment)) - -Rsync can also be run as a daemon, in which case the following options are -accepted: verb( - --daemon run as an rsync daemon - --address=ADDRESS bind to the specified address - --bwlimit=RATE limit socket I/O bandwidth - --config=FILE specify alternate rsyncd.conf file - -M, --dparam=OVERRIDE override global daemon config parameter - --no-detach do not detach from the parent - --port=PORT listen on alternate port number - --log-file=FILE override the "log file" setting - --log-file-format=FMT override the "log format" setting - --sockopts=OPTIONS specify custom TCP options - -v, --verbose increase verbosity - -4, --ipv4 prefer IPv4 - -6, --ipv6 prefer IPv6 - -h, --help show this help (if used after --daemon)) - -manpageoptions() - -Rsync accepts both long (double-dash + word) and short (single-dash + letter) -options. The full list of the available options are described below. If an -option can be specified in more than one way, the choices are comma-separated. -Some options only have a long variant, not a short. If the option takes a -parameter, the parameter is only listed after the long variant, even though it -must also be specified for the short. When specifying a parameter, you can -either use the form --option=param or replace the '=' with whitespace. The -parameter may need to be quoted in some manner for it to survive the shell's -command-line parsing. Keep in mind that a leading tilde (~) in a filename is -substituted by your shell, so --option=~/foo will not change the tilde into -your home directory (remove the '=' for that). - -description( - -dit(bf(--help)) Print a short help page describing the options -available in rsync and exit. For backward-compatibility with older -versions of rsync, the help will also be output if you use the bf(-h) -option without any other args. - -dit(bf(--version)) print the rsync version number and exit. - -dit(bf(-v, --verbose)) This option increases the amount of information you -are given during the transfer. By default, rsync works silently. A -single bf(-v) will give you information about what files are being -transferred and a brief summary at the end. Two bf(-v) options will give you -information on what files are being skipped and slightly more -information at the end. More than two bf(-v) options should only be used if -you are debugging rsync. - -In a modern rsync, the bf(-v) option is equivalent to the setting of groups -of bf(--info) and bf(--debug) options. You can choose to use these newer -options in addition to, or in place of using bf(--verbose), as any -fine-grained settings override the implied settings of bf(-v). Both -bf(--info) and bf(--debug) have a way to ask for help that tells you -exactly what flags are set for each increase in verbosity. - -However, do keep in mind that a daemon's "max verbosity" setting will limit how -high of a level the various individual flags can be set on the daemon side. -For instance, if the max is 2, then any info and/or debug flag that is set to -a higher value than what would be set by bf(-vv) will be downgraded to the -bf(-vv) level in the daemon's logging. - -dit(bf(--info=FLAGS)) -This option lets you have fine-grained control over the -information -output you want to see. An individual flag name may be followed by a level -number, with 0 meaning to silence that output, 1 being the default output -level, and higher numbers increasing the output of that flag (for those -that support higher levels). Use -bf(--info=help) -to see all the available flag names, what they output, and what flag names -are added for each increase in the verbose level. Some examples: - -verb( rsync -a --info=progress2 src/ dest/ - rsync -avv --info=stats2,misc1,flist0 src/ dest/ ) - -Note that bf(--info=name)'s output is affected by the bf(--out-format) and -bf(--itemize-changes) (bf(-i)) options. See those options for more -information on what is output and when. - -This option was added to 3.1.0, so an older rsync on the server side might -reject your attempts at fine-grained control (if one or more flags needed -to be send to the server and the server was too old to understand them). -See also the "max verbosity" caveat above when dealing with a daemon. - -dit(bf(--debug=FLAGS)) -This option lets you have fine-grained control over the debug -output you want to see. An individual flag name may be followed by a level -number, with 0 meaning to silence that output, 1 being the default output -level, and higher numbers increasing the output of that flag (for those -that support higher levels). Use -bf(--debug=help) -to see all the available flag names, what they output, and what flag names -are added for each increase in the verbose level. Some examples: - -verb( rsync -avvv --debug=none src/ dest/ - rsync -avA --del --debug=del2,acl src/ dest/ ) - -Note that some debug messages will only be output when bf(--msgs2stderr) is -specified, especially those pertaining to I/O and buffer debugging. - -This option was added to 3.1.0, so an older rsync on the server side might -reject your attempts at fine-grained control (if one or more flags needed -to be send to the server and the server was too old to understand them). -See also the "max verbosity" caveat above when dealing with a daemon. - -dit(bf(--msgs2stderr)) This option changes rsync to send all its output -directly to stderr rather than to send messages to the client side via the -protocol (which normally outputs info messages via stdout). This is mainly -intended for debugging in order to avoid changing the data sent via the -protocol, since the extra protocol data can change what is being tested. -The option does not affect the remote side of a transfer without using -bf(--remote-option) -- e.g. bf(-M--msgs2stderr). -Also keep in mind that a daemon connection does not have a stderr channel to send -messages back to the client side, so if you are doing any daemon-transfer -debugging using this option, you should start up a daemon using bf(--no-detach) -so that you can see the stderr output on the daemon side. - -This option has the side-effect of making stderr output get line-buffered so -that the merging of the output of 3 programs happens in a more readable manner. - -dit(bf(-q, --quiet)) This option decreases the amount of information you -are given during the transfer, notably suppressing information messages -from the remote server. This option is useful when invoking rsync from -cron. - -dit(bf(--no-motd)) This option affects the information that is output -by the client at the start of a daemon transfer. This suppresses the -message-of-the-day (MOTD) text, but it also affects the list of modules -that the daemon sends in response to the "rsync host::" request (due to -a limitation in the rsync protocol), so omit this option if you want to -request the list of modules from the daemon. - -dit(bf(-I, --ignore-times)) Normally rsync will skip any files that are -already the same size and have the same modification timestamp. -This option turns off this "quick check" behavior, causing all files to -be updated. - -dit(bf(--size-only)) This modifies rsync's "quick check" algorithm for -finding files that need to be transferred, changing it from the default of -transferring files with either a changed size or a changed last-modified -time to just looking for files that have changed in size. This is useful -when starting to use rsync after using another mirroring system which may -not preserve timestamps exactly. - -dit(bf(-@, --modify-window)) When comparing two timestamps, rsync treats the -timestamps as being equal if they differ by no more than the modify-window -value. The default is 0, which matches just integer seconds. If you specify a -negative value (and the receiver is at least version 3.1.3) then nanoseconds -will also be taken into account. Specifying 1 is useful for copies to/from MS -Windows FAT filesystems, because FAT represents times with a 2-second -resolution (allowing times to differ from the original by up to 1 second). - -If you want all your transfers to default to comparing nanoseconds, you can -create a ~/.popt file and put these lines in it: - -quote(tt( rsync alias -a -a@-1)) -quote(tt( rsync alias -t -t@-1)) - -With that as the default, you'd need to specify bf(--modify-window=0) (aka -bf(-@0)) to override it and ignore nanoseconds, e.g. if you're copying between -ext3 and ext4, or if the receiving rsync is older than 3.1.3. - -dit(bf(-c, --checksum)) This changes the way rsync checks if the files have -been changed and are in need of a transfer. Without this option, rsync -uses a "quick check" that (by default) checks if each file's size and time -of last modification match between the sender and receiver. This option -changes this to compare a 128-bit checksum for each file that has a -matching size. Generating the checksums means that both sides will expend -a lot of disk I/O reading all the data in the files in the transfer (and -this is prior to any reading that will be done to transfer changed files), -so this can slow things down significantly. - -The sending side generates its checksums while it is doing the file-system -scan that builds the list of the available files. The receiver generates -its checksums when it is scanning for changed files, and will checksum any -file that has the same size as the corresponding sender's file: files with -either a changed size or a changed checksum are selected for transfer. - -Note that rsync always verifies that each em(transferred) file was -correctly reconstructed on the receiving side by checking a whole-file -checksum that is generated as the file is transferred, but that -automatic after-the-transfer verification has nothing to do with this -option's before-the-transfer "Does this file need to be updated?" check. - -For protocol 30 and beyond (first supported in 3.0.0), the checksum used is -MD5. For older protocols, the checksum used is MD4. - -dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick -way of saying you want recursion and want to preserve almost -everything (with -H being a notable omission). -The only exception to the above equivalence is when bf(--files-from) is -specified, in which case bf(-r) is not implied. - -Note that bf(-a) bf(does not preserve hardlinks), because -finding multiply-linked files is expensive. You must separately -specify bf(-H). - -dit(--no-OPTION) You may turn off one or more implied options by prefixing -the option name with "no-". Not all options may be prefixed with a "no-": -only options that are implied by other options (e.g. bf(--no-D), -bf(--no-perms)) or have different defaults in various circumstances -(e.g. bf(--no-whole-file), bf(--no-blocking-io), bf(--no-dirs)). You may -specify either the short or the long option name after the "no-" prefix -(e.g. bf(--no-R) is the same as bf(--no-relative)). - -For example: if you want to use bf(-a) (bf(--archive)) but don't want -bf(-o) (bf(--owner)), instead of converting bf(-a) into bf(-rlptgD), you -could specify bf(-a --no-o) (or bf(-a --no-owner)). - -The order of the options is important: if you specify bf(--no-r -a), the -bf(-r) option would end up being turned on, the opposite of bf(-a --no-r). -Note also that the side-effects of the bf(--files-from) option are NOT -positional, as it affects the default state of several options and slightly -changes the meaning of bf(-a) (see the bf(--files-from) option for more -details). - -dit(bf(-r, --recursive)) This tells rsync to copy directories -recursively. See also bf(--dirs) (bf(-d)). - -Beginning with rsync 3.0.0, the recursive algorithm used is now an -incremental scan that uses much less memory than before and begins the -transfer after the scanning of the first few directories have been -completed. This incremental scan only affects our recursion algorithm, and -does not change a non-recursive transfer. It is also only possible when -both ends of the transfer are at least version 3.0.0. - -Some options require rsync to know the full file list, so these options -disable the incremental recursion mode. These include: bf(--delete-before), -bf(--delete-after), bf(--prune-empty-dirs), and bf(--delay-updates). -Because of this, the default delete mode when you specify bf(--delete) is now -bf(--delete-during) when both ends of the connection are at least 3.0.0 -(use bf(--del) or bf(--delete-during) to request this improved deletion mode -explicitly). See also the bf(--delete-delay) option that is a better choice -than using bf(--delete-after). - -Incremental recursion can be disabled using the bf(--no-inc-recursive) -option or its shorter bf(--no-i-r) alias. - -dit(bf(-R, --relative)) Use relative paths. This means that the full path -names specified on the command line are sent to the server rather than -just the last parts of the filenames. This is particularly useful when -you want to send several different directories at the same time. For -example, if you used this command: - -quote(tt( rsync -av /foo/bar/baz.c remote:/tmp/)) - -... this would create a file named baz.c in /tmp/ on the remote -machine. If instead you used - -quote(tt( rsync -avR /foo/bar/baz.c remote:/tmp/)) - -then a file named /tmp/foo/bar/baz.c would be created on the remote -machine, preserving its full path. These extra path elements are called -"implied directories" (i.e. the "foo" and the "foo/bar" directories in the -above example). - -Beginning with rsync 3.0.0, rsync always sends these implied directories as -real directories in the file list, even if a path element is really a -symlink on the sending side. This prevents some really unexpected -behaviors when copying the full path of a file that you didn't realize had -a symlink in its path. If you want to duplicate a server-side symlink, -include both the symlink via its path, and referent directory via its real -path. If you're dealing with an older rsync on the sending side, you may -need to use the bf(--no-implied-dirs) option. - -It is also possible to limit the amount of path information that is sent as -implied directories for each path you specify. With a modern rsync on the -sending side (beginning with 2.6.7), you can insert a dot and a slash into -the source path, like this: - -quote(tt( rsync -avR /foo/./bar/baz.c remote:/tmp/)) - -That would create /tmp/bar/baz.c on the remote machine. (Note that the -dot must be followed by a slash, so "/foo/." would not be abbreviated.) -For older rsync versions, you would need to use a chdir to limit the -source path. For example, when pushing files: - -quote(tt( (cd /foo; rsync -avR bar/baz.c remote:/tmp/) )) - -(Note that the parens put the two commands into a sub-shell, so that the -"cd" command doesn't remain in effect for future commands.) -If you're pulling files from an older rsync, use this idiom (but only -for a non-daemon transfer): - -quote( -tt( rsync -avR --rsync-path="cd /foo; rsync" \ )nl() -tt( remote:bar/baz.c /tmp/) -) - -dit(bf(--no-implied-dirs)) This option affects the default behavior of the -bf(--relative) option. When it is specified, the attributes of the implied -directories from the source names are not included in the transfer. This -means that the corresponding path elements on the destination system are -left unchanged if they exist, and any missing implied directories are -created with default attributes. This even allows these implied path -elements to have big differences, such as being a symlink to a directory on -the receiving side. - -For instance, if a command-line arg or a files-from entry told rsync to -transfer the file "path/foo/file", the directories "path" and "path/foo" -are implied when bf(--relative) is used. If "path/foo" is a symlink to -"bar" on the destination system, the receiving rsync would ordinarily -delete "path/foo", recreate it as a directory, and receive the file into -the new directory. With bf(--no-implied-dirs), the receiving rsync updates -"path/foo/file" using the existing path elements, which means that the file -ends up being created in "path/bar". Another way to accomplish this link -preservation is to use the bf(--keep-dirlinks) option (which will also -affect symlinks to directories in the rest of the transfer). - -When pulling files from an rsync older than 3.0.0, you may need to use this -option if the sending side has a symlink in the path you request and you -wish the implied directories to be transferred as normal directories. - -dit(bf(-b, --backup)) With this option, preexisting destination files are -renamed as each file is transferred or deleted. You can control where the -backup file goes and what (if any) suffix gets appended using the -bf(--backup-dir) and bf(--suffix) options. - -Note that if you don't specify bf(--backup-dir), (1) the -bf(--omit-dir-times) option will be forced on, and (2) if bf(--delete) is -also in effect (without bf(--delete-excluded)), rsync will add a "protect" -filter-rule for the backup suffix to the end of all your existing excludes -(e.g. bf(-f "P *~")). This will prevent previously backed-up files from being -deleted. Note that if you are supplying your own filter rules, you may -need to manually insert your own exclude/protect rule somewhere higher up -in the list so that it has a high enough priority to be effective (e.g., if -your rules specify a trailing inclusion/exclusion of '*', the auto-added -rule would never be reached). - -dit(bf(--backup-dir=DIR)) In combination with the bf(--backup) option, this -tells rsync to store all backups in the specified directory on the receiving -side. This can be used for incremental backups. You can additionally -specify a backup suffix using the bf(--suffix) option -(otherwise the files backed up in the specified directory -will keep their original filenames). - -Note that if you specify a relative path, the backup directory will be -relative to the destination directory, so you probably want to specify -either an absolute path or a path that starts with "../". If an rsync -daemon is the receiver, the backup dir cannot go outside the module's path -hierarchy, so take extra care not to delete it or copy into it. - -dit(bf(--suffix=SUFFIX)) This option allows you to override the default -backup suffix used with the bf(--backup) (bf(-b)) option. The default suffix is a ~ -if no -bf(-backup-dir) was specified, otherwise it is an empty string. - -dit(bf(-u, --update)) This forces rsync to skip any files which exist on -the destination and have a modified time that is newer than the source -file. (If an existing destination file has a modification time equal to the -source file's, it will be updated if the sizes are different.) - -Note that this does not affect the copying of dirs, symlinks, or other special -files. Also, a difference of file format between the sender and receiver -is always considered to be important enough for an update, no matter what -date is on the objects. In other words, if the source has a directory -where the destination has a file, the transfer would occur regardless of -the timestamps. - -This option is a transfer rule, not an exclude, so it doesn't affect the -data that goes into the file-lists, and thus it doesn't affect deletions. -It just limits the files that the receiver requests to be transferred. - -dit(bf(--inplace)) This option changes how rsync transfers a file when -its data needs to be updated: instead of the default method of creating -a new copy of the file and moving it into place when it is complete, rsync -instead writes the updated data directly to the destination file. - -This has several effects: - -quote(itemization( - it() Hard links are not broken. This means the new data will be visible - through other hard links to the destination file. Moreover, attempts to - copy differing source files onto a multiply-linked destination file will - result in a "tug of war" with the destination data changing back and forth. - it() In-use binaries cannot be updated (either the OS will prevent this from - happening, or binaries that attempt to swap-in their data will misbehave or - crash). - it() The file's data will be in an inconsistent state during the transfer - and will be left that way if the transfer is interrupted or if an update - fails. - it() A file that rsync cannot write to cannot be updated. While a super user - can update any file, a normal user needs to be granted write permission for - the open of the file for writing to be successful. - it() The efficiency of rsync's delta-transfer algorithm may be reduced if - some data in the destination file is overwritten before it can be copied to - a position later in the file. This does not apply if you use bf(--backup), - since rsync is smart enough to use the backup file as the basis file for the - transfer. -)) - -WARNING: you should not use this option to update files that are being -accessed by others, so be careful when choosing to use this for a copy. - -This option is useful for transferring large files with block-based changes -or appended data, and also on systems that are disk bound, not network -bound. It can also help keep a copy-on-write filesystem snapshot from -diverging the entire contents of a file that only has minor changes. - -The option implies bf(--partial) (since an interrupted transfer does not delete -the file), but conflicts with bf(--partial-dir) and bf(--delay-updates). -Prior to rsync 2.6.4 bf(--inplace) was also incompatible with bf(--compare-dest) -and bf(--link-dest). - -dit(bf(--append)) This causes rsync to update a file by appending data onto -the end of the file, which presumes that the data that already exists on -the receiving side is identical with the start of the file on the sending -side. If a file needs to be transferred and its size on the receiver is -the same or longer than the size on the sender, the file is skipped. This -does not interfere with the updating of a file's non-content attributes -(e.g. permissions, ownership, etc.) when the file does not need to be -transferred, nor does it affect the updating of any non-regular files. -Implies bf(--inplace). - -The use of bf(--append) can be dangerous if you aren't 100% sure that the files -that are longer have only grown by the appending of data onto the end. You -should thus use include/exclude/filter rules to ensure that such a transfer is -only affecting files that you know to be growing via appended data. - -dit(bf(--append-verify)) This works just like the bf(--append) option, but -the existing data on the receiving side is included in the full-file -checksum verification step, which will cause a file to be resent if the -final verification step fails (rsync uses a normal, non-appending -bf(--inplace) transfer for the resend). - -Note: prior to rsync 3.0.0, the bf(--append) option worked like -bf(--append-verify), so if you are interacting with an older rsync (or the -transfer is using a protocol prior to 30), specifying either append option -will initiate an bf(--append-verify) transfer. - -dit(bf(-d, --dirs)) Tell the sending side to include any directories that -are encountered. Unlike bf(--recursive), a directory's contents are not copied -unless the directory name specified is "." or ends with a trailing slash -(e.g. ".", "dir/.", "dir/", etc.). Without this option or the -bf(--recursive) option, rsync will skip all directories it encounters (and -output a message to that effect for each one). If you specify both -bf(--dirs) and bf(--recursive), bf(--recursive) takes precedence. - -The bf(--dirs) option is implied by the bf(--files-from) option -or the bf(--list-only) option (including an implied -bf(--list-only) usage) if bf(--recursive) wasn't specified (so that -directories are seen in the listing). Specify bf(--no-dirs) (or bf(--no-d)) -if you want to turn this off. - -There is also a backward-compatibility helper option, bf(--old-dirs) (or -bf(--old-d)) that tells rsync to use a hack of "-r --exclude='/*/*'" to get -an older rsync to list a single directory without recursing. - -) -description( - -dit(bf(-l, --links)) When symlinks are encountered, recreate the -symlink on the destination. - -dit(bf(-L, --copy-links)) When symlinks are encountered, the item that -they point to (the referent) is copied, rather than the symlink. In older -versions of rsync, this option also had the side-effect of telling the -receiving side to follow symlinks, such as symlinks to directories. In a -modern rsync such as this one, you'll need to specify bf(--keep-dirlinks) (bf(-K)) -to get this extra behavior. The only exception is when sending files to -an rsync that is too old to understand bf(-K) -- in that case, the bf(-L) option -will still have the side-effect of bf(-K) on that older receiving rsync. - -dit(bf(--copy-unsafe-links)) This tells rsync to copy the referent of -symbolic links that point outside the copied tree. Absolute symlinks -are also treated like ordinary files, and so are any symlinks in the -source path itself when bf(--relative) is used. This option has no -additional effect if bf(--copy-links) was also specified. - -dit(bf(--safe-links)) This tells rsync to ignore any symbolic links -which point outside the copied tree. All absolute symlinks are -also ignored. Using this option in conjunction with bf(--relative) may -give unexpected results. - -dit(bf(--munge-links)) This option tells rsync to (1) modify all symlinks on -the receiving side in a way that makes them unusable but recoverable (see -below), or (2) to unmunge symlinks on the sending side that had been stored in -a munged state. This is useful if you don't quite trust the source of the data -to not try to slip in a symlink to a unexpected place. - -The way rsync disables the use of symlinks is to prefix each one with the -string "/rsyncd-munged/". This prevents the links from being used as long as -that directory does not exist. When this option is enabled, rsync will refuse -to run if that path is a directory or a symlink to a directory. - -The option only affects the client side of the transfer, so if you need it to -affect the server, specify it via bf(--remote-option). (Note that in a local -transfer, the client side is the sender.) - -This option has no affect on a daemon, since the daemon configures whether it -wants munged symlinks via its "munge symlinks" parameter. See also the -"munge-symlinks" perl script in the support directory of the source code. - -dit(bf(-k, --copy-dirlinks)) This option causes the sending side to treat -a symlink to a directory as though it were a real directory. This is -useful if you don't want symlinks to non-directories to be affected, as -they would be using bf(--copy-links). - -Without this option, if the sending side has replaced a directory with a -symlink to a directory, the receiving side will delete anything that is in -the way of the new symlink, including a directory hierarchy (as long as -bf(--force) or bf(--delete) is in effect). - -See also bf(--keep-dirlinks) for an analogous option for the receiving -side. - -bf(--copy-dirlinks) applies to all symlinks to directories in the source. If -you want to follow only a few specified symlinks, a trick you can use is to -pass them as additional source args with a trailing slash, using bf(--relative) -to make the paths match up right. For example: - -quote(tt(rsync -r --relative src/./ src/./follow-me/ dest/)) - -This works because rsync calls bf(lstat)(2) on the source arg as given, and the -trailing slash makes bf(lstat)(2) follow the symlink, giving rise to a directory -in the file-list which overrides the symlink found during the scan of "src/./". - -) -description( - -dit(bf(-K, --keep-dirlinks)) This option causes the receiving side to treat -a symlink to a directory as though it were a real directory, but only if it -matches a real directory from the sender. Without this option, the -receiver's symlink would be deleted and replaced with a real directory. - -For example, suppose you transfer a directory "foo" that contains a file -"file", but "foo" is a symlink to directory "bar" on the receiver. Without -bf(--keep-dirlinks), the receiver deletes symlink "foo", recreates it as a -directory, and receives the file into the new directory. With -bf(--keep-dirlinks), the receiver keeps the symlink and "file" ends up in -"bar". - -One note of caution: if you use bf(--keep-dirlinks), you must trust all -the symlinks in the copy! If it is possible for an untrusted user to -create their own symlink to any directory, the user could then (on a -subsequent copy) replace the symlink with a real directory and affect the -content of whatever directory the symlink references. For backup copies, -you are better off using something like a bind mount instead of a symlink -to modify your receiving hierarchy. - -See also bf(--copy-dirlinks) for an analogous option for the sending side. - -dit(bf(-H, --hard-links)) This tells rsync to look for hard-linked files in -the source and link together the corresponding files on the destination. -Without this option, hard-linked files in the source are treated -as though they were separate files. - -This option does NOT necessarily ensure that the pattern of hard links on the -destination exactly matches that on the source. Cases in which the -destination may end up with extra hard links include the following: - -quote(itemization( - it() If the destination contains extraneous hard-links (more linking than - what is present in the source file list), the copying algorithm will not - break them explicitly. However, if one or more of the paths have content - differences, the normal file-update process will break those extra links - (unless you are using the bf(--inplace) option). - it() If you specify a bf(--link-dest) directory that contains hard links, - the linking of the destination files against the bf(--link-dest) files can - cause some paths in the destination to become linked together due to the - bf(--link-dest) associations. -)) - -Note that rsync can only detect hard links between files that are inside -the transfer set. If rsync updates a file that has extra hard-link -connections to files outside the transfer, that linkage will be broken. If -you are tempted to use the bf(--inplace) option to avoid this breakage, be -very careful that you know how your files are being updated so that you are -certain that no unintended changes happen due to lingering hard links (and -see the bf(--inplace) option for more caveats). - -If incremental recursion is active (see bf(--recursive)), rsync may transfer -a missing hard-linked file before it finds that another link for that contents -exists elsewhere in the hierarchy. This does not affect the accuracy of -the transfer (i.e. which files are hard-linked together), just its efficiency -(i.e. copying the data for a new, early copy of a hard-linked file that could -have been found later in the transfer in another member of the hard-linked -set of files). One way to avoid this inefficiency is to disable -incremental recursion using the bf(--no-inc-recursive) option. - -dit(bf(-p, --perms)) This option causes the receiving rsync to set the -destination permissions to be the same as the source permissions. (See -also the bf(--chmod) option for a way to modify what rsync considers to -be the source permissions.) - -When this option is em(off), permissions are set as follows: - -quote(itemization( - it() Existing files (including updated files) retain their existing - permissions, though the bf(--executability) option might change just - the execute permission for the file. - it() New files get their "normal" permission bits set to the source - file's permissions masked with the receiving directory's default - permissions (either the receiving process's umask, or the permissions - specified via the destination directory's default ACL), and - their special permission bits disabled except in the case where a new - directory inherits a setgid bit from its parent directory. -)) - -Thus, when bf(--perms) and bf(--executability) are both disabled, -rsync's behavior is the same as that of other file-copy utilities, -such as bf(cp)(1) and bf(tar)(1). - -In summary: to give destination files (both old and new) the source -permissions, use bf(--perms). To give new files the destination-default -permissions (while leaving existing files unchanged), make sure that the -bf(--perms) option is off and use bf(--chmod=ugo=rwX) (which ensures that -all non-masked bits get enabled). If you'd care to make this latter -behavior easier to type, you could define a popt alias for it, such as -putting this line in the file ~/.popt (the following defines the bf(-Z) option, -and includes --no-g to use the default group of the destination dir): - -quote(tt( rsync alias -Z --no-p --no-g --chmod=ugo=rwX)) - -You could then use this new option in a command such as this one: - -quote(tt( rsync -avZ src/ dest/)) - -(Caveat: make sure that bf(-a) does not follow bf(-Z), or it will re-enable -the two "--no-*" options mentioned above.) - -The preservation of the destination's setgid bit on newly-created -directories when bf(--perms) is off was added in rsync 2.6.7. Older rsync -versions erroneously preserved the three special permission bits for -newly-created files when bf(--perms) was off, while overriding the -destination's setgid bit setting on a newly-created directory. Default ACL -observance was added to the ACL patch for rsync 2.6.7, so older (or -non-ACL-enabled) rsyncs use the umask even if default ACLs are present. -(Keep in mind that it is the version of the receiving rsync that affects -these behaviors.) - -dit(bf(-E, --executability)) This option causes rsync to preserve the -executability (or non-executability) of regular files when bf(--perms) is -not enabled. A regular file is considered to be executable if at least one -'x' is turned on in its permissions. When an existing destination file's -executability differs from that of the corresponding source file, rsync -modifies the destination file's permissions as follows: - -quote(itemization( - it() To make a file non-executable, rsync turns off all its 'x' - permissions. - it() To make a file executable, rsync turns on each 'x' permission that - has a corresponding 'r' permission enabled. -)) - -If bf(--perms) is enabled, this option is ignored. - -dit(bf(-A, --acls)) This option causes rsync to update the destination -ACLs to be the same as the source ACLs. -The option also implies bf(--perms). - -The source and destination systems must have compatible ACL entries for this -option to work properly. See the bf(--fake-super) option for a way to backup -and restore ACLs that are not compatible. - -dit(bf(-X, --xattrs)) This option causes rsync to update the destination -extended attributes to be the same as the source ones. - -For systems that support extended-attribute namespaces, a copy being done by a -super-user copies all namespaces except system.*. A normal user only copies -the user.* namespace. To be able to backup and restore non-user namespaces as -a normal user, see the bf(--fake-super) option. - -The above name filtering can be overridden by using one or more filter options -with the bf(x) modifier. When you specify an xattr-affecting filter rule, rsync -requires that you do your own system/user filtering, as well as any additional -filtering for what xattr names are copied and what names are allowed to be -deleted. For example, to skip the system namespace, you could specify: - -quote(--filter='-x system.*') - -To skip all namespaces except the user namespace, you could specify a -negated-user match: - -quote(--filter='-x! user.*') - -To prevent any attributes from being deleted, you could specify a receiver-only -rule that excludes all names: - -quote(--filter='-xr *') - -Note that the bf(-X) option does not copy rsync's special xattr values (e.g. -those used by bf(--fake-super)) unless you repeat the option (e.g. -XX). -This "copy all xattrs" mode cannot be used with bf(--fake-super). - -dit(bf(--chmod)) This option tells rsync to apply one or more -comma-separated "chmod" modes to the permission of the files in the -transfer. The resulting value is treated as though it were the permissions -that the sending side supplied for the file, which means that this option -can seem to have no effect on existing files if bf(--perms) is not enabled. - -In addition to the normal parsing rules specified in the bf(chmod)(1) -manpage, you can specify an item that should only apply to a directory by -prefixing it with a 'D', or specify an item that should only apply to a -file by prefixing it with a 'F'. For example, the following will ensure -that all directories get marked set-gid, that no files are other-writable, -that both are user-writable and group-writable, and that both have -consistent executability across all bits: - -quote(--chmod=Dg+s,ug+w,Fo-w,+X) - -Using octal mode numbers is also allowed: - -quote(--chmod=D2775,F664) - -It is also legal to specify multiple bf(--chmod) options, as each -additional option is just appended to the list of changes to make. - -See the bf(--perms) and bf(--executability) options for how the resulting -permission value can be applied to the files in the transfer. - -dit(bf(-o, --owner)) This option causes rsync to set the owner of the -destination file to be the same as the source file, but only if the -receiving rsync is being run as the super-user (see also the bf(--super) -and bf(--fake-super) options). -Without this option, the owner of new and/or transferred files are set to -the invoking user on the receiving side. - -The preservation of ownership will associate matching names by default, but -may fall back to using the ID number in some circumstances (see also the -bf(--numeric-ids) option for a full discussion). - -dit(bf(-g, --group)) This option causes rsync to set the group of the -destination file to be the same as the source file. If the receiving -program is not running as the super-user (or if bf(--no-super) was -specified), only groups that the invoking user on the receiving side -is a member of will be preserved. -Without this option, the group is set to the default group of the invoking -user on the receiving side. - -The preservation of group information will associate matching names by -default, but may fall back to using the ID number in some circumstances -(see also the bf(--numeric-ids) option for a full discussion). - -dit(bf(--devices)) This option causes rsync to transfer character and -block device files to the remote system to recreate these devices. -This option has no effect if the receiving rsync is not run as the -super-user (see also the bf(--super) and bf(--fake-super) options). - -dit(bf(--specials)) This option causes rsync to transfer special files -such as named sockets and fifos. - -dit(bf(-D)) The bf(-D) option is equivalent to bf(--devices) bf(--specials). - -dit(bf(-t, --times)) This tells rsync to transfer modification times along -with the files and update them on the remote system. Note that if this -option is not used, the optimization that excludes files that have not been -modified cannot be effective; in other words, a missing bf(-t) or bf(-a) will -cause the next transfer to behave as if it used bf(-I), causing all files to be -updated (though rsync's delta-transfer algorithm will make the update fairly efficient -if the files haven't actually changed, you're much better off using bf(-t)). - -dit(bf(-O, --omit-dir-times)) This tells rsync to omit directories when -it is preserving modification times (see bf(--times)). If NFS is sharing -the directories on the receiving side, it is a good idea to use bf(-O). -This option is inferred if you use bf(--backup) without bf(--backup-dir). - -This option also has the side-effect of avoiding early creation of directories -in incremental recursion copies. The default bf(--inc-recursive) copying -normally does an early-create pass of all the sub-directories in a parent -directory in order for it to be able to then set the modify time of the parent -directory right away (without having to delay that until a bunch of recursive -copying has finished). This early-create idiom is not necessary if directory -modify times are not being preserved, so it is skipped. Since early-create -directories don't have accurate mode, mtime, or ownership, the use of this -option can help when someone wants to avoid these partially-finished -directories. - -dit(bf(-J, --omit-link-times)) This tells rsync to omit symlinks when -it is preserving modification times (see bf(--times)). - -dit(bf(--super)) This tells the receiving side to attempt super-user -activities even if the receiving rsync wasn't run by the super-user. These -activities include: preserving users via the bf(--owner) option, preserving -all groups (not just the current user's groups) via the bf(--groups) -option, and copying devices via the bf(--devices) option. This is useful -for systems that allow such activities without being the super-user, and -also for ensuring that you will get errors if the receiving side isn't -being run as the super-user. To turn off super-user activities, the -super-user can use bf(--no-super). - -dit(bf(--fake-super)) When this option is enabled, rsync simulates -super-user activities by saving/restoring the privileged attributes via -special extended attributes that are attached to each file (as needed). This -includes the file's owner and group (if it is not the default), the file's -device info (device & special files are created as empty text files), and -any permission bits that we won't allow to be set on the real file (e.g. -the real file gets u-s,g-s,o-t for safety) or that would limit the owner's -access (since the real super-user can always access/change a file, the -files we create can always be accessed/changed by the creating user). -This option also handles ACLs (if bf(--acls) was specified) and non-user -extended attributes (if bf(--xattrs) was specified). - -This is a good way to backup data without using a super-user, and to store -ACLs from incompatible systems. - -The bf(--fake-super) option only affects the side where the option is used. -To affect the remote side of a remote-shell connection, use the -bf(--remote-option) (bf(-M)) option: - -quote(tt( rsync -av -M--fake-super /src/ host:/dest/)) - -For a local copy, this option affects both the source and the destination. -If you wish a local copy to enable this option just for the destination -files, specify bf(-M--fake-super). If you wish a local copy to enable -this option just for the source files, combine bf(--fake-super) with -bf(-M--super). - -This option is overridden by both bf(--super) and bf(--no-super). - -See also the "fake super" setting in the daemon's rsyncd.conf file. - -dit(bf(-S, --sparse)) Try to handle sparse files efficiently so they take -up less space on the destination. If combined with bf(--inplace) the -file created might not end up with sparse blocks with some combinations -of kernel version and/or filesystem type. If bf(--whole-file) is in -effect (e.g. for a local copy) then it will always work because rsync -truncates the file prior to writing out the updated version. - -Note that versions of rsync older than 3.1.3 will reject the combination of -bf(--sparse) and bf(--inplace). - -dit(bf(--preallocate)) This tells the receiver to allocate each destination -file to its eventual size before writing data to the file. Rsync will only -use the real filesystem-level preallocation support provided by Linux's -bf(fallocate)(2) system call or Cygwin's bf(posix_fallocate)(3), not the slow -glibc implementation that writes a null byte into each block. - -Without this option, larger files may not be entirely contiguous on the -filesystem, but with this option rsync will probably copy more slowly. If the -destination is not an extent-supporting filesystem (such as ext4, xfs, NTFS, -etc.), this option may have no positive effect at all. - -If combined with bf(--sparse), the file will only have sparse blocks (as -opposed to allocated sequences of null bytes) if the kernel version and -filesystem type support creating holes in the allocated data. - -dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't -make any changes (and produces mostly the same output as a real run). It -is most commonly used in combination with the bf(-v, --verbose) and/or -bf(-i, --itemize-changes) options to see what an rsync command is going -to do before one actually runs it. - -The output of bf(--itemize-changes) is supposed to be exactly the same on a -dry run and a subsequent real run (barring intentional trickery and system -call failures); if it isn't, that's a bug. Other output should be mostly -unchanged, but may differ in some areas. Notably, a dry run does not -send the actual data for file transfers, so bf(--progress) has no effect, -the "bytes sent", "bytes received", "literal data", and "matched data" -statistics are too small, and the "speedup" value is equivalent to a run -where no file transfers were needed. - -dit(bf(-W, --whole-file)) This option disables rsync's delta-transfer algorithm, -which causes all transferred files to be sent whole. The transfer may be -faster if this option is used when the bandwidth between the source and -destination machines is higher than the bandwidth to disk (especially when the -"disk" is actually a networked filesystem). This is the default when both -the source and destination are specified as local paths, but only if no -batch-writing option is in effect. - -dit(bf(--checksum-choice=STR)) This option overrides the checksum algoriths. -If one algorithm name is specified, it is used for both the transfer checksums -and (assuming bf(--checksum) is specifed) the pre-transfer checksumming. If two -comma-separated names are supplied, the first name affects the transfer -checksums, and the second name affects the pre-transfer checksumming. - -The algorithm choices are "auto", "md4", "md5", and "none". If "none" is -specified for the first name, the bf(--whole-file) option is forced on and no -checksum verification is performed on the transferred data. If "none" is -specified for the second name, the bf(--checksum) option cannot be used. The -"auto" option is the default, where rsync bases its algorithm choice on the -protocol version (for backward compatibility with older rsync versions). - -dit(bf(-x, --one-file-system)) This tells rsync to avoid crossing a -filesystem boundary when recursing. This does not limit the user's ability -to specify items to copy from multiple filesystems, just rsync's recursion -through the hierarchy of each directory that the user specified, and also -the analogous recursion on the receiving side during deletion. Also keep -in mind that rsync treats a "bind" mount to the same device as being on the -same filesystem. - -If this option is repeated, rsync omits all mount-point directories from -the copy. Otherwise, it includes an empty directory at each mount-point it -encounters (using the attributes of the mounted directory because those of -the underlying mount-point directory are inaccessible). - -If rsync has been told to collapse symlinks (via bf(--copy-links) or -bf(--copy-unsafe-links)), a symlink to a directory on another device is -treated like a mount-point. Symlinks to non-directories are unaffected -by this option. - -) -description( - -dit(bf(--existing, --ignore-non-existing)) This tells rsync to skip -creating files (including directories) that do not exist -yet on the destination. If this option is -combined with the bf(--ignore-existing) option, no files will be updated -(which can be useful if all you want to do is delete extraneous files). - -This option is a transfer rule, not an exclude, so it doesn't affect the -data that goes into the file-lists, and thus it doesn't affect deletions. -It just limits the files that the receiver requests to be transferred. - -dit(bf(--ignore-existing)) This tells rsync to skip updating files that -already exist on the destination (this does em(not) ignore existing -directories, or nothing would get done). See also bf(--existing). - -This option is a transfer rule, not an exclude, so it doesn't affect the -data that goes into the file-lists, and thus it doesn't affect deletions. -It just limits the files that the receiver requests to be transferred. - -This option can be useful for those doing backups using the bf(--link-dest) -option when they need to continue a backup run that got interrupted. Since -a bf(--link-dest) run is copied into a new directory hierarchy (when it is -used properly), using bf(--ignore existing) will ensure that the -already-handled files don't get tweaked (which avoids a change in -permissions on the hard-linked files). This does mean that this option -is only looking at the existing files in the destination hierarchy itself. - -dit(bf(--remove-source-files)) This tells rsync to remove from the sending -side the files (meaning non-directories) that are a part of the transfer -and have been successfully duplicated on the receiving side. - -Note that you should only use this option on source files that are quiescent. -If you are using this to move files that show up in a particular directory over -to another host, make sure that the finished files get renamed into the source -directory, not directly written into it, so that rsync can't possibly transfer -a file that is not yet fully written. If you can't first write the files into -a different directory, you should use a naming idiom that lets rsync avoid -transferring files that are not yet finished (e.g. name the file "foo.new" when -it is written, rename it to "foo" when it is done, and then use the option -bf(--exclude='*.new') for the rsync transfer). - -Starting with 3.1.0, rsync will skip the sender-side removal (and output an -error) if the file's size or modify time has not stayed unchanged. - -dit(bf(--delete)) This tells rsync to delete extraneous files from the -receiving side (ones that aren't on the sending side), but only for the -directories that are being synchronized. You must have asked rsync to -send the whole directory (e.g. "dir" or "dir/") without using a wildcard -for the directory's contents (e.g. "dir/*") since the wildcard is expanded -by the shell and rsync thus gets a request to transfer individual files, not -the files' parent directory. Files that are excluded from the transfer are -also excluded from being deleted unless you use the bf(--delete-excluded) -option or mark the rules as only matching on the sending side (see the -include/exclude modifiers in the FILTER RULES section). - -Prior to rsync 2.6.7, this option would have no effect unless bf(--recursive) -was enabled. Beginning with 2.6.7, deletions will also occur when bf(--dirs) -(bf(-d)) is enabled, but only for directories whose contents are being copied. - -This option can be dangerous if used incorrectly! It is a very good idea to -first try a run using the bf(--dry-run) option (bf(-n)) to see what files are -going to be deleted. - -If the sending side detects any I/O errors, then the deletion of any -files at the destination will be automatically disabled. This is to -prevent temporary filesystem failures (such as NFS errors) on the -sending side from causing a massive deletion of files on the -destination. You can override this with the bf(--ignore-errors) option. - -The bf(--delete) option may be combined with one of the --delete-WHEN options -without conflict, as well as bf(--delete-excluded). However, if none of the ---delete-WHEN options are specified, rsync will choose the -bf(--delete-during) algorithm when talking to rsync 3.0.0 or newer, and -the bf(--delete-before) algorithm when talking to an older rsync. See also -bf(--delete-delay) and bf(--delete-after). - -dit(bf(--delete-before)) Request that the file-deletions on the receiving -side be done before the transfer starts. -See bf(--delete) (which is implied) for more details on file-deletion. - -Deleting before the transfer is helpful if the filesystem is tight for space -and removing extraneous files would help to make the transfer possible. -However, it does introduce a delay before the start of the transfer, -and this delay might cause the transfer to timeout (if bf(--timeout) was -specified). It also forces rsync to use the old, non-incremental recursion -algorithm that requires rsync to scan all the files in the transfer into -memory at once (see bf(--recursive)). - -dit(bf(--delete-during, --del)) Request that the file-deletions on the -receiving side be done incrementally as the transfer happens. The -per-directory delete scan is done right before each directory is checked -for updates, so it behaves like a more efficient bf(--delete-before), -including doing the deletions prior to any per-directory filter files -being updated. This option was first added in rsync version 2.6.4. -See bf(--delete) (which is implied) for more details on file-deletion. - -dit(bf(--delete-delay)) Request that the file-deletions on the receiving -side be computed during the transfer (like bf(--delete-during)), and then -removed after the transfer completes. This is useful when combined with -bf(--delay-updates) and/or bf(--fuzzy), and is more efficient than using -bf(--delete-after) (but can behave differently, since bf(--delete-after) -computes the deletions in a separate pass after all updates are done). -If the number of removed files overflows an internal buffer, a -temporary file will be created on the receiving side to hold the names (it -is removed while open, so you shouldn't see it during the transfer). If -the creation of the temporary file fails, rsync will try to fall back to -using bf(--delete-after) (which it cannot do if bf(--recursive) is doing an -incremental scan). -See bf(--delete) (which is implied) for more details on file-deletion. - -dit(bf(--delete-after)) Request that the file-deletions on the receiving -side be done after the transfer has completed. This is useful if you -are sending new per-directory merge files as a part of the transfer and -you want their exclusions to take effect for the delete phase of the -current transfer. It also forces rsync to use the old, non-incremental -recursion algorithm that requires rsync to scan all the files in the -transfer into memory at once (see bf(--recursive)). -See bf(--delete) (which is implied) for more details on file-deletion. - -dit(bf(--delete-excluded)) In addition to deleting the files on the -receiving side that are not on the sending side, this tells rsync to also -delete any files on the receiving side that are excluded (see bf(--exclude)). -See the FILTER RULES section for a way to make individual exclusions behave -this way on the receiver, and for a way to protect files from -bf(--delete-excluded). -See bf(--delete) (which is implied) for more details on file-deletion. - -dit(bf(--ignore-missing-args)) When rsync is first processing the explicitly -requested source files (e.g. command-line arguments or bf(--files-from) -entries), it is normally an error if the file cannot be found. This option -suppresses that error, and does not try to transfer the file. This does not -affect subsequent vanished-file errors if a file was initially found to be -present and later is no longer there. - -dit(bf(--delete-missing-args)) This option takes the behavior of (the implied) -bf(--ignore-missing-args) option a step farther: each missing arg will become -a deletion request of the corresponding destination file on the receiving side -(should it exist). If the destination file is a non-empty directory, it will -only be successfully deleted if --force or --delete are in effect. Other than -that, this option is independent of any other type of delete processing. - -The missing source files are represented by special file-list entries which -display as a "*missing" entry in the bf(--list-only) output. - -dit(bf(--ignore-errors)) Tells bf(--delete) to go ahead and delete files -even when there are I/O errors. - -dit(bf(--force)) This option tells rsync to delete a non-empty directory -when it is to be replaced by a non-directory. This is only relevant if -deletions are not active (see bf(--delete) for details). - -Note for older rsync versions: bf(--force) used to still be required when -using bf(--delete-after), and it used to be non-functional unless the -bf(--recursive) option was also enabled. - -dit(bf(--max-delete=NUM)) This tells rsync not to delete more than NUM -files or directories. If that limit is exceeded, all further deletions are -skipped through the end of the transfer. At the end, rsync outputs a warning -(including a count of the skipped deletions) and exits with an error code -of 25 (unless some more important error condition also occurred). - -Beginning with version 3.0.0, you may specify bf(--max-delete=0) to be warned -about any extraneous files in the destination without removing any of them. -Older clients interpreted this as "unlimited", so if you don't know what -version the client is, you can use the less obvious bf(--max-delete=-1) as -a backward-compatible way to specify that no deletions be allowed (though -really old versions didn't warn when the limit was exceeded). - -dit(bf(--max-size=SIZE)) This tells rsync to avoid transferring any -file that is larger than the specified SIZE. The SIZE value can be -suffixed with a string to indicate a size multiplier, and -may be a fractional value (e.g. "bf(--max-size=1.5m)"). - -This option is a transfer rule, not an exclude, so it doesn't affect the -data that goes into the file-lists, and thus it doesn't affect deletions. -It just limits the files that the receiver requests to be transferred. - -The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), -"M" (or "MiB") is a mebibyte (1024*1024), and "G" (or "GiB") is a -gibibyte (1024*1024*1024). -If you want the multiplier to be 1000 instead of 1024, use "KB", -"MB", or "GB". (Note: lower-case is also accepted for all values.) -Finally, if the suffix ends in either "+1" or "-1", the value will -be offset by one byte in the indicated direction. - -Examples: --max-size=1.5mb-1 is 1499999 bytes, and --max-size=2g+1 is -2147483649 bytes. - -Note that rsync versions prior to 3.1.0 did not allow bf(--max-size=0). - -dit(bf(--min-size=SIZE)) This tells rsync to avoid transferring any -file that is smaller than the specified SIZE, which can help in not -transferring small, junk files. -See the bf(--max-size) option for a description of SIZE and other information. - -Note that rsync versions prior to 3.1.0 did not allow bf(--min-size=0). - -dit(bf(-B, --block-size=BLOCKSIZE)) This forces the block size used in -rsync's delta-transfer algorithm to a fixed value. It is normally selected based on -the size of each file being updated. See the technical report for details. - -dit(bf(-e, --rsh=COMMAND)) This option allows you to choose an alternative -remote shell program to use for communication between the local and -remote copies of rsync. Typically, rsync is configured to use ssh by -default, but you may prefer to use rsh on a local network. - -If this option is used with bf([user@]host::module/path), then the -remote shell em(COMMAND) will be used to run an rsync daemon on the -remote host, and all data will be transmitted through that remote -shell connection, rather than through a direct socket connection to a -running rsync daemon on the remote host. See the section "USING -RSYNC-DAEMON FEATURES VIA A REMOTE-SHELL CONNECTION" above. - -Command-line arguments are permitted in COMMAND provided that COMMAND is -presented to rsync as a single argument. You must use spaces (not tabs -or other whitespace) to separate the command and args from each other, -and you can use single- and/or double-quotes to preserve spaces in an -argument (but not backslashes). Note that doubling a single-quote -inside a single-quoted string gives you a single-quote; likewise for -double-quotes (though you need to pay attention to which quotes your -shell is parsing and which quotes rsync is parsing). Some examples: - -quote( -tt( -e 'ssh -p 2234')nl() -tt( -e 'ssh -o "ProxyCommand nohup ssh firewall nc -w1 %h %p"')nl() -) - -(Note that ssh users can alternately customize site-specific connect -options in their .ssh/config file.) - -You can also choose the remote shell program using the RSYNC_RSH -environment variable, which accepts the same range of values as bf(-e). - -See also the bf(--blocking-io) option which is affected by this option. - -dit(bf(--rsync-path=PROGRAM)) Use this to specify what program is to be run -on the remote machine to start-up rsync. Often used when rsync is not in -the default remote-shell's path (e.g. --rsync-path=/usr/local/bin/rsync). -Note that PROGRAM is run with the help of a shell, so it can be any -program, script, or command sequence you'd care to run, so long as it does -not corrupt the standard-in & standard-out that rsync is using to -communicate. - -One tricky example is to set a different default directory on the remote -machine for use with the bf(--relative) option. For instance: - -quote(tt( rsync -avR --rsync-path="cd /a/b && rsync" host:c/d /e/)) - -dit(bf(-M, --remote-option=OPTION)) This option is used for more advanced -situations where you want certain effects to be limited to one side of the -transfer only. For instance, if you want to pass bf(--log-file=FILE) and -bf(--fake-super) to the remote system, specify it like this: - -quote(tt( rsync -av -M --log-file=foo -M--fake-super src/ dest/)) - -If you want to have an option affect only the local side of a transfer when -it normally affects both sides, send its negation to the remote side. Like -this: - -quote(tt( rsync -av -x -M--no-x src/ dest/)) - -Be cautious using this, as it is possible to toggle an option that will cause -rsync to have a different idea about what data to expect next over the socket, -and that will make it fail in a cryptic fashion. - -Note that it is best to use a separate bf(--remote-option) for each option you -want to pass. This makes your useage compatible with the bf(--protect-args) -option. If that option is off, any spaces in your remote options will be split -by the remote shell unless you take steps to protect them. - -When performing a local transfer, the "local" side is the sender and the -"remote" side is the receiver. - -Note some versions of the popt option-parsing library have a bug in them that -prevents you from using an adjacent arg with an equal in it next to a short -option letter (e.g. tt(-M--log-file=/tmp/foo)). If this bug affects your -version of popt, you can use the version of popt that is included with rsync. - -dit(bf(-C, --cvs-exclude)) This is a useful shorthand for excluding a -broad range of files that you often don't want to transfer between -systems. It uses a similar algorithm to CVS to determine if -a file should be ignored. - -The exclude list is initialized to exclude the following items (these -initial items are marked as perishable -- see the FILTER RULES section): - -quote(quote(tt(RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS .make.state -.nse_depinfo *~ #* .#* ,* _$* *$ *.old *.bak *.BAK *.orig *.rej .del-* -*.a *.olb *.o *.obj *.so *.exe *.Z *.elc *.ln core .svn/ .git/ .hg/ .bzr/))) - -then, files listed in a $HOME/.cvsignore are added to the list and any -files listed in the CVSIGNORE environment variable (all cvsignore names -are delimited by whitespace). - -Finally, any file is ignored if it is in the same directory as a -.cvsignore file and matches one of the patterns listed therein. Unlike -rsync's filter/exclude files, these patterns are split on whitespace. -See the bf(cvs)(1) manual for more information. - -If you're combining bf(-C) with your own bf(--filter) rules, you should -note that these CVS excludes are appended at the end of your own rules, -regardless of where the bf(-C) was placed on the command-line. This makes them -a lower priority than any rules you specified explicitly. If you want to -control where these CVS excludes get inserted into your filter rules, you -should omit the bf(-C) as a command-line option and use a combination of -bf(--filter=:C) and bf(--filter=-C) (either on your command-line or by -putting the ":C" and "-C" rules into a filter file with your other rules). -The first option turns on the per-directory scanning for the .cvsignore -file. The second option does a one-time import of the CVS excludes -mentioned above. - -dit(bf(-f, --filter=RULE)) This option allows you to add rules to selectively -exclude certain files from the list of files to be transferred. This is -most useful in combination with a recursive transfer. - -You may use as many bf(--filter) options on the command line as you like -to build up the list of files to exclude. If the filter contains whitespace, -be sure to quote it so that the shell gives the rule to rsync as a single -argument. The text below also mentions that you can use an underscore to -replace the space that separates a rule from its arg. - -See the FILTER RULES section for detailed information on this option. - -dit(bf(-F)) The bf(-F) option is a shorthand for adding two bf(--filter) rules to -your command. The first time it is used is a shorthand for this rule: - -quote(tt( --filter='dir-merge /.rsync-filter')) - -This tells rsync to look for per-directory .rsync-filter files that have -been sprinkled through the hierarchy and use their rules to filter the -files in the transfer. If bf(-F) is repeated, it is a shorthand for this -rule: - -quote(tt( --filter='exclude .rsync-filter')) - -This filters out the .rsync-filter files themselves from the transfer. - -See the FILTER RULES section for detailed information on how these options -work. - -dit(bf(--exclude=PATTERN)) This option is a simplified form of the -bf(--filter) option that defaults to an exclude rule and does not allow -the full rule-parsing syntax of normal filter rules. - -See the FILTER RULES section for detailed information on this option. - -dit(bf(--exclude-from=FILE)) This option is related to the bf(--exclude) -option, but it specifies a FILE that contains exclude patterns (one per line). -Blank lines in the file and lines starting with ';' or '#' are ignored. -If em(FILE) is bf(-), the list will be read from standard input. - -dit(bf(--include=PATTERN)) This option is a simplified form of the -bf(--filter) option that defaults to an include rule and does not allow -the full rule-parsing syntax of normal filter rules. - -See the FILTER RULES section for detailed information on this option. - -dit(bf(--include-from=FILE)) This option is related to the bf(--include) -option, but it specifies a FILE that contains include patterns (one per line). -Blank lines in the file and lines starting with ';' or '#' are ignored. -If em(FILE) is bf(-), the list will be read from standard input. - -dit(bf(--files-from=FILE)) Using this option allows you to specify the -exact list of files to transfer (as read from the specified FILE or bf(-) -for standard input). It also tweaks the default behavior of rsync to make -transferring just the specified files and directories easier: - -quote(itemization( - it() The bf(--relative) (bf(-R)) option is implied, which preserves the path - information that is specified for each item in the file (use - bf(--no-relative) or bf(--no-R) if you want to turn that off). - it() The bf(--dirs) (bf(-d)) option is implied, which will create directories - specified in the list on the destination rather than noisily skipping - them (use bf(--no-dirs) or bf(--no-d) if you want to turn that off). - it() The bf(--archive) (bf(-a)) option's behavior does not imply bf(--recursive) - (bf(-r)), so specify it explicitly, if you want it. - it() These side-effects change the default state of rsync, so the position - of the bf(--files-from) option on the command-line has no bearing on how - other options are parsed (e.g. bf(-a) works the same before or after - bf(--files-from), as does bf(--no-R) and all other options). -)) - -The filenames that are read from the FILE are all relative to the -source dir -- any leading slashes are removed and no ".." references are -allowed to go higher than the source dir. For example, take this -command: - -quote(tt( rsync -a --files-from=/tmp/foo /usr remote:/backup)) - -If /tmp/foo contains the string "bin" (or even "/bin"), the /usr/bin -directory will be created as /backup/bin on the remote host. If it -contains "bin/" (note the trailing slash), the immediate contents of -the directory would also be sent (without needing to be explicitly -mentioned in the file -- this began in version 2.6.4). In both cases, -if the bf(-r) option was enabled, that dir's entire hierarchy would -also be transferred (keep in mind that bf(-r) needs to be specified -explicitly with bf(--files-from), since it is not implied by bf(-a)). -Also note -that the effect of the (enabled by default) bf(--relative) option is to -duplicate only the path info that is read from the file -- it does not -force the duplication of the source-spec path (/usr in this case). - -In addition, the bf(--files-from) file can be read from the remote host -instead of the local host if you specify a "host:" in front of the file -(the host must match one end of the transfer). As a short-cut, you can -specify just a prefix of ":" to mean "use the remote end of the -transfer". For example: - -quote(tt( rsync -a --files-from=:/path/file-list src:/ /tmp/copy)) - -This would copy all the files specified in the /path/file-list file that -was located on the remote "src" host. - -If the bf(--iconv) and bf(--protect-args) options are specified and the -bf(--files-from) filenames are being sent from one host to another, the -filenames will be translated from the sending host's charset to the -receiving host's charset. - -NOTE: sorting the list of files in the --files-from input helps rsync to be -more efficient, as it will avoid re-visiting the path elements that are shared -between adjacent entries. If the input is not sorted, some path elements -(implied directories) may end up being scanned multiple times, and rsync will -eventually unduplicate them after they get turned into file-list elements. - -) -description( - -dit(bf(-0, --from0)) This tells rsync that the rules/filenames it reads from a -file are terminated by a null ('\0') character, not a NL, CR, or CR+LF. -This affects bf(--exclude-from), bf(--include-from), bf(--files-from), and any -merged files specified in a bf(--filter) rule. -It does not affect bf(--cvs-exclude) (since all names read from a .cvsignore -file are split on whitespace). - -dit(bf(-s, --protect-args)) This option sends all filenames and most options to -the remote rsync without allowing the remote shell to interpret them. This -means that spaces are not split in names, and any non-wildcard special -characters are not translated (such as ~, $, ;, &, etc.). Wildcards are -expanded on the remote host by rsync (instead of the shell doing it). - -If you use this option with bf(--iconv), the args related to the remote -side will also be translated -from the local to the remote character-set. The translation happens before -wild-cards are expanded. See also the bf(--files-from) option. - -You may also control this option via the RSYNC_PROTECT_ARGS environment -variable. If this variable has a non-zero value, this option will be enabled -by default, otherwise it will be disabled by default. Either state is -overridden by a manually specified positive or negative version of this option -(note that bf(--no-s) and bf(--no-protect-args) are the negative versions). -Since this option was first introduced in 3.0.0, you'll need to make sure it's -disabled if you ever need to interact with a remote rsync that is older than -that. - -Rsync can also be configured (at build time) to have this option enabled by -default (with is overridden by both the environment and the command-line). -This option will eventually become a new default setting at some -as-yet-undetermined point in the future. - -dit(bf(-T, --temp-dir=DIR)) This option instructs rsync to use DIR as a -scratch directory when creating temporary copies of the files transferred -on the receiving side. The default behavior is to create each temporary -file in the same directory as the associated destination file. -Beginning with rsync 3.1.1, the temp-file names inside the specified DIR will -not be prefixed with an extra dot (though they will still have a random suffix -added). - -This option is most often used when the receiving disk partition does not -have enough free space to hold a copy of the largest file in the transfer. -In this case (i.e. when the scratch directory is on a different disk -partition), rsync will not be able to rename each received temporary file -over the top of the associated destination file, but instead must copy it -into place. Rsync does this by copying the file over the top of the -destination file, which means that the destination file will contain -truncated data during this copy. If this were not done this way (even if -the destination file were first removed, the data locally copied to a -temporary file in the destination directory, and then renamed into place) -it would be possible for the old file to continue taking up disk space (if -someone had it open), and thus there might not be enough room to fit the -new version on the disk at the same time. - -If you are using this option for reasons other than a shortage of disk -space, you may wish to combine it with the bf(--delay-updates) option, -which will ensure that all copied files get put into subdirectories in the -destination hierarchy, awaiting the end of the transfer. If you don't -have enough room to duplicate all the arriving files on the destination -partition, another way to tell rsync that you aren't overly concerned -about disk space is to use the bf(--partial-dir) option with a relative -path; because this tells rsync that it is OK to stash off a copy of a -single file in a subdir in the destination hierarchy, rsync will use the -partial-dir as a staging area to bring over the copied file, and then -rename it into place from there. (Specifying a bf(--partial-dir) with -an absolute path does not have this side-effect.) - -dit(bf(-y, --fuzzy)) This option tells rsync that it should look for a -basis file for any destination file that is missing. The current algorithm -looks in the same directory as the destination file for either a file that -has an identical size and modified-time, or a similarly-named file. If -found, rsync uses the fuzzy basis file to try to speed up the transfer. - -If the option is repeated, the fuzzy scan will also be done in any matching -alternate destination directories that are specified via bf(--compare-dest), -bf(--copy-dest), or bf(--link-dest). - -Note that the use of the bf(--delete) option might get rid of any potential -fuzzy-match files, so either use bf(--delete-after) or specify some -filename exclusions if you need to prevent this. - -dit(bf(--compare-dest=DIR)) This option instructs rsync to use em(DIR) on -the destination machine as an additional hierarchy to compare destination -files against doing transfers (if the files are missing in the destination -directory). If a file is found in em(DIR) that is identical to the -sender's file, the file will NOT be transferred to the destination -directory. This is useful for creating a sparse backup of just files that -have changed from an earlier backup. -This option is typically used to copy into an empty (or newly created) -directory. - -Beginning in version 2.6.4, multiple bf(--compare-dest) directories may be -provided, which will cause rsync to search the list in the order specified -for an exact match. -If a match is found that differs only in attributes, a local copy is made -and the attributes updated. -If a match is not found, a basis file from one of the em(DIR)s will be -selected to try to speed up the transfer. - -If em(DIR) is a relative path, it is relative to the destination directory. -See also bf(--copy-dest) and bf(--link-dest). - -NOTE: beginning with version 3.1.0, rsync will remove a file from a non-empty -destination hierarchy if an exact match is found in one of the compare-dest -hierarchies (making the end result more closely match a fresh copy). - -dit(bf(--copy-dest=DIR)) This option behaves like bf(--compare-dest), but -rsync will also copy unchanged files found in em(DIR) to the destination -directory using a local copy. -This is useful for doing transfers to a new destination while leaving -existing files intact, and then doing a flash-cutover when all files have -been successfully transferred. - -Multiple bf(--copy-dest) directories may be provided, which will cause -rsync to search the list in the order specified for an unchanged file. -If a match is not found, a basis file from one of the em(DIR)s will be -selected to try to speed up the transfer. - -If em(DIR) is a relative path, it is relative to the destination directory. -See also bf(--compare-dest) and bf(--link-dest). - -dit(bf(--link-dest=DIR)) This option behaves like bf(--copy-dest), but -unchanged files are hard linked from em(DIR) to the destination directory. -The files must be identical in all preserved attributes (e.g. permissions, -possibly ownership) in order for the files to be linked together. -An example: - -quote(tt( rsync -av --link-dest=$PWD/prior_dir host:src_dir/ new_dir/)) - -If file's aren't linking, double-check their attributes. Also check if some -attributes are getting forced outside of rsync's control, such a mount option -that squishes root to a single user, or mounts a removable drive with generic -ownership (such as OS X's "Ignore ownership on this volume" option). - -Beginning in version 2.6.4, multiple bf(--link-dest) directories may be -provided, which will cause rsync to search the list in the order specified -for an exact match (there is a limit of 20 such directories). -If a match is found that differs only in attributes, a local copy is made -and the attributes updated. -If a match is not found, a basis file from one of the em(DIR)s will be -selected to try to speed up the transfer. - -This option works best when copying into an empty destination hierarchy, as -existing files may get their attributes tweaked, and that can affect alternate -destination files via hard-links. Also, itemizing of changes can get a bit -muddled. Note that prior to version 3.1.0, an alternate-directory exact match -would never be found (nor linked into the destination) when a destination file -already exists. - -Note that if you combine this option with bf(--ignore-times), rsync will not -link any files together because it only links identical files together as a -substitute for transferring the file, never as an additional check after the -file is updated. - -If em(DIR) is a relative path, it is relative to the destination directory. -See also bf(--compare-dest) and bf(--copy-dest). - -Note that rsync versions prior to 2.6.1 had a bug that could prevent -bf(--link-dest) from working properly for a non-super-user when bf(-o) was -specified (or implied by bf(-a)). You can work-around this bug by avoiding -the bf(-o) option when sending to an old rsync. - -dit(bf(-z, --compress)) With this option, rsync compresses the file data -as it is sent to the destination machine, which reduces the amount of data -being transmitted -- something that is useful over a slow connection. - -Note that this option typically achieves better compression ratios than can -be achieved by using a compressing remote shell or a compressing transport -because it takes advantage of the implicit information in the matching data -blocks that are not explicitly sent over the connection. This matching-data -compression comes at a cost of CPU, though, and can be disabled by repeating -the bf(-z) option, but only if both sides are at least version 3.1.1. - -Note that if your version of rsync was compiled with an external zlib (instead -of the zlib that comes packaged with rsync) then it will not support the -old-style compression, only the new-style (repeated-option) compression. In -the future this new-style compression will likely become the default. - -The client rsync requests new-style compression on the server via the -bf(--new-compress) option, so if you see that option rejected it means that -the server is not new enough to support bf(-zz). Rsync also accepts the -bf(--old-compress) option for a future time when new-style compression -becomes the default. - -See the bf(--skip-compress) option for the default list of file suffixes -that will not be compressed. - -dit(bf(--compress-level=NUM)) Explicitly set the compression level to use -(see bf(--compress)) instead of letting it default. If NUM is non-zero, -the bf(--compress) option is implied. - -dit(bf(--skip-compress=LIST)) Override the list of file suffixes that will -not be compressed. The bf(LIST) should be one or more file suffixes -(without the dot) separated by slashes (/). - -You may specify an empty string to indicate that no file should be skipped. - -Simple character-class matching is supported: each must consist of a list -of letters inside the square brackets (e.g. no special classes, such as -"[:alpha:]", are supported, and '-' has no special meaning). - -The characters asterisk (*) and question-mark (?) have no special meaning. - -Here's an example that specifies 6 suffixes to skip (since 1 of the 5 rules -matches 2 suffixes): - -verb( --skip-compress=gz/jpg/mp[34]/7z/bz2) - -The default list of suffixes that will not be compressed is this (in this -version of rsync): - -bf(7z) -bf(ace) -bf(avi) -bf(bz2) -bf(deb) -bf(gpg) -bf(gz) -bf(iso) -bf(jpeg) -bf(jpg) -bf(lz) -bf(lzma) -bf(lzo) -bf(mov) -bf(mp3) -bf(mp4) -bf(ogg) -bf(png) -bf(rar) -bf(rpm) -bf(rzip) -bf(tbz) -bf(tgz) -bf(tlz) -bf(txz) -bf(xz) -bf(z) -bf(zip) - -This list will be replaced by your bf(--skip-compress) list in all but one -situation: a copy from a daemon rsync will add your skipped suffixes to -its list of non-compressing files (and its list may be configured to a -different default). - -dit(bf(--numeric-ids)) With this option rsync will transfer numeric group -and user IDs rather than using user and group names and mapping them -at both ends. - -By default rsync will use the username and groupname to determine -what ownership to give files. The special uid 0 and the special group -0 are never mapped via user/group names even if the bf(--numeric-ids) -option is not specified. - -If a user or group has no name on the source system or it has no match -on the destination system, then the numeric ID -from the source system is used instead. See also the comments on the -"use chroot" setting in the rsyncd.conf manpage for information on how -the chroot setting affects rsync's ability to look up the names of the -users and groups and what you can do about it. - -dit(bf(--usermap=STRING, --groupmap=STRING)) These options allow you to -specify users and groups that should be mapped to other values by the -receiving side. The bf(STRING) is one or more bf(FROM):bf(TO) pairs of -values separated by commas. Any matching bf(FROM) value from the sender is -replaced with a bf(TO) value from the receiver. You may specify usernames -or user IDs for the bf(FROM) and bf(TO) values, and the bf(FROM) value may -also be a wild-card string, which will be matched against the sender's -names (wild-cards do NOT match against ID numbers, though see below for -why a '*' matches everything). You may instead specify a range of ID -numbers via an inclusive range: LOW-HIGH. For example: - -verb( --usermap=0-99:nobody,wayne:admin,*:normal --groupmap=usr:1,1:usr) - -The first match in the list is the one that is used. You should specify -all your user mappings using a single bf(--usermap) option, and/or all -your group mappings using a single bf(--groupmap) option. - -Note that the sender's name for the 0 user and group are not transmitted -to the receiver, so you should either match these values using a 0, or use -the names in effect on the receiving side (typically "root"). All other -bf(FROM) names match those in use on the sending side. All bf(TO) names -match those in use on the receiving side. - -Any IDs that do not have a name on the sending side are treated as having an -empty name for the purpose of matching. This allows them to be matched via -a "*" or using an empty name. For instance: - -verb( --usermap=:nobody --groupmap=*:nobody) - -When the bf(--numeric-ids) option is used, the sender does not send any -names, so all the IDs are treated as having an empty name. This means that -you will need to specify numeric bf(FROM) values if you want to map these -nameless IDs to different values. - -For the bf(--usermap) option to have any effect, the bf(-o) (bf(--owner)) -option must be used (or implied), and the receiver will need to be running -as a super-user (see also the bf(--fake-super) option). For the bf(--groupmap) -option to have any effect, the bf(-g) (bf(--groups)) option must be used -(or implied), and the receiver will need to have permissions to set that -group. - -dit(bf(--chown=USER:GROUP)) This option forces all files to be owned by USER -with group GROUP. This is a simpler interface than using bf(--usermap) and -bf(--groupmap) directly, but it is implemented using those options internally, -so you cannot mix them. If either the USER or GROUP is empty, no mapping for -the omitted user/group will occur. If GROUP is empty, the trailing colon may -be omitted, but if USER is empty, a leading colon must be supplied. - -If you specify "--chown=foo:bar, this is exactly the same as specifying -"--usermap=*:foo --groupmap=*:bar", only easier. - -dit(bf(--timeout=TIMEOUT)) This option allows you to set a maximum I/O -timeout in seconds. If no data is transferred for the specified time -then rsync will exit. The default is 0, which means no timeout. - -dit(bf(--contimeout)) This option allows you to set the amount of time -that rsync will wait for its connection to an rsync daemon to succeed. -If the timeout is reached, rsync exits with an error. - -dit(bf(--address)) By default rsync will bind to the wildcard address when -connecting to an rsync daemon. The bf(--address) option allows you to -specify a specific IP address (or hostname) to bind to. See also this -option in the bf(--daemon) mode section. - -dit(bf(--port=PORT)) This specifies an alternate TCP port number to use -rather than the default of 873. This is only needed if you are using the -double-colon (::) syntax to connect with an rsync daemon (since the URL -syntax has a way to specify the port as a part of the URL). See also this -option in the bf(--daemon) mode section. - -dit(bf(--sockopts)) This option can provide endless fun for people -who like to tune their systems to the utmost degree. You can set all -sorts of socket options which may make transfers faster (or -slower!). Read the man page for the code(setsockopt()) system call for -details on some of the options you may be able to set. By default no -special socket options are set. This only affects direct socket -connections to a remote rsync daemon. This option also exists in the -bf(--daemon) mode section. - -dit(bf(--blocking-io)) This tells rsync to use blocking I/O when launching -a remote shell transport. If the remote shell is either rsh or remsh, -rsync defaults to using -blocking I/O, otherwise it defaults to using non-blocking I/O. (Note that -ssh prefers non-blocking I/O.) - -dit(bf(--outbuf=MODE)) This sets the output buffering mode. The mode can be -None (aka Unbuffered), Line, or Block (aka Full). You may specify as little -as a single letter for the mode, and use upper or lower case. - -The main use of this option is to change Full buffering to Line buffering -when rsync's output is going to a file or pipe. - -dit(bf(-i, --itemize-changes)) Requests a simple itemized list of the -changes that are being made to each file, including attribute changes. -This is exactly the same as specifying bf(--out-format='%i %n%L'). -If you repeat the option, unchanged files will also be output, but only -if the receiving rsync is at least version 2.6.7 (you can use bf(-vv) -with older versions of rsync, but that also turns on the output of other -verbose messages). - -The "%i" escape has a cryptic output that is 11 letters long. The general -format is like the string bf(YXcstpoguax), where bf(Y) is replaced by the -type of update being done, bf(X) is replaced by the file-type, and the -other letters represent attributes that may be output if they are being -modified. - -The update types that replace the bf(Y) are as follows: - -quote(itemization( - it() A bf(<) means that a file is being transferred to the remote host - (sent). - it() A bf(>) means that a file is being transferred to the local host - (received). - it() A bf(c) means that a local change/creation is occurring for the item - (such as the creation of a directory or the changing of a symlink, etc.). - it() A bf(h) means that the item is a hard link to another item (requires - bf(--hard-links)). - it() A bf(.) means that the item is not being updated (though it might - have attributes that are being modified). - it() A bf(*) means that the rest of the itemized-output area contains - a message (e.g. "deleting"). -)) - -The file-types that replace the bf(X) are: bf(f) for a file, a bf(d) for a -directory, an bf(L) for a symlink, a bf(D) for a device, and a bf(S) for a -special file (e.g. named sockets and fifos). - -The other letters in the string above are the actual letters that -will be output if the associated attribute for the item is being updated or -a "." for no change. Three exceptions to this are: (1) a newly created -item replaces each letter with a "+", (2) an identical item replaces the -dots with spaces, and (3) an unknown attribute replaces each letter with -a "?" (this can happen when talking to an older rsync). - -The attribute that is associated with each letter is as follows: - -quote(itemization( - it() A bf(c) means either that a regular file has a different checksum - (requires bf(--checksum)) or that a symlink, device, or special file has - a changed value. - Note that if you are sending files to an rsync prior to 3.0.1, this - change flag will be present only for checksum-differing regular files. - it() A bf(s) means the size of a regular file is different and will be updated - by the file transfer. - it() A bf(t) means the modification time is different and is being updated - to the sender's value (requires bf(--times)). An alternate value of bf(T) - means that the modification time will be set to the transfer time, which happens - when a file/symlink/device is updated without bf(--times) and when a - symlink is changed and the receiver can't set its time. - (Note: when using an rsync 3.0.0 client, you might see the bf(s) flag combined - with bf(t) instead of the proper bf(T) flag for this time-setting failure.) - it() A bf(p) means the permissions are different and are being updated to - the sender's value (requires bf(--perms)). - it() An bf(o) means the owner is different and is being updated to the - sender's value (requires bf(--owner) and super-user privileges). - it() A bf(g) means the group is different and is being updated to the - sender's value (requires bf(--group) and the authority to set the group). - it() The bf(u) slot is reserved for future use. - it() The bf(a) means that the ACL information changed. - it() The bf(x) means that the extended attribute information changed. -)) - -One other output is possible: when deleting files, the "%i" will output -the string "*deleting" for each item that is being removed (assuming that -you are talking to a recent enough rsync that it logs deletions instead of -outputting them as a verbose message). - -dit(bf(--out-format=FORMAT)) This allows you to specify exactly what the -rsync client outputs to the user on a per-update basis. The format is a -text string containing embedded single-character escape sequences prefixed -with a percent (%) character. A default format of "%n%L" is assumed if -either bf(--info=name) or bf(-v) is specified (this tells you just the name -of the file and, if the item is a link, where it points). For a full list -of the possible escape characters, see the "log format" setting in the -rsyncd.conf manpage. - -Specifying the bf(--out-format) option implies the bf(--info=name) option, -which will mention each file, dir, etc. that gets updated in a significant -way (a transferred file, a recreated symlink/device, or a touched -directory). In addition, if the itemize-changes escape (%i) is included in -the string (e.g. if the bf(--itemize-changes) option was used), the logging -of names increases to mention any item that is changed in any way (as long -as the receiving side is at least 2.6.4). See the bf(--itemize-changes) -option for a description of the output of "%i". - -Rsync will output the out-format string prior to a file's transfer unless -one of the transfer-statistic escapes is requested, in which case the -logging is done at the end of the file's transfer. When this late logging -is in effect and bf(--progress) is also specified, rsync will also output -the name of the file being transferred prior to its progress information -(followed, of course, by the out-format output). - -dit(bf(--log-file=FILE)) This option causes rsync to log what it is doing -to a file. This is similar to the logging that a daemon does, but can be -requested for the client side and/or the server side of a non-daemon -transfer. If specified as a client option, transfer logging will be -enabled with a default format of "%i %n%L". See the bf(--log-file-format) -option if you wish to override this. - -Here's a example command that requests the remote side to log what is -happening: - -verb( rsync -av --remote-option=--log-file=/tmp/rlog src/ dest/) - -This is very useful if you need to debug why a connection is closing -unexpectedly. - -dit(bf(--log-file-format=FORMAT)) This allows you to specify exactly what -per-update logging is put into the file specified by the bf(--log-file) option -(which must also be specified for this option to have any effect). If you -specify an empty string, updated files will not be mentioned in the log file. -For a list of the possible escape characters, see the "log format" setting -in the rsyncd.conf manpage. - -The default FORMAT used if bf(--log-file) is specified and this option is not -is '%i %n%L'. - -dit(bf(--stats)) This tells rsync to print a verbose set of statistics -on the file transfer, allowing you to tell how effective rsync's delta-transfer -algorithm is for your data. This option is equivalent to bf(--info=stats2) -if combined with 0 or 1 bf(-v) options, or bf(--info=stats3) if combined -with 2 or more bf(-v) options. - -The current statistics are as follows: quote(itemization( - it() bf(Number of files) is the count of all "files" (in the generic - sense), which includes directories, symlinks, etc. The total count will - be followed by a list of counts by filetype (if the total is non-zero). - For example: "(reg: 5, dir: 3, link: 2, dev: 1, special: 1)" lists the - totals for regular files, directories, symlinks, devices, and special - files. If any of value is 0, it is completely omitted from the list. - it() bf(Number of created files) is the count of how many "files" (generic - sense) were created (as opposed to updated). The total count will be - followed by a list of counts by filetype (if the total is non-zero). - it() bf(Number of deleted files) is the count of how many "files" (generic - sense) were created (as opposed to updated). The total count will be - followed by a list of counts by filetype (if the total is non-zero). - Note that this line is only output if deletions are in effect, and only - if protocol 31 is being used (the default for rsync 3.1.x). - it() bf(Number of regular files transferred) is the count of normal files - that were updated via rsync's delta-transfer algorithm, which does not - include dirs, symlinks, etc. Note that rsync 3.1.0 added the word - "regular" into this heading. - it() bf(Total file size) is the total sum of all file sizes in the transfer. - This does not count any size for directories or special files, but does - include the size of symlinks. - it() bf(Total transferred file size) is the total sum of all files sizes - for just the transferred files. - it() bf(Literal data) is how much unmatched file-update data we had to - send to the receiver for it to recreate the updated files. - it() bf(Matched data) is how much data the receiver got locally when - recreating the updated files. - it() bf(File list size) is how big the file-list data was when the sender - sent it to the receiver. This is smaller than the in-memory size for the - file list due to some compressing of duplicated data when rsync sends the - list. - it() bf(File list generation time) is the number of seconds that the - sender spent creating the file list. This requires a modern rsync on the - sending side for this to be present. - it() bf(File list transfer time) is the number of seconds that the sender - spent sending the file list to the receiver. - it() bf(Total bytes sent) is the count of all the bytes that rsync sent - from the client side to the server side. - it() bf(Total bytes received) is the count of all non-message bytes that - rsync received by the client side from the server side. "Non-message" - bytes means that we don't count the bytes for a verbose message that the - server sent to us, which makes the stats more consistent. -)) - -dit(bf(-8, --8-bit-output)) This tells rsync to leave all high-bit characters -unescaped in the output instead of trying to test them to see if they're -valid in the current locale and escaping the invalid ones. All control -characters (but never tabs) are always escaped, regardless of this option's -setting. - -The escape idiom that started in 2.6.7 is to output a literal backslash (\) -and a hash (#), followed by exactly 3 octal digits. For example, a newline -would output as "\#012". A literal backslash that is in a filename is not -escaped unless it is followed by a hash and 3 digits (0-9). - -) -description( - -dit(bf(-h, --human-readable)) Output numbers in a more human-readable format. -There are 3 possible levels: (1) output numbers with a separator between each -set of 3 digits (either a comma or a period, depending on if the decimal point -is represented by a period or a comma); (2) output numbers in units of 1000 -(with a character suffix for larger units -- see below); (3) output numbers in -units of 1024. - -The default is human-readable level 1. Each bf(-h) option increases the level -by one. You can take the level down to 0 (to output numbers as pure digits) by -specifing the bf(--no-human-readable) (bf(--no-h)) option. - -The unit letters that are appended in levels 2 and 3 are: K (kilo), M (mega), -G (giga), or T (tera). For example, a 1234567-byte file would output as 1.23M -in level-2 (assuming that a period is your local decimal point). - -Backward compatibility note: versions of rsync prior to 3.1.0 do not support -human-readable level 1, and they default to level 0. Thus, specifying one or -two bf(-h) options will behave in a comparable manner in old and new versions -as long as you didn't specify a bf(--no-h) option prior to one or more bf(-h) -options. See the bf(--list-only) option for one difference. - -dit(bf(--partial)) By default, rsync will delete any partially -transferred file if the transfer is interrupted. In some circumstances -it is more desirable to keep partially transferred files. Using the -bf(--partial) option tells rsync to keep the partial file which should -make a subsequent transfer of the rest of the file much faster. - -dit(bf(--partial-dir=DIR)) A better way to keep partial files than the -bf(--partial) option is to specify a em(DIR) that will be used to hold the -partial data (instead of writing it out to the destination file). -On the next transfer, rsync will use a file found in this -dir as data to speed up the resumption of the transfer and then delete it -after it has served its purpose. - -Note that if bf(--whole-file) is specified (or implied), any partial-dir -file that is found for a file that is being updated will simply be removed -(since -rsync is sending files without using rsync's delta-transfer algorithm). - -Rsync will create the em(DIR) if it is missing (just the last dir -- not -the whole path). This makes it easy to use a relative path (such as -"bf(--partial-dir=.rsync-partial)") to have rsync create the -partial-directory in the destination file's directory when needed, and then -remove it again when the partial file is deleted. - -If the partial-dir value is not an absolute path, rsync will add an exclude -rule at the end of all your existing excludes. This will prevent the -sending of any partial-dir files that may exist on the sending side, and -will also prevent the untimely deletion of partial-dir items on the -receiving side. An example: the above bf(--partial-dir) option would add -the equivalent of "bf(-f '-p .rsync-partial/')" at the end of any other -filter rules. - -If you are supplying your own exclude rules, you may need to add your own -exclude/hide/protect rule for the partial-dir because (1) the auto-added -rule may be ineffective at the end of your other rules, or (2) you may wish -to override rsync's exclude choice. For instance, if you want to make -rsync clean-up any left-over partial-dirs that may be lying around, you -should specify bf(--delete-after) and add a "risk" filter rule, e.g. -bf(-f 'R .rsync-partial/'). (Avoid using bf(--delete-before) or -bf(--delete-during) unless you don't need rsync to use any of the -left-over partial-dir data during the current run.) - -IMPORTANT: the bf(--partial-dir) should not be writable by other users or it -is a security risk. E.g. AVOID "/tmp". - -You can also set the partial-dir value the RSYNC_PARTIAL_DIR environment -variable. Setting this in the environment does not force bf(--partial) to be -enabled, but rather it affects where partial files go when bf(--partial) is -specified. For instance, instead of using bf(--partial-dir=.rsync-tmp) -along with bf(--progress), you could set RSYNC_PARTIAL_DIR=.rsync-tmp in your -environment and then just use the bf(-P) option to turn on the use of the -.rsync-tmp dir for partial transfers. The only times that the bf(--partial) -option does not look for this environment value are (1) when bf(--inplace) was -specified (since bf(--inplace) conflicts with bf(--partial-dir)), and (2) when -bf(--delay-updates) was specified (see below). - -For the purposes of the daemon-config's "refuse options" setting, -bf(--partial-dir) does em(not) imply bf(--partial). This is so that a -refusal of the bf(--partial) option can be used to disallow the overwriting -of destination files with a partial transfer, while still allowing the -safer idiom provided by bf(--partial-dir). - -dit(bf(--delay-updates)) This option puts the temporary file from each -updated file into a holding directory until the end of the -transfer, at which time all the files are renamed into place in rapid -succession. This attempts to make the updating of the files a little more -atomic. By default the files are placed into a directory named ".~tmp~" in -each file's destination directory, but if you've specified the -bf(--partial-dir) option, that directory will be used instead. See the -comments in the bf(--partial-dir) section for a discussion of how this -".~tmp~" dir will be excluded from the transfer, and what you can do if -you want rsync to cleanup old ".~tmp~" dirs that might be lying around. -Conflicts with bf(--inplace) and bf(--append). - -This option uses more memory on the receiving side (one bit per file -transferred) and also requires enough free disk space on the receiving -side to hold an additional copy of all the updated files. Note also that -you should not use an absolute path to bf(--partial-dir) unless (1) -there is no -chance of any of the files in the transfer having the same name (since all -the updated files will be put into a single directory if the path is -absolute) -and (2) there are no mount points in the hierarchy (since the -delayed updates will fail if they can't be renamed into place). - -See also the "atomic-rsync" perl script in the "support" subdir for an -update algorithm that is even more atomic (it uses bf(--link-dest) and a -parallel hierarchy of files). - -dit(bf(-m, --prune-empty-dirs)) This option tells the receiving rsync to get -rid of empty directories from the file-list, including nested directories -that have no non-directory children. This is useful for avoiding the -creation of a bunch of useless directories when the sending rsync is -recursively scanning a hierarchy of files using include/exclude/filter -rules. - -Note that the use of transfer rules, such as the bf(--min-size) option, does -not affect what goes into the file list, and thus does not leave directories -empty, even if none of the files in a directory match the transfer rule. - -Because the file-list is actually being pruned, this option also affects -what directories get deleted when a delete is active. However, keep in -mind that excluded files and directories can prevent existing items from -being deleted due to an exclude both hiding source files and protecting -destination files. See the perishable filter-rule option for how to avoid -this. - -You can prevent the pruning of certain empty directories from the file-list -by using a global "protect" filter. For instance, this option would ensure -that the directory "emptydir" was kept in the file-list: - -quote( --filter 'protect emptydir/') - -Here's an example that copies all .pdf files in a hierarchy, only creating -the necessary destination directories to hold the .pdf files, and ensures -that any superfluous files and directories in the destination are removed -(note the hide filter of non-directories being used instead of an exclude): - -quote( rsync -avm --del --include='*.pdf' -f 'hide,! */' src/ dest) - -If you didn't want to remove superfluous destination files, the more -time-honored options of "bf(--include='*/' --exclude='*')" would work fine -in place of the hide-filter (if that is more natural to you). - -dit(bf(--progress)) This option tells rsync to print information -showing the progress of the transfer. This gives a bored user -something to watch. -With a modern rsync this is the same as specifying -bf(--info=flist2,name,progress), but any user-supplied settings for those -info flags takes precedence (e.g. "--info=flist0 --progress"). - -While rsync is transferring a regular file, it updates a progress line that -looks like this: - -verb( 782448 63% 110.64kB/s 0:00:04) - -In this example, the receiver has reconstructed 782448 bytes or 63% of the -sender's file, which is being reconstructed at a rate of 110.64 kilobytes -per second, and the transfer will finish in 4 seconds if the current rate -is maintained until the end. - -These statistics can be misleading if rsync's delta-transfer algorithm is -in use. For example, if the sender's file consists of the basis file -followed by additional data, the reported rate will probably drop -dramatically when the receiver gets to the literal data, and the transfer -will probably take much longer to finish than the receiver estimated as it -was finishing the matched part of the file. - -When the file transfer finishes, rsync replaces the progress line with a -summary line that looks like this: - -verb( 1,238,099 100% 146.38kB/s 0:00:08 (xfr#5, to-chk=169/396)) - -In this example, the file was 1,238,099 bytes long in total, the average rate -of transfer for the whole file was 146.38 kilobytes per second over the 8 -seconds that it took to complete, it was the 5th transfer of a regular file -during the current rsync session, and there are 169 more files for the -receiver to check (to see if they are up-to-date or not) remaining out of -the 396 total files in the file-list. - -In an incremental recursion scan, rsync won't know the total number of files -in the file-list until it reaches the ends of the scan, but since it starts to -transfer files during the scan, it will display a line with the text "ir-chk" -(for incremental recursion check) instead of "to-chk" until the point that it -knows the full size of the list, at which point it will switch to using -"to-chk". Thus, seeing "ir-chk" lets you know that the total count of files -in the file list is still going to increase (and each time it does, the count -of files left to check will increase by the number of the files added to the -list). - -dit(bf(-P)) The bf(-P) option is equivalent to bf(--partial) bf(--progress). Its -purpose is to make it much easier to specify these two options for a long -transfer that may be interrupted. - -There is also a bf(--info=progress2) option that outputs statistics based -on the whole transfer, rather than individual files. Use this flag without -outputting a filename (e.g. avoid bf(-v) or specify bf(--info=name0)) if you -want to see how the transfer is doing without scrolling the screen with a -lot of names. (You don't need to specify the bf(--progress) option in -order to use bf(--info=progress2).) - -dit(bf(--password-file=FILE)) This option allows you to provide a password for -accessing an rsync daemon via a file or via standard input if bf(FILE) is -bf(-). The file should contain just the password on the first line (all other -lines are ignored). Rsync will exit with an error if bf(FILE) is world -readable or if a root-run rsync command finds a non-root-owned file. - -This option does not supply a password to a remote shell transport such as -ssh; to learn how to do that, consult the remote shell's documentation. -When accessing an rsync daemon using a remote shell as the transport, this -option only comes into effect after the remote shell finishes its -authentication (i.e. if you have also specified a password in the daemon's -config file). - -dit(bf(--list-only)) This option will cause the source files to be listed -instead of transferred. This option is inferred if there is a single source -arg and no destination specified, so its main uses are: (1) to turn a copy -command that includes a -destination arg into a file-listing command, or (2) to be able to specify -more than one source arg (note: be sure to include the destination). -Caution: keep in mind that a source arg with a wild-card is expanded by the -shell into multiple args, so it is never safe to try to list such an arg -without using this option. For example: - -verb( rsync -av --list-only foo* dest/) - -Starting with rsync 3.1.0, the sizes output by bf(--list-only) are affected -by the bf(--human-readable) option. By default they will contain digit -separators, but higher levels of readability will output the sizes with -unit suffixes. Note also that the column width for the size output has -increased from 11 to 14 characters for all human-readable levels. Use -bf(--no-h) if you want just digits in the sizes, and the old column width -of 11 characters. - -Compatibility note: when requesting a remote listing of files from an rsync -that is version 2.6.3 or older, you may encounter an error if you ask for a -non-recursive listing. This is because a file listing implies the bf(--dirs) -option w/o bf(--recursive), and older rsyncs don't have that option. To -avoid this problem, either specify the bf(--no-dirs) option (if you don't -need to expand a directory's content), or turn on recursion and exclude -the content of subdirectories: bf(-r --exclude='/*/*'). - -dit(bf(--bwlimit=RATE)) This option allows you to specify the maximum transfer -rate for the data sent over the socket, specified in units per second. The -RATE value can be suffixed with a string to indicate a size multiplier, and may -be a fractional value (e.g. "bf(--bwlimit=1.5m)"). If no suffix is specified, -the value will be assumed to be in units of 1024 bytes (as if "K" or "KiB" had -been appended). See the bf(--max-size) option for a description of all the -available suffixes. A value of zero specifies no limit. - -For backward-compatibility reasons, the rate limit will be rounded to the -nearest KiB unit, so no rate smaller than 1024 bytes per second is possible. - -Rsync writes data over the socket in blocks, and this option both limits the -size of the blocks that rsync writes, and tries to keep the average transfer -rate at the requested limit. Some "burstiness" may be seen where rsync writes -out a block of data and then sleeps to bring the average rate into compliance. - -Due to the internal buffering of data, the bf(--progress) option may not be an -accurate reflection on how fast the data is being sent. This is because some -files can show up as being rapidly sent when the data is quickly buffered, -while other can show up as very slow when the flushing of the output buffer -occurs. This may be fixed in a future version. - -dit(bf(--write-batch=FILE)) Record a file that can later be applied to -another identical destination with bf(--read-batch). See the "BATCH MODE" -section for details, and also the bf(--only-write-batch) option. - -dit(bf(--only-write-batch=FILE)) Works like bf(--write-batch), except that -no updates are made on the destination system when creating the batch. -This lets you transport the changes to the destination system via some -other means and then apply the changes via bf(--read-batch). - -Note that you can feel free to write the batch directly to some portable -media: if this media fills to capacity before the end of the transfer, you -can just apply that partial transfer to the destination and repeat the -whole process to get the rest of the changes (as long as you don't mind a -partially updated destination system while the multi-update cycle is -happening). - -Also note that you only save bandwidth when pushing changes to a remote -system because this allows the batched data to be diverted from the sender -into the batch file without having to flow over the wire to the receiver -(when pulling, the sender is remote, and thus can't write the batch). - -dit(bf(--read-batch=FILE)) Apply all of the changes stored in FILE, a -file previously generated by bf(--write-batch). -If em(FILE) is bf(-), the batch data will be read from standard input. -See the "BATCH MODE" section for details. - -) -description( - -dit(bf(--protocol=NUM)) Force an older protocol version to be used. This -is useful for creating a batch file that is compatible with an older -version of rsync. For instance, if rsync 2.6.4 is being used with the -bf(--write-batch) option, but rsync 2.6.3 is what will be used to run the -bf(--read-batch) option, you should use "--protocol=28" when creating the -batch file to force the older protocol version to be used in the batch -file (assuming you can't upgrade the rsync on the reading system). - -dit(bf(--iconv=CONVERT_SPEC)) Rsync can convert filenames between character -sets using this option. Using a CONVERT_SPEC of "." tells rsync to look up -the default character-set via the locale setting. Alternately, you can -fully specify what conversion to do by giving a local and a remote charset -separated by a comma in the order bf(--iconv=LOCAL,REMOTE), e.g. -bf(--iconv=utf8,iso88591). This order ensures that the option -will stay the same whether you're pushing or pulling files. -Finally, you can specify either bf(--no-iconv) or a CONVERT_SPEC of "-" -to turn off any conversion. -The default setting of this option is site-specific, and can also be -affected via the RSYNC_ICONV environment variable. - -For a list of what charset names your local iconv library supports, you can -run "iconv --list". - -If you specify the bf(--protect-args) option (bf(-s)), rsync will translate -the filenames you specify on the command-line that are being sent to the -remote host. See also the bf(--files-from) option. - -Note that rsync does not do any conversion of names in filter files -(including include/exclude files). It is up to you to ensure that you're -specifying matching rules that can match on both sides of the transfer. -For instance, you can specify extra include/exclude rules if there are -filename differences on the two sides that need to be accounted for. - -When you pass an bf(--iconv) option to an rsync daemon that allows it, the -daemon uses the charset specified in its "charset" configuration parameter -regardless of the remote charset you actually pass. Thus, you may feel free to -specify just the local charset for a daemon transfer (e.g. bf(--iconv=utf8)). - -dit(bf(-4, --ipv4) or bf(-6, --ipv6)) Tells rsync to prefer IPv4/IPv6 -when creating sockets. This only affects sockets that rsync has direct -control over, such as the outgoing socket when directly contacting an -rsync daemon. See also these options in the bf(--daemon) mode section. - -If rsync was complied without support for IPv6, the bf(--ipv6) option -will have no effect. The bf(--version) output will tell you if this -is the case. - -dit(bf(--checksum-seed=NUM)) Set the checksum seed to the integer NUM. This 4 -byte checksum seed is included in each block and MD4 file checksum calculation -(the more modern MD5 file checksums don't use a seed). By default the checksum -seed is generated by the server and defaults to the current code(time()). This -option is used to set a specific checksum seed, which is useful for -applications that want repeatable block checksums, or in the case where the -user wants a more random checksum seed. Setting NUM to 0 causes rsync to use -the default of code(time()) for checksum seed. - -) - -manpagesection(DAEMON OPTIONS) - -The options allowed when starting an rsync daemon are as follows: - -description( - -dit(bf(--daemon)) This tells rsync that it is to run as a daemon. The -daemon you start running may be accessed using an rsync client using -the bf(host::module) or bf(rsync://host/module/) syntax. - -If standard input is a socket then rsync will assume that it is being -run via inetd, otherwise it will detach from the current terminal and -become a background daemon. The daemon will read the config file -(rsyncd.conf) on each connect made by a client and respond to -requests accordingly. See the bf(rsyncd.conf)(5) man page for more -details. - -dit(bf(--address)) By default rsync will bind to the wildcard address when -run as a daemon with the bf(--daemon) option. The bf(--address) option -allows you to specify a specific IP address (or hostname) to bind to. This -makes virtual hosting possible in conjunction with the bf(--config) option. -See also the "address" global option in the rsyncd.conf manpage. - -dit(bf(--bwlimit=RATE)) This option allows you to specify the maximum transfer -rate for the data the daemon sends over the socket. The client can still -specify a smaller bf(--bwlimit) value, but no larger value will be allowed. -See the client version of this option (above) for some extra details. - -dit(bf(--config=FILE)) This specifies an alternate config file than -the default. This is only relevant when bf(--daemon) is specified. -The default is /etc/rsyncd.conf unless the daemon is running over -a remote shell program and the remote user is not the super-user; in that case -the default is rsyncd.conf in the current directory (typically $HOME). - -dit(bf(-M, --dparam=OVERRIDE)) This option can be used to set a daemon-config -parameter when starting up rsync in daemon mode. It is equivalent to adding -the parameter at the end of the global settings prior to the first module's -definition. The parameter names can be specified without spaces, if you so -desire. For instance: - -verb( rsync --daemon -M pidfile=/path/rsync.pid ) - -dit(bf(--no-detach)) When running as a daemon, this option instructs -rsync to not detach itself and become a background process. This -option is required when running as a service on Cygwin, and may also -be useful when rsync is supervised by a program such as -bf(daemontools) or AIX's bf(System Resource Controller). -bf(--no-detach) is also recommended when rsync is run under a -debugger. This option has no effect if rsync is run from inetd or -sshd. - -dit(bf(--port=PORT)) This specifies an alternate TCP port number for the -daemon to listen on rather than the default of 873. See also the "port" -global option in the rsyncd.conf manpage. - -dit(bf(--log-file=FILE)) This option tells the rsync daemon to use the -given log-file name instead of using the "log file" setting in the config -file. - -dit(bf(--log-file-format=FORMAT)) This option tells the rsync daemon to use the -given FORMAT string instead of using the "log format" setting in the config -file. It also enables "transfer logging" unless the string is empty, in which -case transfer logging is turned off. - -dit(bf(--sockopts)) This overrides the bf(socket options) setting in the -rsyncd.conf file and has the same syntax. - -dit(bf(-v, --verbose)) This option increases the amount of information the -daemon logs during its startup phase. After the client connects, the -daemon's verbosity level will be controlled by the options that the client -used and the "max verbosity" setting in the module's config section. - -dit(bf(-4, --ipv4) or bf(-6, --ipv6)) Tells rsync to prefer IPv4/IPv6 -when creating the incoming sockets that the rsync daemon will use to -listen for connections. One of these options may be required in older -versions of Linux to work around an IPv6 bug in the kernel (if you see -an "address already in use" error when nothing else is using the port, -try specifying bf(--ipv6) or bf(--ipv4) when starting the daemon). - -If rsync was complied without support for IPv6, the bf(--ipv6) option -will have no effect. The bf(--version) output will tell you if this -is the case. - -dit(bf(-h, --help)) When specified after bf(--daemon), print a short help -page describing the options available for starting an rsync daemon. - -) - -manpagesection(FILTER RULES) - -The filter rules allow for flexible selection of which files to transfer -(include) and which files to skip (exclude). The rules either directly -specify include/exclude patterns or they specify a way to acquire more -include/exclude patterns (e.g. to read them from a file). - -As the list of files/directories to transfer is built, rsync checks each -name to be transferred against the list of include/exclude patterns in -turn, and the first matching pattern is acted on: if it is an exclude -pattern, then that file is skipped; if it is an include pattern then that -filename is not skipped; if no matching pattern is found, then the -filename is not skipped. - -Rsync builds an ordered list of filter rules as specified on the -command-line. Filter rules have the following syntax: - -quote( -tt(RULE [PATTERN_OR_FILENAME])nl() -tt(RULE,MODIFIERS [PATTERN_OR_FILENAME])nl() -) - -You have your choice of using either short or long RULE names, as described -below. If you use a short-named rule, the ',' separating the RULE from the -MODIFIERS is optional. The PATTERN or FILENAME that follows (when present) -must come after either a single space or an underscore (_). -Here are the available rule prefixes: - -quote( -bf(exclude, -) specifies an exclude pattern. nl() -bf(include, +) specifies an include pattern. nl() -bf(merge, .) specifies a merge-file to read for more rules. nl() -bf(dir-merge, :) specifies a per-directory merge-file. nl() -bf(hide, H) specifies a pattern for hiding files from the transfer. nl() -bf(show, S) files that match the pattern are not hidden. nl() -bf(protect, P) specifies a pattern for protecting files from deletion. nl() -bf(risk, R) files that match the pattern are not protected. nl() -bf(clear, !) clears the current include/exclude list (takes no arg) nl() -) - -When rules are being read from a file, empty lines are ignored, as are -comment lines that start with a "#". - -Note that the bf(--include)/bf(--exclude) command-line options do not allow the -full range of rule parsing as described above -- they only allow the -specification of include/exclude patterns plus a "!" token to clear the -list (and the normal comment parsing when rules are read from a file). -If a pattern -does not begin with "- " (dash, space) or "+ " (plus, space), then the -rule will be interpreted as if "+ " (for an include option) or "- " (for -an exclude option) were prefixed to the string. A bf(--filter) option, on -the other hand, must always contain either a short or long rule name at the -start of the rule. - -Note also that the bf(--filter), bf(--include), and bf(--exclude) options take one -rule/pattern each. To add multiple ones, you can repeat the options on -the command-line, use the merge-file syntax of the bf(--filter) option, or -the bf(--include-from)/bf(--exclude-from) options. - -manpagesection(INCLUDE/EXCLUDE PATTERN RULES) - -You can include and exclude files by specifying patterns using the "+", -"-", etc. filter rules (as introduced in the FILTER RULES section above). -The include/exclude rules each specify a pattern that is matched against -the names of the files that are going to be transferred. These patterns -can take several forms: - -itemization( - it() if the pattern starts with a / then it is anchored to a - particular spot in the hierarchy of files, otherwise it is matched - against the end of the pathname. This is similar to a leading ^ in - regular expressions. - Thus "/foo" would match a name of "foo" at either the "root of the - transfer" (for a global rule) or in the merge-file's directory (for a - per-directory rule). - An unqualified "foo" would match a name of "foo" anywhere in the - tree because the algorithm is applied recursively from the - top down; it behaves as if each path component gets a turn at being the - end of the filename. Even the unanchored "sub/foo" would match at - any point in the hierarchy where a "foo" was found within a directory - named "sub". See the section on ANCHORING INCLUDE/EXCLUDE PATTERNS for - a full discussion of how to specify a pattern that matches at the root - of the transfer. - it() if the pattern ends with a / then it will only match a - directory, not a regular file, symlink, or device. - it() rsync chooses between doing a simple string match and wildcard - matching by checking if the pattern contains one of these three wildcard - characters: '*', '?', and '[' . - it() a '*' matches any path component, but it stops at slashes. - it() use '**' to match anything, including slashes. - it() a '?' matches any character except a slash (/). - it() a '[' introduces a character class, such as [a-z] or [[:alpha:]]. - it() in a wildcard pattern, a backslash can be used to escape a wildcard - character, but it is matched literally when no wildcards are present. - This means that there is an extra level of backslash removal when a - pattern contains wildcard characters compared to a pattern that has none. - e.g. if you add a wildcard to "foo\bar" (which matches the backslash) you - would need to use "foo\\bar*" to avoid the "\b" becoming just "b". - it() if the pattern contains a / (not counting a trailing /) or a "**", - then it is matched against the full pathname, including any leading - directories. If the pattern doesn't contain a / or a "**", then it is - matched only against the final component of the filename. - (Remember that the algorithm is applied recursively so "full filename" - can actually be any portion of a path from the starting directory on - down.) - it() a trailing "dir_name/***" will match both the directory (as if - "dir_name/" had been specified) and everything in the directory - (as if "dir_name/**" had been specified). This behavior was added in - version 2.6.7. -) - -Note that, when using the bf(--recursive) (bf(-r)) option (which is implied by -bf(-a)), every subdir component of every path is visited left to right, with -each directory having a chance for exclusion before its content. In this way -include/exclude patterns are applied recursively to the pathname of each node -in the filesystem's tree (those inside the transfer). The exclude patterns -short-circuit the directory traversal stage as rsync finds the files to send. - -For instance, to include "/foo/bar/baz", the directories "/foo" and "/foo/bar" -must not be excluded. Excluding one of those parent directories prevents the -examination of its content, cutting off rsync's recursion into those paths and -rendering the include for "/foo/bar/baz" ineffectual (since rsync can't match -something it never sees in the cut-off section of the directory hierarchy). - -The concept path exclusion is particularly important when using a trailing '*' -rule. For instance, this won't work: - -quote( -tt(+ /some/path/this-file-will-not-be-found)nl() -tt(+ /file-is-included)nl() -tt(- *)nl() -) - -This fails because the parent directory "some" is excluded by the '*' -rule, so rsync never visits any of the files in the "some" or "some/path" -directories. One solution is to ask for all directories in the hierarchy -to be included by using a single rule: "+ */" (put it somewhere before the -"- *" rule), and perhaps use the bf(--prune-empty-dirs) option. Another -solution is to add specific include rules for all -the parent dirs that need to be visited. For instance, this set of rules -works fine: - -quote( -tt(+ /some/)nl() -tt(+ /some/path/)nl() -tt(+ /some/path/this-file-is-found)nl() -tt(+ /file-also-included)nl() -tt(- *)nl() -) - -Here are some examples of exclude/include matching: - -itemization( - it() "- *.o" would exclude all names matching *.o - it() "- /foo" would exclude a file (or directory) named foo in the - transfer-root directory - it() "- foo/" would exclude any directory named foo - it() "- /foo/*/bar" would exclude any file named bar which is at two - levels below a directory named foo in the transfer-root directory - it() "- /foo/**/bar" would exclude any file named bar two - or more levels below a directory named foo in the transfer-root directory - it() The combination of "+ */", "+ *.c", and "- *" would include all - directories and C source files but nothing else (see also the - bf(--prune-empty-dirs) option) - it() The combination of "+ foo/", "+ foo/bar.c", and "- *" would include - only the foo directory and foo/bar.c (the foo directory must be - explicitly included or it would be excluded by the "*") -) - -The following modifiers are accepted after a "+" or "-": - -itemization( - it() A bf(/) specifies that the include/exclude rule should be matched - against the absolute pathname of the current item. For example, - "-/ /etc/passwd" would exclude the passwd file any time the transfer - was sending files from the "/etc" directory, and "-/ subdir/foo" - would always exclude "foo" when it is in a dir named "subdir", even - if "foo" is at the root of the current transfer. - it() A bf(!) specifies that the include/exclude should take effect if - the pattern fails to match. For instance, "-! */" would exclude all - non-directories. - it() A bf(C) is used to indicate that all the global CVS-exclude rules - should be inserted as excludes in place of the "-C". No arg should - follow. - it() An bf(s) is used to indicate that the rule applies to the sending - side. When a rule affects the sending side, it prevents files from - being transferred. The default is for a rule to affect both sides - unless bf(--delete-excluded) was specified, in which case default rules - become sender-side only. See also the hide (H) and show (S) rules, - which are an alternate way to specify sending-side includes/excludes. - it() An bf(r) is used to indicate that the rule applies to the receiving - side. When a rule affects the receiving side, it prevents files from - being deleted. See the bf(s) modifier for more info. See also the - protect (P) and risk (R) rules, which are an alternate way to - specify receiver-side includes/excludes. - it() A bf(p) indicates that a rule is perishable, meaning that it is - ignored in directories that are being deleted. For instance, the bf(-C) - option's default rules that exclude things like "CVS" and "*.o" are - marked as perishable, and will not prevent a directory that was removed - on the source from being deleted on the destination. - it() An bf(x) indicates that a rule affects xattr names in xattr copy/delete - operations (and is thus ignored when matching file/dir names). If no - xattr-matching rules are specified, a default xattr filtering rule is - used (see the bf(--xattrs) option). -) - -manpagesection(MERGE-FILE FILTER RULES) - -You can merge whole files into your filter rules by specifying either a -merge (.) or a dir-merge (:) filter rule (as introduced in the FILTER RULES -section above). - -There are two kinds of merged files -- single-instance ('.') and -per-directory (':'). A single-instance merge file is read one time, and -its rules are incorporated into the filter list in the place of the "." -rule. For per-directory merge files, rsync will scan every directory that -it traverses for the named file, merging its contents when the file exists -into the current list of inherited rules. These per-directory rule files -must be created on the sending side because it is the sending side that is -being scanned for the available files to transfer. These rule files may -also need to be transferred to the receiving side if you want them to -affect what files don't get deleted (see PER-DIRECTORY RULES AND DELETE -below). - -Some examples: - -quote( -tt(merge /etc/rsync/default.rules)nl() -tt(. /etc/rsync/default.rules)nl() -tt(dir-merge .per-dir-filter)nl() -tt(dir-merge,n- .non-inherited-per-dir-excludes)nl() -tt(:n- .non-inherited-per-dir-excludes)nl() -) - -The following modifiers are accepted after a merge or dir-merge rule: - -itemization( - it() A bf(-) specifies that the file should consist of only exclude - patterns, with no other rule-parsing except for in-file comments. - it() A bf(+) specifies that the file should consist of only include - patterns, with no other rule-parsing except for in-file comments. - it() A bf(C) is a way to specify that the file should be read in a - CVS-compatible manner. This turns on 'n', 'w', and '-', but also - allows the list-clearing token (!) to be specified. If no filename is - provided, ".cvsignore" is assumed. - it() A bf(e) will exclude the merge-file name from the transfer; e.g. - "dir-merge,e .rules" is like "dir-merge .rules" and "- .rules". - it() An bf(n) specifies that the rules are not inherited by subdirectories. - it() A bf(w) specifies that the rules are word-split on whitespace instead - of the normal line-splitting. This also turns off comments. Note: the - space that separates the prefix from the rule is treated specially, so - "- foo + bar" is parsed as two rules (assuming that prefix-parsing wasn't - also disabled). - it() You may also specify any of the modifiers for the "+" or "-" rules - (above) in order to have the rules that are read in from the file - default to having that modifier set (except for the bf(!) modifier, which - would not be useful). For instance, "merge,-/ .excl" would - treat the contents of .excl as absolute-path excludes, - while "dir-merge,s .filt" and ":sC" would each make all their - per-directory rules apply only on the sending side. If the merge rule - specifies sides to affect (via the bf(s) or bf(r) modifier or both), - then the rules in the file must not specify sides (via a modifier or - a rule prefix such as bf(hide)). -) - -Per-directory rules are inherited in all subdirectories of the directory -where the merge-file was found unless the 'n' modifier was used. Each -subdirectory's rules are prefixed to the inherited per-directory rules -from its parents, which gives the newest rules a higher priority than the -inherited rules. The entire set of dir-merge rules are grouped together in -the spot where the merge-file was specified, so it is possible to override -dir-merge rules via a rule that got specified earlier in the list of global -rules. When the list-clearing rule ("!") is read from a per-directory -file, it only clears the inherited rules for the current merge file. - -Another way to prevent a single rule from a dir-merge file from being inherited is to -anchor it with a leading slash. Anchored rules in a per-directory -merge-file are relative to the merge-file's directory, so a pattern "/foo" -would only match the file "foo" in the directory where the dir-merge filter -file was found. - -Here's an example filter file which you'd specify via bf(--filter=". file":) - -quote( -tt(merge /home/user/.global-filter)nl() -tt(- *.gz)nl() -tt(dir-merge .rules)nl() -tt(+ *.[ch])nl() -tt(- *.o)nl() -) - -This will merge the contents of the /home/user/.global-filter file at the -start of the list and also turns the ".rules" filename into a per-directory -filter file. All rules read in prior to the start of the directory scan -follow the global anchoring rules (i.e. a leading slash matches at the root -of the transfer). - -If a per-directory merge-file is specified with a path that is a parent -directory of the first transfer directory, rsync will scan all the parent -dirs from that starting point to the transfer directory for the indicated -per-directory file. For instance, here is a common filter (see bf(-F)): - -quote(tt(--filter=': /.rsync-filter')) - -That rule tells rsync to scan for the file .rsync-filter in all -directories from the root down through the parent directory of the -transfer prior to the start of the normal directory scan of the file in -the directories that are sent as a part of the transfer. (Note: for an -rsync daemon, the root is always the same as the module's "path".) - -Some examples of this pre-scanning for per-directory files: - -quote( -tt(rsync -avF /src/path/ /dest/dir)nl() -tt(rsync -av --filter=': ../../.rsync-filter' /src/path/ /dest/dir)nl() -tt(rsync -av --filter=': .rsync-filter' /src/path/ /dest/dir)nl() -) - -The first two commands above will look for ".rsync-filter" in "/" and -"/src" before the normal scan begins looking for the file in "/src/path" -and its subdirectories. The last command avoids the parent-dir scan -and only looks for the ".rsync-filter" files in each directory that is -a part of the transfer. - -If you want to include the contents of a ".cvsignore" in your patterns, -you should use the rule ":C", which creates a dir-merge of the .cvsignore -file, but parsed in a CVS-compatible manner. You can -use this to affect where the bf(--cvs-exclude) (bf(-C)) option's inclusion of the -per-directory .cvsignore file gets placed into your rules by putting the -":C" wherever you like in your filter rules. Without this, rsync would -add the dir-merge rule for the .cvsignore file at the end of all your other -rules (giving it a lower priority than your command-line rules). For -example: - -quote( -tt(cat <<EOT | rsync -avC --filter='. -' a/ b)nl() -tt(+ foo.o)nl() -tt(:C)nl() -tt(- *.old)nl() -tt(EOT)nl() -tt(rsync -avC --include=foo.o -f :C --exclude='*.old' a/ b)nl() -) - -Both of the above rsync commands are identical. Each one will merge all -the per-directory .cvsignore rules in the middle of the list rather than -at the end. This allows their dir-specific rules to supersede the rules -that follow the :C instead of being subservient to all your rules. To -affect the other CVS exclude rules (i.e. the default list of exclusions, -the contents of $HOME/.cvsignore, and the value of $CVSIGNORE) you should -omit the bf(-C) command-line option and instead insert a "-C" rule into -your filter rules; e.g. "bf(--filter=-C)". - -manpagesection(LIST-CLEARING FILTER RULE) - -You can clear the current include/exclude list by using the "!" filter -rule (as introduced in the FILTER RULES section above). The "current" -list is either the global list of rules (if the rule is encountered while -parsing the filter options) or a set of per-directory rules (which are -inherited in their own sub-list, so a subdirectory can use this to clear -out the parent's rules). - -manpagesection(ANCHORING INCLUDE/EXCLUDE PATTERNS) - -As mentioned earlier, global include/exclude patterns are anchored at the -"root of the transfer" (as opposed to per-directory patterns, which are -anchored at the merge-file's directory). If you think of the transfer as -a subtree of names that are being sent from sender to receiver, the -transfer-root is where the tree starts to be duplicated in the destination -directory. This root governs where patterns that start with a / match. - -Because the matching is relative to the transfer-root, changing the -trailing slash on a source path or changing your use of the bf(--relative) -option affects the path you need to use in your matching (in addition to -changing how much of the file tree is duplicated on the destination -host). The following examples demonstrate this. - -Let's say that we want to match two source files, one with an absolute -path of "/home/me/foo/bar", and one with a path of "/home/you/bar/baz". -Here is how the various command choices differ for a 2-source transfer: - -quote( - Example cmd: rsync -a /home/me /home/you /dest nl() - +/- pattern: /me/foo/bar nl() - +/- pattern: /you/bar/baz nl() - Target file: /dest/me/foo/bar nl() - Target file: /dest/you/bar/baz nl() -) - -quote( - Example cmd: rsync -a /home/me/ /home/you/ /dest nl() - +/- pattern: /foo/bar (note missing "me") nl() - +/- pattern: /bar/baz (note missing "you") nl() - Target file: /dest/foo/bar nl() - Target file: /dest/bar/baz nl() -) - -quote( - Example cmd: rsync -a --relative /home/me/ /home/you /dest nl() - +/- pattern: /home/me/foo/bar (note full path) nl() - +/- pattern: /home/you/bar/baz (ditto) nl() - Target file: /dest/home/me/foo/bar nl() - Target file: /dest/home/you/bar/baz nl() -) - -quote( - Example cmd: cd /home; rsync -a --relative me/foo you/ /dest nl() - +/- pattern: /me/foo/bar (starts at specified path) nl() - +/- pattern: /you/bar/baz (ditto) nl() - Target file: /dest/me/foo/bar nl() - Target file: /dest/you/bar/baz nl() -) - -The easiest way to see what name you should filter is to just -look at the output when using bf(--verbose) and put a / in front of the name -(use the bf(--dry-run) option if you're not yet ready to copy any files). - -manpagesection(PER-DIRECTORY RULES AND DELETE) - -Without a delete option, per-directory rules are only relevant on the -sending side, so you can feel free to exclude the merge files themselves -without affecting the transfer. To make this easy, the 'e' modifier adds -this exclude for you, as seen in these two equivalent commands: - -quote( -tt(rsync -av --filter=': .excl' --exclude=.excl host:src/dir /dest)nl() -tt(rsync -av --filter=':e .excl' host:src/dir /dest)nl() -) - -However, if you want to do a delete on the receiving side AND you want some -files to be excluded from being deleted, you'll need to be sure that the -receiving side knows what files to exclude. The easiest way is to include -the per-directory merge files in the transfer and use bf(--delete-after), -because this ensures that the receiving side gets all the same exclude -rules as the sending side before it tries to delete anything: - -quote(tt(rsync -avF --delete-after host:src/dir /dest)) - -However, if the merge files are not a part of the transfer, you'll need to -either specify some global exclude rules (i.e. specified on the command -line), or you'll need to maintain your own per-directory merge files on -the receiving side. An example of the first is this (assume that the -remote .rules files exclude themselves): - -verb(rsync -av --filter=': .rules' --filter='. /my/extra.rules' - --delete host:src/dir /dest) - -In the above example the extra.rules file can affect both sides of the -transfer, but (on the sending side) the rules are subservient to the rules -merged from the .rules files because they were specified after the -per-directory merge rule. - -In one final example, the remote side is excluding the .rsync-filter -files from the transfer, but we want to use our own .rsync-filter files -to control what gets deleted on the receiving side. To do this we must -specifically exclude the per-directory merge files (so that they don't get -deleted) and then put rules into the local files to control what else -should not get deleted. Like one of these commands: - -verb( rsync -av --filter=':e /.rsync-filter' --delete \ - host:src/dir /dest - rsync -avFF --delete host:src/dir /dest) - -manpagesection(BATCH MODE) - -Batch mode can be used to apply the same set of updates to many -identical systems. Suppose one has a tree which is replicated on a -number of hosts. Now suppose some changes have been made to this -source tree and those changes need to be propagated to the other -hosts. In order to do this using batch mode, rsync is run with the -write-batch option to apply the changes made to the source tree to one -of the destination trees. The write-batch option causes the rsync -client to store in a "batch file" all the information needed to repeat -this operation against other, identical destination trees. - -Generating the batch file once saves having to perform the file -status, checksum, and data block generation more than once when -updating multiple destination trees. Multicast transport protocols can -be used to transfer the batch update files in parallel to many hosts -at once, instead of sending the same data to every host individually. - -To apply the recorded changes to another destination tree, run rsync -with the read-batch option, specifying the name of the same batch -file, and the destination tree. Rsync updates the destination tree -using the information stored in the batch file. - -For your convenience, a script file is also created when the write-batch -option is used: it will be named the same as the batch file with ".sh" -appended. This script file contains a command-line suitable for updating a -destination tree using the associated batch file. It can be executed using -a Bourne (or Bourne-like) shell, optionally passing in an alternate -destination tree pathname which is then used instead of the original -destination path. This is useful when the destination tree path on the -current host differs from the one used to create the batch file. - -Examples: - -quote( -tt($ rsync --write-batch=foo -a host:/source/dir/ /adest/dir/)nl() -tt($ scp foo* remote:)nl() -tt($ ssh remote ./foo.sh /bdest/dir/)nl() -) - -quote( -tt($ rsync --write-batch=foo -a /source/dir/ /adest/dir/)nl() -tt($ ssh remote rsync --read-batch=- -a /bdest/dir/ <foo)nl() -) - -In these examples, rsync is used to update /adest/dir/ from /source/dir/ -and the information to repeat this operation is stored in "foo" and -"foo.sh". The host "remote" is then updated with the batched data going -into the directory /bdest/dir. The differences between the two examples -reveals some of the flexibility you have in how you deal with batches: - -itemization( - it() The first example shows that the initial copy doesn't have to be - local -- you can push or pull data to/from a remote host using either the - remote-shell syntax or rsync daemon syntax, as desired. - it() The first example uses the created "foo.sh" file to get the right - rsync options when running the read-batch command on the remote host. - it() The second example reads the batch data via standard input so that - the batch file doesn't need to be copied to the remote machine first. - This example avoids the foo.sh script because it needed to use a modified - bf(--read-batch) option, but you could edit the script file if you wished to - make use of it (just be sure that no other option is trying to use - standard input, such as the "bf(--exclude-from=-)" option). -) - -Caveats: - -The read-batch option expects the destination tree that it is updating -to be identical to the destination tree that was used to create the -batch update fileset. When a difference between the destination trees -is encountered the update might be discarded with a warning (if the file -appears to be up-to-date already) or the file-update may be attempted -and then, if the file fails to verify, the update discarded with an -error. This means that it should be safe to re-run a read-batch operation -if the command got interrupted. If you wish to force the batched-update to -always be attempted regardless of the file's size and date, use the bf(-I) -option (when reading the batch). -If an error occurs, the destination tree will probably be in a -partially updated state. In that case, rsync can -be used in its regular (non-batch) mode of operation to fix up the -destination tree. - -The rsync version used on all destinations must be at least as new as the -one used to generate the batch file. Rsync will die with an error if the -protocol version in the batch file is too new for the batch-reading rsync -to handle. See also the bf(--protocol) option for a way to have the -creating rsync generate a batch file that an older rsync can understand. -(Note that batch files changed format in version 2.6.3, so mixing versions -older than that with newer versions will not work.) - -When reading a batch file, rsync will force the value of certain options -to match the data in the batch file if you didn't set them to the same -as the batch-writing command. Other options can (and should) be changed. -For instance bf(--write-batch) changes to bf(--read-batch), -bf(--files-from) is dropped, and the -bf(--filter)/bf(--include)/bf(--exclude) options are not needed unless -one of the bf(--delete) options is specified. - -The code that creates the BATCH.sh file transforms any filter/include/exclude -options into a single list that is appended as a "here" document to the -shell script file. An advanced user can use this to modify the exclude -list if a change in what gets deleted by bf(--delete) is desired. A normal -user can ignore this detail and just use the shell script as an easy way -to run the appropriate bf(--read-batch) command for the batched data. - -The original batch mode in rsync was based on "rsync+", but the latest -version uses a new implementation. - -manpagesection(SYMBOLIC LINKS) - -Three basic behaviors are possible when rsync encounters a symbolic -link in the source directory. - -By default, symbolic links are not transferred at all. A message -"skipping non-regular" file is emitted for any symlinks that exist. - -If bf(--links) is specified, then symlinks are recreated with the same -target on the destination. Note that bf(--archive) implies -bf(--links). - -If bf(--copy-links) is specified, then symlinks are "collapsed" by -copying their referent, rather than the symlink. - -Rsync can also distinguish "safe" and "unsafe" symbolic links. An -example where this might be used is a web site mirror that wishes to -ensure that the rsync module that is copied does not include symbolic links to -bf(/etc/passwd) in the public section of the site. Using -bf(--copy-unsafe-links) will cause any links to be copied as the file -they point to on the destination. Using bf(--safe-links) will cause -unsafe links to be omitted altogether. (Note that you must specify -bf(--links) for bf(--safe-links) to have any effect.) - -Symbolic links are considered unsafe if they are absolute symlinks -(start with bf(/)), empty, or if they contain enough ".." -components to ascend from the directory being copied. - -Here's a summary of how the symlink options are interpreted. The list is -in order of precedence, so if your combination of options isn't mentioned, -use the first line that is a complete subset of your options: - -dit(bf(--copy-links)) Turn all symlinks into normal files (leaving no -symlinks for any other options to affect). - -dit(bf(--links --copy-unsafe-links)) Turn all unsafe symlinks into files -and duplicate all safe symlinks. - -dit(bf(--copy-unsafe-links)) Turn all unsafe symlinks into files, noisily -skip all safe symlinks. - -dit(bf(--links --safe-links)) Duplicate safe symlinks and skip unsafe -ones. - -dit(bf(--links)) Duplicate all symlinks. - -manpagediagnostics() - -rsync occasionally produces error messages that may seem a little -cryptic. The one that seems to cause the most confusion is "protocol -version mismatch -- is your shell clean?". - -This message is usually caused by your startup scripts or remote shell -facility producing unwanted garbage on the stream that rsync is using -for its transport. The way to diagnose this problem is to run your -remote shell like this: - -quote(tt(ssh remotehost /bin/true > out.dat)) - -then look at out.dat. If everything is working correctly then out.dat -should be a zero length file. If you are getting the above error from -rsync then you will probably find that out.dat contains some text or -data. Look at the contents and try to work out what is producing -it. The most common cause is incorrectly configured shell startup -scripts (such as .cshrc or .profile) that contain output statements -for non-interactive logins. - -If you are having trouble debugging filter patterns, then -try specifying the bf(-vv) option. At this level of verbosity rsync will -show why each individual file is included or excluded. - -manpagesection(EXIT VALUES) - -description( -dit(bf(0)) Success -dit(bf(1)) Syntax or usage error -dit(bf(2)) Protocol incompatibility -dit(bf(3)) Errors selecting input/output files, dirs -dit(bf(4)) Requested action not supported: an attempt -was made to manipulate 64-bit files on a platform that cannot support -them; or an option was specified that is supported by the client and -not by the server. -dit(bf(5)) Error starting client-server protocol -dit(bf(6)) Daemon unable to append to log-file -dit(bf(10)) Error in socket I/O -dit(bf(11)) Error in file I/O -dit(bf(12)) Error in rsync protocol data stream -dit(bf(13)) Errors with program diagnostics -dit(bf(14)) Error in IPC code -dit(bf(20)) Received SIGUSR1 or SIGINT -dit(bf(21)) Some error returned by code(waitpid()) -dit(bf(22)) Error allocating core memory buffers -dit(bf(23)) Partial transfer due to error -dit(bf(24)) Partial transfer due to vanished source files -dit(bf(25)) The --max-delete limit stopped deletions -dit(bf(30)) Timeout in data send/receive -dit(bf(35)) Timeout waiting for daemon connection -) - -manpagesection(ENVIRONMENT VARIABLES) - -description( -dit(bf(CVSIGNORE)) The CVSIGNORE environment variable supplements any -ignore patterns in .cvsignore files. See the bf(--cvs-exclude) option for -more details. -dit(bf(RSYNC_ICONV)) Specify a default bf(--iconv) setting using this -environment variable. (First supported in 3.0.0.) -dit(bf(RSYNC_PROTECT_ARGS)) Specify a non-zero numeric value if you want the -bf(--protect-args) option to be enabled by default, or a zero value to make -sure that it is disabled by default. (First supported in 3.1.0.) -dit(bf(RSYNC_RSH)) The RSYNC_RSH environment variable allows you to -override the default shell used as the transport for rsync. Command line -options are permitted after the command name, just as in the bf(-e) option. -dit(bf(RSYNC_PROXY)) The RSYNC_PROXY environment variable allows you to -redirect your rsync client to use a web proxy when connecting to a -rsync daemon. You should set RSYNC_PROXY to a hostname:port pair. -dit(bf(RSYNC_PASSWORD)) Setting RSYNC_PASSWORD to the required -password allows you to run authenticated rsync connections to an rsync -daemon without user intervention. Note that this does not supply a -password to a remote shell transport such as ssh; to learn how to do that, -consult the remote shell's documentation. -dit(bf(USER) or bf(LOGNAME)) The USER or LOGNAME environment variables -are used to determine the default username sent to an rsync daemon. -If neither is set, the username defaults to "nobody". -dit(bf(HOME)) The HOME environment variable is used to find the user's -default .cvsignore file. -) - -manpagefiles() - -/etc/rsyncd.conf or rsyncd.conf - -manpageseealso() - -bf(rsyncd.conf)(5) - -manpagebugs() - -times are transferred as *nix time_t values - -When transferring to FAT filesystems rsync may re-sync -unmodified files. -See the comments on the bf(--modify-window) option. - -file permissions, devices, etc. are transferred as native numerical -values - -see also the comments on the bf(--delete) option - -Please report bugs! See the web site at -url(http://rsync.samba.org/)(http://rsync.samba.org/) - -manpagesection(VERSION) - -This man page is current for version 3.1.3 of rsync. - -manpagesection(INTERNAL OPTIONS) - -The options bf(--server) and bf(--sender) are used internally by rsync, -and should never be typed by a user under normal circumstances. Some -awareness of these options may be needed in certain scenarios, such as -when setting up a login that can only run an rsync command. For instance, -the support directory of the rsync distribution has an example script -named rrsync (for restricted rsync) that can be used with a restricted -ssh login. - -manpagesection(CREDITS) - -rsync is distributed under the GNU General Public License. See the file -COPYING for details. - -A WEB site is available at -url(http://rsync.samba.org/)(http://rsync.samba.org/). The site -includes an FAQ-O-Matic which may cover questions unanswered by this -manual page. - -The primary ftp site for rsync is -url(ftp://rsync.samba.org/pub/rsync)(ftp://rsync.samba.org/pub/rsync). - -We would be delighted to hear from you if you like this program. -Please contact the mailing-list at rsync@lists.samba.org. - -This program uses the excellent zlib compression library written by -Jean-loup Gailly and Mark Adler. - -manpagesection(THANKS) - -Special thanks go out to: John Van Essen, Matt McCutchen, Wesley W. Terpstra, -David Dykstra, Jos Backus, Sebastian Krahmer, Martin Pool, and our -gone-but-not-forgotten compadre, J.W. Schultz. - -Thanks also to Richard Brent, Brendan Mackay, Bill Waite, Stephen Rothwell -and David Bell. I've probably missed some people, my apologies if I have. - -manpageauthor() - -rsync was originally written by Andrew Tridgell and Paul Mackerras. -Many people have later contributed to it. It is currently maintained -by Wayne Davison. - -Mailing lists for support and development are available at -url(http://lists.samba.org)(lists.samba.org) diff --git a/rsyncd.conf.5 b/rsyncd.conf.5 index 5e7d80ff..0af01e78 100644 --- a/rsyncd.conf.5 +++ b/rsyncd.conf.5 @@ -1,1038 +1,1165 @@ -.TH "rsyncd.conf" "5" "28 Jan 2018" "" "" +.TH "rsyncd.conf" "5" "19 Jun 2020" "rsyncd.conf 3.2.0" "User Commands" +.P .SH "NAME" +.P rsyncd.conf \- configuration file for rsync in daemon mode +.P .SH "SYNOPSIS" - -.PP +.P rsyncd.conf -.PP +.P .SH "DESCRIPTION" - -.PP -The rsyncd.conf file is the runtime configuration file for rsync when -run as an rsync daemon. -.PP -The rsyncd.conf file controls authentication, access, logging and -available modules. -.PP +.P +The rsyncd.conf file is the runtime configuration file for rsync when run as an +rsync daemon. +.P +The rsyncd.conf file controls authentication, access, logging and available +modules. +.P .SH "FILE FORMAT" - -.PP -The file consists of modules and parameters. A module begins with the -name of the module in square brackets and continues until the next -module begins. Modules contain parameters of the form \(dq\&name = value\(dq\&. -.PP -The file is line\-based \-\- that is, each newline\-terminated line represents +.P +The file consists of modules and parameters. A module begins with the name of +the module in square brackets and continues until the next module begins. +Modules contain parameters of the form \fBname\ =\ value\fP. +.P +The file is line-based\ \-\- that is, each newline-terminated line represents either a comment, a module name or a parameter. -.PP -Only the first equals sign in a parameter is significant. Whitespace before -or after the first equals sign is discarded. Leading, trailing and internal -whitespace in module and parameter names is irrelevant. Leading and -trailing whitespace in a parameter value is discarded. Internal whitespace -within a parameter value is retained verbatim. -.PP -Any line \fBbeginning\fP with a hash (#) is ignored, as are lines containing +.P +Only the first equals sign in a parameter is significant. Whitespace before or +after the first equals sign is discarded. Leading, trailing and internal +whitespace in module and parameter names is irrelevant. Leading and trailing +whitespace in a parameter value is discarded. Internal whitespace within a +parameter value is retained verbatim. +.P +Any line \fBbeginning\fP with a hash (\fB#\fP) is ignored, as are lines containing only whitespace. (If a hash occurs after anything other than leading -whitespace, it is considered a part of the line\(cq\&s content.) -.PP -Any line ending in a \e is \(dq\&continued\(dq\& on the next line in the -customary UNIX fashion. -.PP -The values following the equals sign in parameters are all either a string -(no quotes needed) or a boolean, which may be given as yes/no, 0/1 or -true/false. Case is not significant in boolean values, but is preserved -in string values. -.PP +whitespace, it is considered a part of the line's content.) +.P +Any line ending in a \fB\\\fP is "continued" on the next line in the customary UNIX +fashion. +.P +The values following the equals sign in parameters are all either a string (no +quotes needed) or a boolean, which may be given as yes/no, 0/1 or true/false. +Case is not significant in boolean values, but is preserved in string values. +.P .SH "LAUNCHING THE RSYNC DAEMON" - -.PP +.P The rsync daemon is launched by specifying the \fB\-\-daemon\fP option to rsync. -.PP -The daemon must run with root privileges if you wish to use chroot, to -bind to a port numbered under 1024 (as is the default 873), or to set -file ownership. Otherwise, it must just have permission to read and -write the appropriate data, log, and lock files. -.PP -You can launch it either via inetd, as a stand\-alone daemon, or from -an rsync client via a remote shell. If run as a stand\-alone daemon then -just run the command \(dq\&\fBrsync \-\-daemon\fP\(dq\& from a suitable startup script. -.PP +.P +The daemon must run with root privileges if you wish to use chroot, to bind to +a port numbered under 1024 (as is the default 873), or to set file ownership. +Otherwise, it must just have permission to read and write the appropriate data, +log, and lock files. +.P +You can launch it either via inetd, as a stand-alone daemon, or from an rsync +client via a remote shell. If run as a stand-alone daemon then just run the +command "\fBrsync\ \-\-daemon\fP" from a suitable startup script. +.P When run via inetd you should add a line like this to /etc/services: -.PP -.nf - rsync 873/tcp -.fi - -.PP +.RS 4 +.P +.nf +rsync 873/tcp +.fi +.RE +.P and a single line something like this to /etc/inetd.conf: -.PP -.nf - rsync stream tcp nowait root /usr/bin/rsync rsyncd \-\-daemon -.fi - -.PP -Replace \(dq\&/usr/bin/rsync\(dq\& with the path to where you have rsync installed on +.RS 4 +.P +.nf +rsync stream tcp nowait root /usr/bin/rsync rsyncd --daemon +.fi +.RE +.P +Replace "/usr/bin/rsync" with the path to where you have rsync installed on your system. You will then need to send inetd a HUP signal to tell it to reread its config file. -.PP -Note that you should \fBnot\fP send the rsync daemon a HUP signal to force -it to reread the \f(CWrsyncd.conf\fP file. The file is re\-read on each client -connection. -.PP +.P +Note that you should \fBnot\fP send the rsync daemon a HUP signal to force it to +reread the \fBrsyncd.conf\fP file. The file is re-read on each client connection. +.P .SH "GLOBAL PARAMETERS" - -.PP -The first parameters in the file (before a [module] header) are the -global parameters. -Rsync also allows for the use of a \(dq\&[global]\(dq\& module name to indicate the -start of one or more global\-parameter sections (the name must be lower case). -.PP -You may also include any module parameters in the global part of the -config file in which case the supplied value will override the -default for that parameter. -.PP +.P +The first parameters in the file (before a [module] header) are the global +parameters. Rsync also allows for the use of a "[global]" module name to +indicate the start of one or more global-parameter sections (the name must be +lower case). +.P +You may also include any module parameters in the global part of the config +file in which case the supplied value will override the default for that +parameter. +.P You may use references to environment variables in the values of parameters. String parameters will have %VAR% references expanded as late as possible (when -the string is used in the program), allowing for the use of variables that -rsync sets at connection time, such as RSYNC_USER_NAME. Non\-string parameters -(such as true/false settings) are expanded when read from the config file. If -a variable does not exist in the environment, or if a sequence of characters is -not a valid reference (such as an un\-paired percent sign), the raw characters -are passed through unchanged. This helps with backward compatibility and -safety (e.g. expanding a non\-existent %VAR% to an empty string in a path could -result in a very unsafe path). The safest way to insert a literal % into a -value is to use %%. -.PP -.IP "\fBmotd file\fP" -This parameter allows you to specify a -\(dq\&message of the day\(dq\& to display to clients on each connect. This -usually contains site information and any legal notices. The default -is no motd file. -This can be overridden by the \fB\-\-dparam=motdfile=FILE\fP -command\-line option when starting the daemon. -.IP -.IP "\fBpid file\fP" -This parameter tells the rsync daemon to write -its process ID to that file. If the file already exists, the rsync -daemon will abort rather than overwrite the file. -This can be overridden by the \fB\-\-dparam=pidfile=FILE\fP -command\-line option when starting the daemon. -.IP +the string is first used in the program), allowing for the use of variables +that rsync sets at connection time, such as RSYNC_USER_NAME. Non-string +parameters (such as true/false settings) are expanded when read from the config +file. If a variable does not exist in the environment, or if a sequence of +characters is not a valid reference (such as an un-paired percent sign), the +raw characters are passed through unchanged. This helps with backward +compatibility and safety (e.g. expanding a non-existent %VAR% to an empty +string in a path could result in a very unsafe path). The safest way to insert +a literal % into a value is to use %%. +.P +.IP "\fBmotd\ file\fP" +This parameter allows you to specify a "message of the day" to display to +clients on each connect. This usually contains site information and any +legal notices. The default is no motd file. This can be overridden by the +\fB\-\-dparam=motdfile=FILE\fP command-line option when starting the daemon. +.IP "\fBpid\ file\fP" +This parameter tells the rsync daemon to write its process ID to that file. +The rsync keeps the file locked so that it can know when it is safe to +overwrite an existing file. +.IP +The filename can be overridden by the \fB\-\-dparam=pidfile=FILE\fP command-line +option when starting the daemon. .IP "\fBport\fP" -You can override the default port the daemon will listen on -by specifying this value (defaults to 873). This is ignored if the daemon -is being run by inetd, and is superseded by the \fB\-\-port\fP command\-line option. -.IP +You can override the default port the daemon will listen on by specifying +this value (defaults to 873). This is ignored if the daemon is being run +by inetd, and is superseded by the \fB\-\-port\fP command-line option. .IP "\fBaddress\fP" -You can override the default IP address the daemon -will listen on by specifying this value. This is ignored if the daemon is -being run by inetd, and is superseded by the \fB\-\-address\fP command\-line option. -.IP -.IP "\fBsocket options\fP" -This parameter can provide endless fun for people -who like to tune their systems to the utmost degree. You can set all -sorts of socket options which may make transfers faster (or -slower!). Read the man page for the -\f(CWsetsockopt()\fP -system call for -details on some of the options you may be able to set. By default no -special socket options are set. These settings can also be specified -via the \fB\-\-sockopts\fP command\-line option. -.IP -.IP "\fBlisten backlog\fP" -You can override the default backlog value when the -daemon listens for connections. It defaults to 5. -.IP +You can override the default IP address the daemon will listen on by +specifying this value. This is ignored if the daemon is being run by +inetd, and is superseded by the \fB\-\-address\fP command-line option. +.IP "\fBsocket\ options\fP" +This parameter can provide endless fun for people who like to tune their +systems to the utmost degree. You can set all sorts of socket options which +may make transfers faster (or slower!). Read the man page for the +\fBsetsockopt()\fP system call for details on some of the options you may be +able to set. By default no special socket options are set. These settings +can also be specified via the \fB\-\-sockopts\fP command-line option. +.IP "\fBlisten\ backlog\fP" +You can override the default backlog value when the daemon listens for +connections. It defaults to 5. +.P .SH "MODULE PARAMETERS" - -.PP -After the global parameters you should define a number of modules, each -module exports a directory tree as a symbolic name. Modules are -exported by specifying a module name in square brackets [module] -followed by the parameters for that module. -The module name cannot contain a slash or a closing square bracket. If the -name contains whitespace, each internal sequence of whitespace will be +.P +After the global parameters you should define a number of modules, each module +exports a directory tree as a symbolic name. Modules are exported by specifying +a module name in square brackets [module] followed by the parameters for that +module. The module name cannot contain a slash or a closing square bracket. +If the name contains whitespace, each internal sequence of whitespace will be changed into a single space, while leading or trailing whitespace will be -discarded. -Also, the name cannot be \(dq\&global\(dq\& as that exact name indicates that +discarded. Also, the name cannot be "global" as that exact name indicates that global parameters follow (see above). -.PP +.P As with GLOBAL PARAMETERS, you may use references to environment variables in the values of parameters. See the GLOBAL PARAMETERS section for more details. -.PP +.P .IP "\fBcomment\fP" -This parameter specifies a description string -that is displayed next to the module name when clients obtain a list -of available modules. The default is no comment. -.IP +This parameter specifies a description string that is displayed next to the +module name when clients obtain a list of available modules. The default is +no comment. .IP "\fBpath\fP" -This parameter specifies the directory in the daemon\(cq\&s -filesystem to make available in this module. You must specify this parameter -for each module in \f(CWrsyncd.conf\fP. -.IP -You may base the path\(cq\&s value off of an environment variable by surrounding +This parameter specifies the directory in the daemon's filesystem to make +available in this module. You must specify this parameter for each module +in \fBrsyncd.conf\fP. +.IP +You may base the path's value off of an environment variable by surrounding the variable name with percent signs. You can even reference a variable -that is set by rsync when the user connects. -For example, this would use the authorizing user\(cq\&s name in the path: -.IP -.nf - path = /home/%RSYNC_USER_NAME% -.fi - -.IP -It is fine if the path includes internal spaces \-\- they will be retained -verbatim (which means that you shouldn\(cq\&t try to escape them). If your final -directory has a trailing space (and this is somehow not something you wish to -fix), append a trailing slash to the path to avoid losing the trailing -whitespace. -.IP -.IP "\fBuse chroot\fP" -If \(dq\&use chroot\(dq\& is true, the rsync daemon will chroot -to the \(dq\&path\(dq\& before starting the file transfer with the client. This has -the advantage of extra protection against possible implementation security -holes, but it has the disadvantages of requiring super\-user privileges, -of not being able to follow symbolic links that are either absolute or outside -of the new root path, and of complicating the preservation of users and groups -by name (see below). -.IP -As an additional safety feature, you can specify a dot\-dir in the module\(cq\&s -\(dq\&path\(dq\& to indicate the point where the chroot should occur. This allows rsync -to run in a chroot with a non\-\(dq\&/\(dq\& path for the top of the transfer hierarchy. -Doing this guards against unintended library loading (since those absolute -paths will not be inside the transfer hierarchy unless you have used an unwise -pathname), and lets you setup libraries for the chroot that are outside of the -transfer. For example, specifying \(dq\&/var/rsync/./module1\(dq\& will chroot to the -\(dq\&/var/rsync\(dq\& directory and set the inside\-chroot path to \(dq\&/module1\(dq\&. If you -had omitted the dot\-dir, the chroot would have used the whole path, and the -inside\-chroot path would have been \(dq\&/\(dq\&. -.IP -When both \(dq\&use chroot\(dq\& and \(dq\&daemon chroot\(dq\& are false, OR the inside\-chroot path -of \(dq\&use chroot\(dq\& is not \(dq\&/\(dq\&, rsync will: (1) munge symlinks by -default for security reasons (see \(dq\&munge symlinks\(dq\& for a way to turn this -off, but only if you trust your users), (2) substitute leading slashes in -absolute paths with the module\(cq\&s path (so that options such as -\fB\-\-backup\-dir\fP, \fB\-\-compare\-dest\fP, etc. interpret an absolute path as -rooted in the module\(cq\&s \(dq\&path\(dq\& dir), and (3) trim \(dq\&..\(dq\& path elements from -args if rsync believes they would escape the module hierarchy. -The default for \(dq\&use chroot\(dq\& is true, and is the safer choice (especially -if the module is not read\-only). -.IP -When this parameter is enabled, the \(dq\&numeric\-ids\(dq\& option will also default to -being enabled (disabling name lookups). See below for what a chroot needs in -order for name lookups to succeed. -.IP -If you copy library resources into the module\(cq\&s chroot area, you -should protect them through your OS\(cq\&s normal user/group or ACL settings (to -prevent the rsync module\(cq\&s user from being able to change them), and then -hide them from the user\(cq\&s view via \(dq\&exclude\(dq\& (see how in the discussion of -that parameter). At that point it will be safe to enable the mapping of users -and groups by name using the \(dq\&numeric ids\(dq\& daemon parameter (see below). -.IP +that is set by rsync when the user connects. For example, this would use +the authorizing user's name in the path: +.RS 4 +.IP +.nf +path = /home/%RSYNC_USER_NAME% +.fi +.RE +.IP +It is fine if the path includes internal spaces\ \-\- they will be retained +verbatim (which means that you shouldn't try to escape them). If your +final directory has a trailing space (and this is somehow not something you +wish to fix), append a trailing slash to the path to avoid losing the +trailing whitespace. +.IP "\fBuse\ chroot\fP" +If "use chroot" is true, the rsync daemon will chroot to the "path" before +starting the file transfer with the client. This has the advantage of +extra protection against possible implementation security holes, but it has +the disadvantages of requiring super-user privileges, of not being able to +follow symbolic links that are either absolute or outside of the new root +path, and of complicating the preservation of users and groups by name (see +below). +.IP +As an additional safety feature, you can specify a dot-dir in the module's +"path" to indicate the point where the chroot should occur. This allows +rsync to run in a chroot with a non-"/" path for the top of the transfer +hierarchy. Doing this guards against unintended library loading (since +those absolute paths will not be inside the transfer hierarchy unless you +have used an unwise pathname), and lets you setup libraries for the chroot +that are outside of the transfer. For example, specifying +"/var/rsync/./module1" will chroot to the "/var/rsync" directory and set +the inside-chroot path to "/module1". If you had omitted the dot-dir, the +chroot would have used the whole path, and the inside-chroot path would +have been "/". +.IP +When both "use chroot" and "daemon chroot" are false, OR the inside-chroot +path of "use chroot" is not "/", rsync will: (1) munge symlinks by default +for security reasons (see "munge symlinks" for a way to turn this off, but +only if you trust your users), (2) substitute leading slashes in absolute +paths with the module's path (so that options such as \fB\-\-backup-dir\fP, +\fB\-\-compare-dest\fP, etc. interpret an absolute path as rooted in the module's +"path" dir), and (3) trim ".." path elements from args if rsync believes +they would escape the module hierarchy. The default for "use chroot" is +true, and is the safer choice (especially if the module is not read-only). +.IP +When this parameter is enabled, the "numeric-ids" option will also default +to being enabled (disabling name lookups). See below for what a chroot +needs in order for name lookups to succeed. +.IP +If you copy library resources into the module's chroot area, you should +protect them through your OS's normal user/group or ACL settings (to +prevent the rsync module's user from being able to change them), and then +hide them from the user's view via "exclude" (see how in the discussion of +that parameter). At that point it will be safe to enable the mapping of +users and groups by name using the "numeric ids" daemon parameter (see +below). +.IP Note also that you are free to setup custom user/group information in the chroot area that is different from your normal system. For example, you could abbreviate the list of users and groups. -.IP -.IP "\fBdaemon chroot\fP" -This parameter specifies a path to which the daemon will -chroot before beginning communication with clients. Module paths (and any \(dq\&use -chroot\(dq\& settings) will then be related to this one. This lets you choose if you -want the whole daemon to be chrooted (with this setting), just the transfers to -be chrooted (with \(dq\&use chroot\(dq\&), or both. Keep in mind that the \(dq\&daemon chroot\(dq\& -area may need various OS/lib/etc files installed to allow the daemon to function. -By default the daemon runs without any chrooting. -.IP -.IP "\fBnumeric ids\fP" -Enabling this parameter disables the mapping -of users and groups by name for the current daemon module. This prevents -the daemon from trying to load any user/group\-related files or libraries. -This enabling makes the transfer behave as if the client had passed -the \fB\-\-numeric\-ids\fP command\-line option. By default, this parameter is -enabled for chroot modules and disabled for non\-chroot modules. -Also keep in mind that uid/gid preservation requires the module to be -running as root (see \(dq\&uid\(dq\&) or for \(dq\&fake super\(dq\& to be configured. -.IP -A chroot\-enabled module should not have this parameter enabled unless you\(cq\&ve -taken steps to ensure that the module has the necessary resources it needs -to translate names, and that it is not possible for a user to change those -resources. That includes being the code being able to call functions like -\f(CWgetpwuid()\fP -, -\f(CWgetgrgid()\fP -, -\f(CWgetpwname()\fP -, and -\f(CWgetgrnam()\fP -\&. +.IP "\fBdaemon\ chroot\fP" +This parameter specifies a path to which the daemon will chroot before +beginning communication with clients. Module paths (and any "use chroot" +settings) will then be related to this one. This lets you choose if you +want the whole daemon to be chrooted (with this setting), just the +transfers to be chrooted (with "use chroot"), or both. Keep in mind that +the "daemon chroot" area may need various OS/lib/etc files installed to +allow the daemon to function. By default the daemon runs without any +chrooting. +.IP "\fBproxy\ protocol\fP" +When this parameter is enabled, all incoming connections must start with a +V1 or V2 proxy protocol header. If the header is not found, the connection +is closed. +.IP +Setting this to \fBtrue\fP requires a proxy server to forward source IP +information to rsync, allowing you to log proper IP/host info and make use +of client-oriented IP restrictions. The default of \fBfalse\fP means that the +IP information comes directly from the socket's metadata. If rsync is not +behind a proxy, this should be disabled. +.IP +\fICAUTION\fP: using this option can be dangerous if you do not ensure that +only the proxy is allowed to connect to the rsync port. If any non-proxied +connections are allowed through, the client will be able to use a modified +rsync to spoof any remote IP address that they desire. You can lock this +down using something like iptables \fB\-uid-owner\ root\fP rules (for strict +localhost access), various firewall rules, or you can require password +authorization so that any spoofing by users will not grant extra access. +.IP +This setting is global. If you need some modules to require this and not +others, then you will need to setup multiple rsync daemon processes on +different ports. +.IP "\fBnumeric\ ids\fP" +Enabling this parameter disables the mapping of users and groups by name +for the current daemon module. This prevents the daemon from trying to +load any user/group-related files or libraries. This enabling makes the +transfer behave as if the client had passed the \fB\-\-numeric-ids\fP +command-line option. By default, this parameter is enabled for chroot +modules and disabled for non-chroot modules. Also keep in mind that +uid/gid preservation requires the module to be running as root (see "uid") +or for "fake super" to be configured. +.IP +A chroot-enabled module should not have this parameter enabled unless +you've taken steps to ensure that the module has the necessary resources it +needs to translate names, and that it is not possible for a user to change +those resources. That includes being the code being able to call functions +like \fBgetpwuid()\fP, \fBgetgrgid()\fP, \fBgetpwname()\fP, and \fBgetgrnam()\fP. You should test what libraries and config files are required for your OS and get those setup before starting to test name mapping in rsync. -.IP -.IP "\fBmunge symlinks\fP" -This parameter tells rsync to modify -all symlinks in the same way as the (non\-daemon\-affecting) -\fB\-\-munge\-links\fP command\-line option (using a method described below). -This should help protect your files from user trickery when -your daemon module is writable. The default is disabled when \(dq\&use chroot\(dq\& -is on with an inside\-chroot path of \(dq\&/\(dq\&, OR if \(dq\&daemon chroot\(dq\& is on, -otherwise it is enabled. -.IP -If you disable this parameter on a daemon that is not read\-only, there -are tricks that a user can play with uploaded symlinks to access -daemon\-excluded items (if your module has any), and, if \(dq\&use chroot\(dq\& -is off, rsync can even be tricked into showing or changing data that -is outside the module\(cq\&s path (as access\-permissions allow). -.IP -The way rsync disables the use of symlinks is to prefix each one with -the string \(dq\&/rsyncd\-munged/\(dq\&. This prevents the links from being used -as long as that directory does not exist. When this parameter is enabled, -rsync will refuse to run if that path is a directory or a symlink to -a directory. When using the \(dq\&munge symlinks\(dq\& parameter in a chroot area -that has an inside\-chroot path of \(dq\&/\(dq\&, you should add \(dq\&/rsyncd\-munged/\(dq\& -to the exclude setting for the module so that -a user can\(cq\&t try to create it. -.IP -Note: rsync makes no attempt to verify that any pre\-existing symlinks in -the module\(cq\&s hierarchy are as safe as you want them to be (unless, of +.IP "\fBmunge\ symlinks\fP" +This parameter tells rsync to modify all symlinks in the same way as the +(non-daemon-affecting) \fB\-\-munge-links\fP command-line option (using a method +described below). This should help protect your files from user trickery +when your daemon module is writable. The default is disabled when +"use chroot" is on with an inside-chroot path of "/", OR if "daemon chroot" +is on, otherwise it is enabled. +.IP +If you disable this parameter on a daemon that is not read-only, there are +tricks that a user can play with uploaded symlinks to access +daemon-excluded items (if your module has any), and, if "use chroot" is +off, rsync can even be tricked into showing or changing data that is +outside the module's path (as access-permissions allow). +.IP +The way rsync disables the use of symlinks is to prefix each one with the +string "/rsyncd-munged/". This prevents the links from being used as long +as that directory does not exist. When this parameter is enabled, rsync +will refuse to run if that path is a directory or a symlink to a directory. +When using the "munge symlinks" parameter in a chroot area that has an +inside-chroot path of "/", you should add "/rsyncd-munged/" to the exclude +setting for the module so that a user can't try to create it. +.IP +Note: rsync makes no attempt to verify that any pre-existing symlinks in +the module's hierarchy are as safe as you want them to be (unless, of course, it just copied in the whole hierarchy). If you setup an rsync daemon on a new area or locally add symlinks, you can manually protect your -symlinks from being abused by prefixing \(dq\&/rsyncd\-munged/\(dq\& to the start of -every symlink\(cq\&s value. There is a perl script in the support directory -of the source code named \(dq\&munge\-symlinks\(dq\& that can be used to add or remove +symlinks from being abused by prefixing "/rsyncd-munged/" to the start of +every symlink's value. There is a perl script in the support directory of +the source code named "munge-symlinks" that can be used to add or remove this prefix from your symlinks. -.IP -When this parameter is disabled on a writable module and \(dq\&use chroot\(dq\& is off -(or the inside\-chroot path is not \(dq\&/\(dq\&), -incoming symlinks will be modified to drop a leading slash and to remove \(dq\&..\(dq\& -path elements that rsync believes will allow a symlink to escape the module\(cq\&s -hierarchy. There are tricky ways to work around this, though, so you had -better trust your users if you choose this combination of parameters. -.IP +.IP +When this parameter is disabled on a writable module and "use chroot" is +off (or the inside-chroot path is not "/"), incoming symlinks will be +modified to drop a leading slash and to remove ".." path elements that +rsync believes will allow a symlink to escape the module's hierarchy. +There are tricky ways to work around this, though, so you had better trust +your users if you choose this combination of parameters. .IP "\fBcharset\fP" -This specifies the name of the character set in which the -module\(cq\&s filenames are stored. If the client uses an \fB\-\-iconv\fP option, -the daemon will use the value of the \(dq\&charset\(dq\& parameter regardless of the -character set the client actually passed. This allows the daemon to -support charset conversion in a chroot module without extra files in the -chroot area, and also ensures that name\-translation is done in a consistent -manner. If the \(dq\&charset\(dq\& parameter is not set, the \fB\-\-iconv\fP option is -refused, just as if \(dq\&iconv\(dq\& had been specified via \(dq\&refuse options\(dq\&. -.IP -If you wish to force users to always use \fB\-\-iconv\fP for a particular -module, add \(dq\&no\-iconv\(dq\& to the \(dq\&refuse options\(dq\& parameter. Keep in mind -that this will restrict access to your module to very new rsync clients. -.IP -.IP "\fBmax connections\fP" -This parameter allows you to -specify the maximum number of simultaneous connections you will allow. -Any clients connecting when the maximum has been reached will receive a -message telling them to try later. The default is 0, which means no limit. -A negative value disables the module. -See also the \(dq\&lock file\(dq\& parameter. -.IP -.IP "\fBlog file\fP" -When the \(dq\&log file\(dq\& parameter is set to a non\-empty -string, the rsync daemon will log messages to the indicated file rather -than using syslog. This is particularly useful on systems (such as AIX) -where -\f(CWsyslog()\fP -doesn\(cq\&t work for chrooted programs. The file is -opened before -\f(CWchroot()\fP -is called, allowing it to be placed outside -the transfer. If this value is set on a per\-module basis instead of -globally, the global log will still contain any authorization failures -or config\-file error messages. -.IP -If the daemon fails to open the specified file, it will fall back to -using syslog and output an error about the failure. (Note that the -failure to open the specified log file used to be a fatal error.) -.IP -This setting can be overridden by using the \fB\-\-log\-file=FILE\fP or -\fB\-\-dparam=logfile=FILE\fP command\-line options. The former overrides -all the log\-file parameters of the daemon and all module settings. -The latter sets the daemon\(cq\&s log file and the default for all the -modules, which still allows modules to override the default setting. -.IP -.IP "\fBsyslog facility\fP" -This parameter allows you to -specify the syslog facility name to use when logging messages from the -rsync daemon. You may use any standard syslog facility name which is -defined on your system. Common names are auth, authpriv, cron, daemon, -ftp, kern, lpr, mail, news, security, syslog, user, uucp, local0, -local1, local2, local3, local4, local5, local6 and local7. The default -is daemon. This setting has no effect if the \(dq\&log file\(dq\& setting is a -non\-empty string (either set in the per\-modules settings, or inherited -from the global settings). -.IP -.IP "\fBsyslog tag\fP" -This parameter allows you to specify the syslog -tag to use when logging messages from the rsync daemon. The default is -\(dq\&rsyncd\(dq\&. This setting has no effect if the \(dq\&log file\(dq\& setting is a -non\-empty string (either set in the per\-modules settings, or inherited -from the global settings). -.IP -For example, if you wanted each authenticated user\(cq\&s name to be -included in the syslog tag, you could do something like this: -.IP -.nf - syslog tag = rsyncd.%RSYNC_USER_NAME% -.fi - -.IP -.IP "\fBmax verbosity\fP" -This parameter allows you to control -the maximum amount of verbose information that you\(cq\&ll allow the daemon to -generate (since the information goes into the log file). The default is 1, -which allows the client to request one level of verbosity. -.IP -This also affects the user\(cq\&s ability to request higher levels of \fB\-\-info\fP and -\fB\-\-debug\fP logging. If the max value is 2, then no info and/or debug value -that is higher than what would be set by \fB\-vv\fP will be honored by the daemon -in its logging. To see how high of a verbosity level you need to accept for a -particular info/debug level, refer to \(dq\&rsync \-\-info=help\(dq\& and \(dq\&rsync \-\-debug=help\(dq\&. -For instance, it takes max\-verbosity 4 to be able to output debug TIME2 and FLIST3. -.IP -.IP "\fBlock file\fP" -This parameter specifies the file to use to -support the \(dq\&max connections\(dq\& parameter. The rsync daemon uses record -locking on this file to ensure that the max connections limit is not -exceeded for the modules sharing the lock file. -The default is \f(CW/var/run/rsyncd.lock\fP. -.IP -.IP "\fBread only\fP" -This parameter determines whether clients -will be able to upload files or not. If \(dq\&read only\(dq\& is true then any -attempted uploads will fail. If \(dq\&read only\(dq\& is false then uploads will -be possible if file permissions on the daemon side allow them. The default -is for all modules to be read only. -.IP -Note that \(dq\&auth users\(dq\& can override this setting on a per\-user basis. -.IP -.IP "\fBwrite only\fP" -This parameter determines whether clients -will be able to download files or not. If \(dq\&write only\(dq\& is true then any -attempted downloads will fail. If \(dq\&write only\(dq\& is false then downloads -will be possible if file permissions on the daemon side allow them. The -default is for this parameter to be disabled. -.IP -Helpful hint: you probably want to specify \(dq\&refuse options = delete\(dq\& for a -write\-only module. -.IP +This specifies the name of the character set in which the module's +filenames are stored. If the client uses an \fB\-\-iconv\fP option, the daemon +will use the value of the "charset" parameter regardless of the character +set the client actually passed. This allows the daemon to support charset +conversion in a chroot module without extra files in the chroot area, and +also ensures that name-translation is done in a consistent manner. If the +"charset" parameter is not set, the \fB\-\-iconv\fP option is refused, just as if +"iconv" had been specified via "refuse options". +.IP +If you wish to force users to always use \fB\-\-iconv\fP for a particular module, +add "no-iconv" to the "refuse options" parameter. Keep in mind that this +will restrict access to your module to very new rsync clients. +.IP "\fBmax\ connections\fP" +This parameter allows you to specify the maximum number of simultaneous +connections you will allow. Any clients connecting when the maximum has +been reached will receive a message telling them to try later. The default +is 0, which means no limit. A negative value disables the module. See +also the "lock file" parameter. +.IP "\fBlog\ file\fP" +When the "log file" parameter is set to a non-empty string, the rsync +daemon will log messages to the indicated file rather than using syslog. +This is particularly useful on systems (such as AIX) where \fBsyslog()\fP +doesn't work for chrooted programs. The file is opened before \fBchroot()\fP +is called, allowing it to be placed outside the transfer. If this value is +set on a per-module basis instead of globally, the global log will still +contain any authorization failures or config-file error messages. +.IP +If the daemon fails to open the specified file, it will fall back to using +syslog and output an error about the failure. (Note that the failure to +open the specified log file used to be a fatal error.) +.IP +This setting can be overridden by using the \fB\-\-log-file=FILE\fP or +\fB\-\-dparam=logfile=FILE\fP command-line options. The former overrides all the +log-file parameters of the daemon and all module settings. The latter sets +the daemon's log file and the default for all the modules, which still +allows modules to override the default setting. +.IP "\fBsyslog\ facility\fP" +This parameter allows you to specify the syslog facility name to use when +logging messages from the rsync daemon. You may use any standard syslog +facility name which is defined on your system. Common names are auth, +authpriv, cron, daemon, ftp, kern, lpr, mail, news, security, syslog, user, +uucp, local0, local1, local2, local3, local4, local5, local6 and local7. +The default is daemon. This setting has no effect if the "log file" +setting is a non-empty string (either set in the per-modules settings, or +inherited from the global settings). +.IP "\fBsyslog\ tag\fP" +This parameter allows you to specify the syslog tag to use when logging +messages from the rsync daemon. The default is "rsyncd". This setting has +no effect if the "log file" setting is a non-empty string (either set in +the per-modules settings, or inherited from the global settings). +.IP +For example, if you wanted each authenticated user's name to be included in +the syslog tag, you could do something like this: +.RS 4 +.IP +.nf +syslog tag = rsyncd.%RSYNC_USER_NAME% +.fi +.RE +.IP "\fBmax\ verbosity\fP" +This parameter allows you to control the maximum amount of verbose +information that you'll allow the daemon to generate (since the information +goes into the log file). The default is 1, which allows the client to +request one level of verbosity. +.IP +This also affects the user's ability to request higher levels of \fB\-\-info\fP +and \fB\-\-debug\fP logging. If the max value is 2, then no info and/or debug +value that is higher than what would be set by \fB\-vv\fP will be honored by the +daemon in its logging. To see how high of a verbosity level you need to +accept for a particular info/debug level, refer to \fBrsync\ \-\-info=help\fP and +\fBrsync\ \-\-debug=help\fP. For instance, it takes max-verbosity 4 to be able to +output debug TIME2 and FLIST3. +.IP "\fBlock\ file\fP" +This parameter specifies the file to use to support the "max connections" +parameter. The rsync daemon uses record locking on this file to ensure that +the max connections limit is not exceeded for the modules sharing the lock +file. The default is \fB/var/run/rsyncd.lock\fP. +.IP "\fBread\ only\fP" +This parameter determines whether clients will be able to upload files or +not. If "read only" is true then any attempted uploads will fail. If +"read only" is false then uploads will be possible if file permissions on +the daemon side allow them. The default is for all modules to be read only. +.IP +Note that "auth users" can override this setting on a per-user basis. +.IP "\fBwrite\ only\fP" +This parameter determines whether clients will be able to download files or +not. If "write only" is true then any attempted downloads will fail. If +"write only" is false then downloads will be possible if file permissions +on the daemon side allow them. The default is for this parameter to be +disabled. +.IP +Helpful hint: you probably want to specify "refuse options = delete" for a +write-only module. .IP "\fBlist\fP" -This parameter determines whether this module is -listed when the client asks for a listing of available modules. In addition, -if this is false, the daemon will pretend the module does not exist -when a client denied by \(dq\&hosts allow\(dq\& or \(dq\&hosts deny\(dq\& attempts to access it. -Realize that if \(dq\&reverse lookup\(dq\& is disabled globally but enabled for the -module, the resulting reverse lookup to a potentially client\-controlled DNS -server may still reveal to the client that it hit an existing module. -The default is for modules to be listable. -.IP +This parameter determines whether this module is listed when the client +asks for a listing of available modules. In addition, if this is false, +the daemon will pretend the module does not exist when a client denied by +"hosts allow" or "hosts deny" attempts to access it. Realize that if +"reverse lookup" is disabled globally but enabled for the module, the +resulting reverse lookup to a potentially client-controlled DNS server may +still reveal to the client that it hit an existing module. The default is +for modules to be listable. .IP "\fBuid\fP" -This parameter specifies the user name or user ID that -file transfers to and from that module should take place as when the daemon -was run as root. In combination with the \(dq\&gid\(dq\& parameter this determines what -file permissions are available. The default when run by a super\-user is to -switch to the system\(cq\&s \(dq\&nobody\(dq\& user. The default for a non\-super\-user is to -not try to change the user. See also the \(dq\&gid\(dq\& parameter. -.IP -The RSYNC_USER_NAME environment variable may be used to request that rsync run -as the authorizing user. For example, if you want a rsync to run as the same -user that was received for the rsync authentication, this setup is useful: -.IP -.nf - uid = %RSYNC_USER_NAME% - gid = * -.fi - -.IP +This parameter specifies the user name or user ID that file transfers to +and from that module should take place as when the daemon was run as root. +In combination with the "gid" parameter this determines what file +permissions are available. The default when run by a super-user is to +switch to the system's "nobody" user. The default for a non-super-user is +to not try to change the user. See also the "gid" parameter. +.IP +The RSYNC_USER_NAME environment variable may be used to request that rsync +run as the authorizing user. For example, if you want a rsync to run as +the same user that was received for the rsync authentication, this setup is +useful: +.RS 4 +.IP +.nf +uid = %RSYNC_USER_NAME% +gid = * +.fi +.RE .IP "\fBgid\fP" -This parameter specifies one or more group names/IDs that will be -used when accessing the module. The first one will be the default group, and -any extra ones be set as supplemental groups. You may also specify a \(dq\&*\(dq\& as -the first gid in the list, which will be replaced by all the normal groups for -the transfer\(cq\&s user (see \(dq\&uid\(dq\&). The default when run by a super\-user is to -switch to your OS\(cq\&s \(dq\&nobody\(dq\& (or perhaps \(dq\&nogroup\(dq\&) group with no other -supplementary groups. The default for a non\-super\-user is to not change any -group attributes (and indeed, your OS may not allow a non\-super\-user to try to -change their group settings). -.IP -.IP "\fBdaemon uid\fP" -This parameter specifies a uid under which the daemon will -run. The daemon usually runs as user root, and when this is left unset the user -is left unchanged. See also the \(dq\&uid\(dq\& parameter. -.IP -.IP "\fBdaemon gid\fP" -This parameter specifies a gid under which the daemon will -run. The daemon usually runs as group root, and when this is left unset, the -group is left unchanged. See also the \(dq\&gid\(dq\& parameter. -.IP -.IP "\fBfake super\fP" -Setting \(dq\&fake super = yes\(dq\& for a module causes the -daemon side to behave as if the \fB\-\-fake\-super\fP command\-line option had -been specified. This allows the full attributes of a file to be stored -without having to have the daemon actually running as root. -.IP +This parameter specifies one or more group names/IDs that will be used when +accessing the module. The first one will be the default group, and any +extra ones be set as supplemental groups. You may also specify a "\fB*\fP" as +the first gid in the list, which will be replaced by all the normal groups +for the transfer's user (see "uid"). The default when run by a super-user +is to switch to your OS's "nobody" (or perhaps "nogroup") group with no +other supplementary groups. The default for a non-super-user is to not +change any group attributes (and indeed, your OS may not allow a +non-super-user to try to change their group settings). +.IP +The specified list is normally split into tokens based on spaces and +commas. However, if the list starts with a comma, then the list is only +split on commas, which allows a group name to contain a space. In either +case any leading and/or trailing whitespace is removed from the tokens and +empty tokens are ignored. +.IP "\fBdaemon\ uid\fP" +This parameter specifies a uid under which the daemon will run. The daemon +usually runs as user root, and when this is left unset the user is left +unchanged. See also the "uid" parameter. +.IP "\fBdaemon\ gid\fP" +This parameter specifies a gid under which the daemon will run. The daemon +usually runs as group root, and when this is left unset, the group is left +unchanged. See also the "gid" parameter. +.IP "\fBfake\ super\fP" +Setting "fake super = yes" for a module causes the daemon side to behave as +if the \fB\-\-fake-super\fP command-line option had been specified. This allows +the full attributes of a file to be stored without having to have the +daemon actually running as root. .IP "\fBfilter\fP" -The daemon has its own filter chain that determines what files -it will let the client access. This chain is not sent to the client and is -independent of any filters the client may have specified. Files excluded by -the daemon filter chain (\fBdaemon\-excluded\fP files) are treated as non\-existent -if the client tries to pull them, are skipped with an error message if the -client tries to push them (triggering exit code 23), and are never deleted from -the module. You can use daemon filters to prevent clients from downloading or -tampering with private administrative files, such as files you may add to -support uid/gid name translations. -.IP -The daemon filter chain is built from the \(dq\&filter\(dq\&, \(dq\&include from\(dq\&, \(dq\&include\(dq\&, -\(dq\&exclude from\(dq\&, and \(dq\&exclude\(dq\& parameters, in that order of priority. Anchored -patterns are anchored at the root of the module. To prevent access to an -entire subtree, for example, \(dq\&/secret\(dq\&, you \fImust\fP exclude everything in the -subtree; the easiest way to do this is with a triple\-star pattern like -\(dq\&/secret/***\(dq\&. -.IP -The \(dq\&filter\(dq\& parameter takes a space\-separated list of daemon filter rules, -though it is smart enough to know not to split a token at an internal space in -a rule (e.g. \(dq\&\- /foo \- /bar\(dq\& is parsed as two rules). You may specify one or -more merge\-file rules using the normal syntax. Only one \(dq\&filter\(dq\& parameter can -apply to a given module in the config file, so put all the rules you want in a -single parameter. Note that per\-directory merge\-file rules do not provide as -much protection as global rules, but they can be used to make \fB\-\-delete\fP work -better during a client download operation if the per\-dir merge files are -included in the transfer and the client requests that they be used. -.IP +The daemon has its own filter chain that determines what files it will let +the client access. This chain is not sent to the client and is independent +of any filters the client may have specified. Files excluded by the daemon +filter chain (\fBdaemon-excluded\fP files) are treated as non-existent if the +client tries to pull them, are skipped with an error message if the client +tries to push them (triggering exit code 23), and are never deleted from +the module. You can use daemon filters to prevent clients from downloading +or tampering with private administrative files, such as files you may add +to support uid/gid name translations. +.IP +The daemon filter chain is built from the "filter", "include from", +"include", "exclude from", and "exclude" parameters, in that order of +priority. Anchored patterns are anchored at the root of the module. To +prevent access to an entire subtree, for example, "\fB/secret\fP", you \fBmust\fP +exclude everything in the subtree; the easiest way to do this is with a +triple-star pattern like "\fB/secret/***\fP". +.IP +The "filter" parameter takes a space-separated list of daemon filter rules, +though it is smart enough to know not to split a token at an internal space +in a rule (e.g. "\fB\-\ /foo\ \-\ /bar\fP" is parsed as two rules). You may specify +one or more merge-file rules using the normal syntax. Only one "filter" +parameter can apply to a given module in the config file, so put all the +rules you want in a single parameter. Note that per-directory merge-file +rules do not provide as much protection as global rules, but they can be +used to make \fB\-\-delete\fP work better during a client download operation if +the per-dir merge files are included in the transfer and the client +requests that they be used. .IP "\fBexclude\fP" -This parameter takes a space\-separated list of daemon -exclude patterns. As with the client \fB\-\-exclude\fP option, patterns can be -qualified with \(dq\&\- \(dq\& or \(dq\&+ \(dq\& to explicitly indicate exclude/include. Only one -\(dq\&exclude\(dq\& parameter can apply to a given module. See the \(dq\&filter\(dq\& parameter -for a description of how excluded files affect the daemon. -.IP +This parameter takes a space-separated list of daemon exclude patterns. As +with the client \fB\-\-exclude\fP option, patterns can be qualified with "\fB\-\fP" or +"\fB+\fP" to explicitly indicate exclude/include. Only one "exclude" parameter +can apply to a given module. See the "filter" parameter for a description +of how excluded files affect the daemon. .IP "\fBinclude\fP" -Use an \(dq\&include\(dq\& to override the effects of the \(dq\&exclude\(dq\& -parameter. Only one \(dq\&include\(dq\& parameter can apply to a given module. See the -\(dq\&filter\(dq\& parameter for a description of how excluded files affect the daemon. -.IP -.IP "\fBexclude from\fP" -This parameter specifies the name of a file -on the daemon that contains daemon exclude patterns, one per line. Only one -\(dq\&exclude from\(dq\& parameter can apply to a given module; if you have multiple -exclude\-from files, you can specify them as a merge file in the \(dq\&filter\(dq\& -parameter. See the \(dq\&filter\(dq\& parameter for a description of how excluded files -affect the daemon. -.IP -.IP "\fBinclude from\fP" -Analogue of \(dq\&exclude from\(dq\& for a file of daemon include -patterns. Only one \(dq\&include from\(dq\& parameter can apply to a given module. See -the \(dq\&filter\(dq\& parameter for a description of how excluded files affect the +Use an "include" to override the effects of the "exclude" parameter. Only +one "include" parameter can apply to a given module. See the "filter" +parameter for a description of how excluded files affect the daemon. +.IP "\fBexclude\ from\fP" +This parameter specifies the name of a file on the daemon that contains +daemon exclude patterns, one per line. Only one "exclude from" parameter +can apply to a given module; if you have multiple exclude-from files, you +can specify them as a merge file in the "filter" parameter. See the +"filter" parameter for a description of how excluded files affect the daemon. -.IP -.IP "\fBincoming chmod\fP" -This parameter allows you to specify a set of -comma\-separated chmod strings that will affect the permissions of all -incoming files (files that are being received by the daemon). These -changes happen after all other permission calculations, and this will -even override destination\-default and/or existing permissions when the -client does not specify \fB\-\-perms\fP. -See the description of the \fB\-\-chmod\fP rsync option and the \fBchmod\fP(1) -manpage for information on the format of this string. -.IP -.IP "\fBoutgoing chmod\fP" -This parameter allows you to specify a set of -comma\-separated chmod strings that will affect the permissions of all -outgoing files (files that are being sent out from the daemon). These -changes happen first, making the sent permissions appear to be different -than those stored in the filesystem itself. For instance, you could -disable group write permissions on the server while having it appear to -be on to the clients. +.IP "\fBinclude\ from\fP" +Analogue of "exclude from" for a file of daemon include patterns. Only one +"include from" parameter can apply to a given module. See the "filter" +parameter for a description of how excluded files affect the daemon. +.IP "\fBincoming\ chmod\fP" +This parameter allows you to specify a set of comma-separated chmod strings +that will affect the permissions of all incoming files (files that are +being received by the daemon). These changes happen after all other +permission calculations, and this will even override destination-default +and/or existing permissions when the client does not specify \fB\-\-perms\fP. See the description of the \fB\-\-chmod\fP rsync option and the \fBchmod\fP(1) manpage for information on the format of this string. -.IP -.IP "\fBauth users\fP" -This parameter specifies a comma and/or space\-separated -list of authorization rules. In its simplest form, you list the usernames -that will be allowed to connect to -this module. The usernames do not need to exist on the local -system. The rules may contain shell wildcard characters that will be matched -against the username provided by the client for authentication. If -\(dq\&auth users\(dq\& is set then the client will be challenged to supply a -username and password to connect to the module. A challenge response -authentication protocol is used for this exchange. The plain text +.IP "\fBoutgoing\ chmod\fP" +This parameter allows you to specify a set of comma-separated chmod strings +that will affect the permissions of all outgoing files (files that are +being sent out from the daemon). These changes happen first, making the +sent permissions appear to be different than those stored in the filesystem +itself. For instance, you could disable group write permissions on the +server while having it appear to be on to the clients. See the description +of the \fB\-\-chmod\fP rsync option and the \fBchmod\fP(1) manpage for information +on the format of this string. +.IP "\fBauth\ users\fP" +This parameter specifies a comma and/or space-separated list of +authorization rules. In its simplest form, you list the usernames that +will be allowed to connect to this module. The usernames do not need to +exist on the local system. The rules may contain shell wildcard characters +that will be matched against the username provided by the client for +authentication. If "auth users" is set then the client will be challenged +to supply a username and password to connect to the module. A challenge +response authentication protocol is used for this exchange. The plain text usernames and passwords are stored in the file specified by the -\(dq\&secrets file\(dq\& parameter. The default is for all users to be able to -connect without a password (this is called \(dq\&anonymous rsync\(dq\&). -.IP -In addition to username matching, you can specify groupname matching via a \(cq\&@\(cq\& -prefix. When using groupname matching, the authenticating username must be a -real user on the system, or it will be assumed to be a member of no groups. -For example, specifying \(dq\&@rsync\(dq\& will match the authenticating user if the -named user is a member of the rsync group. -.IP -Finally, options may be specified after a colon (:). The options allow you to -\(dq\&deny\(dq\& a user or a group, set the access to \(dq\&ro\(dq\& (read\-only), or set the access -to \(dq\&rw\(dq\& (read/write). Setting an auth\-rule\-specific ro/rw setting overrides -the module\(cq\&s \(dq\&read only\(dq\& setting. -.IP -Be sure to put the rules in the order you want them to be matched, because the -checking stops at the first matching user or group, and that is the only auth -that is checked. For example: -.IP -.nf - auth users = joe:deny @guest:deny admin:rw @rsync:ro susan joe sam -.fi - -.IP +"secrets file" parameter. The default is for all users to be able to +connect without a password (this is called "anonymous rsync"). +.IP +In addition to username matching, you can specify groupname matching via a +\&'@' prefix. When using groupname matching, the authenticating username +must be a real user on the system, or it will be assumed to be a member of +no groups. For example, specifying "@rsync" will match the authenticating +user if the named user is a member of the rsync group. +.IP +Finally, options may be specified after a colon (:). The options allow you +to "deny" a user or a group, set the access to "ro" (read-only), or set the +access to "rw" (read/write). Setting an auth-rule-specific ro/rw setting +overrides the module's "read only" setting. +.IP +Be sure to put the rules in the order you want them to be matched, because +the checking stops at the first matching user or group, and that is the +only auth that is checked. For example: +.RS 4 +.IP +.nf +auth users = joe:deny @guest:deny admin:rw @rsync:ro susan joe sam +.fi +.RE +.IP In the above rule, user joe will be denied access no matter what. Any user -that is in the group \(dq\&guest\(dq\& is also denied access. The user \(dq\&admin\(dq\& gets -access in read/write mode, but only if the admin user is not in group \(dq\&guest\(dq\& -(because the admin user\-matching rule would never be reached if the user is in -group \(dq\&guest\(dq\&). Any other user who is in group \(dq\&rsync\(dq\& will get read\-only -access. Finally, users susan, joe, and sam get the ro/rw setting of the -module, but only if the user didn\(cq\&t match an earlier group\-matching rule. -.IP -If you need to specify a user or group name with a space in it, start your list -with a comma to indicate that the list should only be split on commas (though -leading and trailing whitespace will also be removed, and empty entries are -just ignored). For example: -.IP -.nf - auth users = , joe:deny, @Some Group:deny, admin:rw, @RO Group:ro -.fi - -.IP -See the description of the secrets file for how you can have per\-user passwords -as well as per\-group passwords. It also explains how a user can authenticate -using their user password or (when applicable) a group password, depending on -what rule is being authenticated. -.IP -See also the section entitled \(dq\&USING RSYNC\-DAEMON FEATURES VIA A REMOTE -SHELL CONNECTION\(dq\& in \fBrsync\fP(1) for information on how handle an -rsyncd.conf\-level username that differs from the remote\-shell\-level +that is in the group "guest" is also denied access. The user "admin" gets +access in read/write mode, but only if the admin user is not in group +"guest" (because the admin user-matching rule would never be reached if the +user is in group "guest"). Any other user who is in group "rsync" will get +read-only access. Finally, users susan, joe, and sam get the ro/rw setting +of the module, but only if the user didn't match an earlier group-matching +rule. +.IP +If you need to specify a user or group name with a space in it, start your +list with a comma to indicate that the list should only be split on commas +(though leading and trailing whitespace will also be removed, and empty +entries are just ignored). For example: +.RS 4 +.IP +.nf +auth users = , joe:deny, @Some Group:deny, admin:rw, @RO Group:ro +.fi +.RE +.IP +See the description of the secrets file for how you can have per-user +passwords as well as per-group passwords. It also explains how a user can +authenticate using their user password or (when applicable) a group +password, depending on what rule is being authenticated. +.IP +See also the section entitled "USING RSYNC-DAEMON FEATURES VIA A REMOTE +SHELL CONNECTION" in \fBrsync\fP(1) for information on how handle an +rsyncd.conf-level username that differs from the remote-shell-level username when using a remote shell to connect to an rsync daemon. -.IP -.IP "\fBsecrets file\fP" -This parameter specifies the name of a file that contains -the username:password and/or @groupname:password pairs used for authenticating -this module. This file is only consulted if the \(dq\&auth users\(dq\& parameter is -specified. The file is line\-based and contains one name:password pair per +.IP "\fBsecrets\ file\fP" +This parameter specifies the name of a file that contains the +username:password and/or @groupname:password pairs used for authenticating +this module. This file is only consulted if the "auth users" parameter is +specified. The file is line-based and contains one name:password pair per line. Any line has a hash (#) as the very first character on the line is -considered a comment and is skipped. The passwords can contain any characters -but be warned that many operating systems limit the length of passwords that -can be typed at the client end, so you may find that passwords longer than 8 -characters don\(cq\&t work. -.IP -The use of group\-specific lines are only relevant when the module is being -authorized using a matching \(dq\&@groupname\(dq\& rule. When that happens, the user -can be authorized via either their \(dq\&username:password\(dq\& line or the -\(dq\&@groupname:password\(dq\& line for the group that triggered the authentication. -.IP +considered a comment and is skipped. The passwords can contain any +characters but be warned that many operating systems limit the length of +passwords that can be typed at the client end, so you may find that +passwords longer than 8 characters don't work. +.IP +The use of group-specific lines are only relevant when the module is being +authorized using a matching "@groupname" rule. When that happens, the user +can be authorized via either their "username:password" line or the +"@groupname:password" line for the group that triggered the authentication. +.IP It is up to you what kind of password entries you want to include, either -users, groups, or both. The use of group rules in \(dq\&auth users\(dq\& does not +users, groups, or both. The use of group rules in "auth users" does not require that you specify a group password if you do not want to use shared passwords. -.IP -There is no default for the \(dq\&secrets file\(dq\& parameter, you must choose a name -(such as \f(CW/etc/rsyncd.secrets\fP). The file must normally not be readable -by \(dq\&other\(dq\&; see \(dq\&strict modes\(dq\&. If the file is not found or is rejected, no -logins for a \(dq\&user auth\(dq\& module will be possible. -.IP -.IP "\fBstrict modes\fP" -This parameter determines whether or not -the permissions on the secrets file will be checked. If \(dq\&strict modes\(dq\& is -true, then the secrets file must not be readable by any user ID other -than the one that the rsync daemon is running under. If \(dq\&strict modes\(dq\& is -false, the check is not performed. The default is true. This parameter -was added to accommodate rsync running on the Windows operating system. -.IP -.IP "\fBhosts allow\fP" -This parameter allows you to specify a list of comma\- -and/or whitespace\-separated patterns that are matched against a connecting -client\(cq\&s hostname and IP address. If none of the patterns match, then the +.IP +There is no default for the "secrets file" parameter, you must choose a +name (such as \fB/etc/rsyncd.secrets\fP). The file must normally not be +readable by "other"; see "strict modes". If the file is not found or is +rejected, no logins for a "user auth" module will be possible. +.IP "\fBstrict\ modes\fP" +This parameter determines whether or not the permissions on the secrets +file will be checked. If "strict modes" is true, then the secrets file +must not be readable by any user ID other than the one that the rsync +daemon is running under. If "strict modes" is false, the check is not +performed. The default is true. This parameter was added to accommodate +rsync running on the Windows operating system. +.IP "\fBhosts\ allow\fP" +This parameter allows you to specify a list of comma- and/or +whitespace-separated patterns that are matched against a connecting +client's hostname and IP address. If none of the patterns match, then the connection is rejected. -.IP +.IP Each pattern can be in one of five forms: -.IP -.RS -.IP o -a dotted decimal IPv4 address of the form a.b.c.d, or an IPv6 address -of the form a:b:c::d:e:f. In this case the incoming machine\(cq\&s IP address +.IP +.RS +.IP o +a dotted decimal IPv4 address of the form a.b.c.d, or an IPv6 address of +the form a:b:c::d:e:f. In this case the incoming machine's IP address must match exactly. -.IP o -an address/mask in the form ipaddr/n where ipaddr is the IP address -and n is the number of one bits in the netmask. All IP addresses which -match the masked IP address will be allowed in. -.IP o -an address/mask in the form ipaddr/maskaddr where ipaddr is the -IP address and maskaddr is the netmask in dotted decimal notation for IPv4, +.IP o +an address/mask in the form ipaddr/n where ipaddr is the IP address and n +is the number of one bits in the netmask. All IP addresses which match +the masked IP address will be allowed in. +.IP o +an address/mask in the form ipaddr/maskaddr where ipaddr is the IP +address and maskaddr is the netmask in dotted decimal notation for IPv4, or similar for IPv6, e.g. ffff:ffff:ffff:ffff:: instead of /64. All IP addresses which match the masked IP address will be allowed in. -.IP o +.IP o a hostname pattern using wildcards. If the hostname of the connecting IP -(as determined by a reverse lookup) matches the wildcarded name (using the -same rules as normal unix filename matching), the client is allowed in. This -only works if \(dq\&reverse lookup\(dq\& is enabled (the default). -.IP o +(as determined by a reverse lookup) matches the wildcarded name (using +the same rules as normal unix filename matching), the client is allowed +in. This only works if "reverse lookup" is enabled (the default). +.IP o a hostname. A plain hostname is matched against the reverse DNS of the -connecting IP (if \(dq\&reverse lookup\(dq\& is enabled), and/or the IP of the given -hostname is matched against the connecting IP (if \(dq\&forward lookup\(dq\& is -enabled, as it is by default). Any match will be allowed in. +connecting IP (if "reverse lookup" is enabled), and/or the IP of the +given hostname is matched against the connecting IP (if "forward lookup" +is enabled, as it is by default). Any match will be allowed in. .RE - -.IP -Note IPv6 link\-local addresses can have a scope in the address specification: -.IP -.RS -\f(CW fe80::1%link1\fP -.br -\f(CW fe80::%link1/64\fP -.br -\f(CW fe80::%link1/ffff:ffff:ffff:ffff::\fP -.br +.IP +Note IPv6 link-local addresses can have a scope in the address +specification: +.RS 4 +.IP +.nf +fe80::1%link1 +fe80::%link1/64 +fe80::%link1/ffff:ffff:ffff:ffff:: +.fi .RE - -.IP -You can also combine \(dq\&hosts allow\(dq\& with a separate \(dq\&hosts deny\(dq\& -parameter. If both parameters are specified then the \(dq\&hosts allow\(dq\& parameter is -checked first and a match results in the client being able to -connect. The \(dq\&hosts deny\(dq\& parameter is then checked and a match means -that the host is rejected. If the host does not match either the -\(dq\&hosts allow\(dq\& or the \(dq\&hosts deny\(dq\& patterns then it is allowed to +.IP +You can also combine "hosts allow" with a separate "hosts deny" parameter. +If both parameters are specified then the "hosts allow" parameter is +checked first and a match results in the client being able to connect. The +"hosts deny" parameter is then checked and a match means that the host is +rejected. If the host does not match either the "hosts allow" or the +"hosts deny" patterns then it is allowed to connect. +.IP +The default is no "hosts allow" parameter, which means all hosts can connect. -.IP -The default is no \(dq\&hosts allow\(dq\& parameter, which means all hosts can connect. -.IP -.IP "\fBhosts deny\fP" -This parameter allows you to specify a list of comma\- -and/or whitespace\-separated patterns that are matched against a connecting -clients hostname and IP address. If the pattern matches then the connection is -rejected. See the \(dq\&hosts allow\(dq\& parameter for more information. -.IP -The default is no \(dq\&hosts deny\(dq\& parameter, which means all hosts can connect. -.IP -.IP "\fBreverse lookup\fP" -Controls whether the daemon performs a reverse lookup -on the client\(cq\&s IP address to determine its hostname, which is used for -\(dq\&hosts allow\(dq\&/\(dq\&hosts deny\(dq\& checks and the \(dq\&%h\(dq\& log escape. This is enabled by -default, but you may wish to disable it to save time if you know the lookup will -not return a useful result, in which case the daemon will use the name -\(dq\&UNDETERMINED\(dq\& instead. -.IP +.IP "\fBhosts\ deny\fP" +This parameter allows you to specify a list of comma- and/or +whitespace-separated patterns that are matched against a connecting clients +hostname and IP address. If the pattern matches then the connection is +rejected. See the "hosts allow" parameter for more information. +.IP +The default is no "hosts deny" parameter, which means all hosts can +connect. +.IP "\fBreverse\ lookup\fP" +Controls whether the daemon performs a reverse lookup on the client's IP +address to determine its hostname, which is used for "hosts allow" & +"hosts deny" checks and the "%h" log escape. This is enabled by default, +but you may wish to disable it to save time if you know the lookup will not +return a useful result, in which case the daemon will use the name +"UNDETERMINED" instead. +.IP If this parameter is enabled globally (even by default), rsync performs the lookup as soon as a client connects, so disabling it for a module will not avoid the lookup. Thus, you probably want to disable it globally and then enable it for modules that need the information. -.IP -.IP "\fBforward lookup\fP" -Controls whether the daemon performs a forward lookup -on any hostname specified in an hosts allow/deny setting. By default this is -enabled, allowing the use of an explicit hostname that would not be returned -by reverse DNS of the connecting IP. -.IP -.IP "\fBignore errors\fP" -This parameter tells rsyncd to -ignore I/O errors on the daemon when deciding whether to run the delete -phase of the transfer. Normally rsync skips the \fB\-\-delete\fP step if any -I/O errors have occurred in order to prevent disastrous deletion due -to a temporary resource shortage or other I/O error. In some cases this -test is counter productive so you can use this parameter to turn off this -behavior. -.IP -.IP "\fBignore nonreadable\fP" -This tells the rsync daemon to completely -ignore files that are not readable by the user. This is useful for -public archives that may have some non\-readable files among the -directories, and the sysadmin doesn\(cq\&t want those files to be seen at all. -.IP -.IP "\fBtransfer logging\fP" -This parameter enables per\-file -logging of downloads and uploads in a format somewhat similar to that -used by ftp daemons. The daemon always logs the transfer at the end, so -if a transfer is aborted, no mention will be made in the log file. -.IP -If you want to customize the log lines, see the \(dq\&log format\(dq\& parameter. -.IP -.IP "\fBlog format\fP" -This parameter allows you to specify the -format used for logging file transfers when transfer logging is enabled. -The format is a text string containing embedded single\-character escape -sequences prefixed with a percent (%) character. An optional numeric -field width may also be specified between the percent and the escape -letter (e.g. \(dq\&\fB%\-50n %8l %07p\fP\(dq\&). -In addition, one or more apostrophes may be specified prior to a numerical -escape to indicate that the numerical value should be made more human\-readable. -The 3 supported levels are the same as for the \fB\-\-human\-readable\fP -command\-line option, though the default is for human\-readability to be off. -Each added apostrophe increases the level (e.g. \(dq\&\fB%'\&'\&l %'\&b %f\fP\(dq\&). -.IP -The default log format is \(dq\&%o %h [%a] %m (%u) %f %l\(dq\&, and a \(dq\&%t [%p] \(dq\& -is always prefixed when using the \(dq\&log file\(dq\& parameter. -(A perl script that will summarize this default log format is included -in the rsync source code distribution in the \(dq\&support\(dq\& subdirectory: -rsyncstats.) -.IP -The single\-character escapes that are understood are as follows: -.IP -.RS -.IP o +.IP "\fBforward\ lookup\fP" +Controls whether the daemon performs a forward lookup on any hostname +specified in an hosts allow/deny setting. By default this is enabled, +allowing the use of an explicit hostname that would not be returned by +reverse DNS of the connecting IP. +.IP "\fBignore\ errors\fP" +This parameter tells rsyncd to ignore I/O errors on the daemon when +deciding whether to run the delete phase of the transfer. Normally rsync +skips the \fB\-\-delete\fP step if any I/O errors have occurred in order to +prevent disastrous deletion due to a temporary resource shortage or other +I/O error. In some cases this test is counter productive so you can use +this parameter to turn off this behavior. +.IP "\fBignore\ nonreadable\fP" +This tells the rsync daemon to completely ignore files that are not +readable by the user. This is useful for public archives that may have some +non-readable files among the directories, and the sysadmin doesn't want +those files to be seen at all. +.IP "\fBtransfer\ logging\fP" +This parameter enables per-file logging of downloads and uploads in a +format somewhat similar to that used by ftp daemons. The daemon always +logs the transfer at the end, so if a transfer is aborted, no mention will +be made in the log file. +.IP +If you want to customize the log lines, see the "log format" parameter. +.IP "\fBlog\ format\fP" +This parameter allows you to specify the format used for logging file +transfers when transfer logging is enabled. The format is a text string +containing embedded single-character escape sequences prefixed with a +percent (%) character. An optional numeric field width may also be +specified between the percent and the escape letter (e.g. +"\fB%\-50n\ %8l\ %07p\fP"). In addition, one or more apostrophes may be specified +prior to a numerical escape to indicate that the numerical value should be +made more human-readable. The 3 supported levels are the same as for the +\fB\-\-human-readable\fP command-line option, though the default is for +human-readability to be off. Each added apostrophe increases the level +(e.g. "\fB%''l\ %'b\ %f\fP"). +.IP +The default log format is "\fB%o\ %h\ [%a]\ %m\ (%u)\ %f\ %l\fP", and a "\fB%t\ [%p]\fP" +is always prefixed when using the "log file" parameter. (A perl script +that will summarize this default log format is included in the rsync source +code distribution in the "support" subdirectory: rsyncstats.) +.IP +The single-character escapes that are understood are as follows: +.IP +.RS +.IP o %a the remote IP address (only available for a daemon) -.IP o +.IP o %b the number of bytes actually transferred -.IP o +.IP o %B the permission bits of the file (e.g. rwxrwxrwt) -.IP o -%c the total size of the block checksums received for the basis file (only when sending) -.IP o -%C the full\-file checksum if it is known for the file. For older rsync protocols/versions, the checksum was salted, and is thus not a useful value (and is not displayed when that is the case). For the checksum to output for a file, either the \fB\-\-checksum\fP option must be in\-effect or the file must have been transferred without a salted checksum being used. See the \fB\-\-checksum\-choice\fP option for a way to choose the algorithm. -.IP o -%f the filename (long form on sender; no trailing \(dq\&/\(dq\&) -.IP o -%G the gid of the file (decimal) or \(dq\&DEFAULT\(dq\& -.IP o +.IP o +%c the total size of the block checksums received for the basis file +(only when sending) +.IP o +%C the full-file checksum if it is known for the file. For older rsync +protocols/versions, the checksum was salted, and is thus not a useful +value (and is not displayed when that is the case). For the checksum to +output for a file, either the \fB\-\-checksum\fP option must be in-effect or +the file must have been transferred without a salted checksum being used. +See the \fB\-\-checksum-choice\fP option for a way to choose the algorithm. +.IP o +%f the filename (long form on sender; no trailing "/") +.IP o +%G the gid of the file (decimal) or "DEFAULT" +.IP o %h the remote host name (only available for a daemon) -.IP o +.IP o %i an itemized list of what is being updated -.IP o +.IP o %l the length of the file in bytes -.IP o -%L the string \(dq\& \-> SYMLINK\(dq\&, \(dq\& => HARDLINK\(dq\&, or \(dq\&\(dq\& (where \fBSYMLINK\fP or \fBHARDLINK\fP is a filename) -.IP o +.IP o +%L the string "\fB\->\ SYMLINK\fP", "\fB=>\ HARDLINK\fP", or "" (where \fBSYMLINK\fP +or \fBHARDLINK\fP is a filename) +.IP o %m the module name -.IP o -%M the last\-modified time of the file -.IP o -%n the filename (short form; trailing \(dq\&/\(dq\& on dir) -.IP o -%o the operation, which is \(dq\&send\(dq\&, \(dq\&recv\(dq\&, or \(dq\&del.\(dq\& (the latter includes the trailing period) -.IP o +.IP o +%M the last-modified time of the file +.IP o +%n the filename (short form; trailing "/" on dir) +.IP o +%o the operation, which is "send", "recv", or "del." (the latter includes +the trailing period) +.IP o %p the process ID of this rsync session -.IP o +.IP o %P the module path -.IP o +.IP o %t the current date time -.IP o +.IP o %u the authenticated username or an empty string -.IP o +.IP o %U the uid of the file (decimal) .RE - -.IP -For a list of what the characters mean that are output by \(dq\&%i\(dq\&, see the -\fB\-\-itemize\-changes\fP option in the rsync manpage. -.IP -Note that some of the logged output changes when talking with older -rsync versions. For instance, deleted files were only output as verbose -messages prior to rsync 2.6.4. -.IP +.IP +For a list of what the characters mean that are output by "%i", see the +\fB\-\-itemize-changes\fP option in the rsync manpage. +.IP +Note that some of the logged output changes when talking with older rsync +versions. For instance, deleted files were only output as verbose messages +prior to rsync 2.6.4. .IP "\fBtimeout\fP" -This parameter allows you to override the -clients choice for I/O timeout for this module. Using this parameter you -can ensure that rsync won\(cq\&t wait on a dead client forever. The timeout -is specified in seconds. A value of zero means no timeout and is the -default. A good choice for anonymous rsync daemons may be 600 (giving -a 10 minute timeout). -.IP -.IP "\fBrefuse options\fP" -This parameter allows you to -specify a space\-separated list of rsync command line options that will -be refused by your rsync daemon. -You may specify the full option name, its one\-letter abbreviation, or a -wild\-card string that matches multiple options. +This parameter allows you to override the clients choice for I/O timeout +for this module. Using this parameter you can ensure that rsync won't wait +on a dead client forever. The timeout is specified in seconds. A value of +zero means no timeout and is the default. A good choice for anonymous rsync +daemons may be 600 (giving a 10 minute timeout). +.IP "\fBrefuse\ options\fP" +This parameter allows you to specify a space-separated list of rsync +command-line options that will be refused by your rsync daemon. You may +specify the full option name, its one-letter abbreviation, or a wild-card +string that matches multiple options. Beginning in 3.2.0, you can also +negate a match term by starting it with a "!". +.IP +When an option is refused, the daemon prints an error message and exits. +.IP For example, this would refuse \fB\-\-checksum\fP (\fB\-c\fP) and all the various delete options: -.IP -.RS -\f(CW refuse options = c delete\fP +.RS 4 +.IP +.nf +refuse options = c delete +.fi .RE - -.IP +.IP The reason the above refuses all delete options is that the options imply \fB\-\-delete\fP, and implied options are refused just like explicit options. -As an additional safety feature, the refusal of \(dq\&delete\(dq\& also refuses -\fBremove\-source\-files\fP when the daemon is the sender; if you want the latter -without the former, instead refuse \(dq\&delete\-*\(dq\& \-\- that refuses all the -delete modes without affecting \fB\-\-remove\-source\-files\fP. -.IP -When an option is refused, the daemon prints an error message and exits. -To prevent all compression when serving files, -you can use \(dq\&dont compress = *\(dq\& (see below) -instead of \(dq\&refuse options = compress\(dq\& to avoid returning an error to a -client that requests compression. -.IP -.IP "\fBdont compress\fP" -This parameter allows you to select -filenames based on wildcard patterns that should not be compressed -when pulling files from the daemon (no analogous parameter exists to -govern the pushing of files to a daemon). -Compression is expensive in terms of CPU usage, so it -is usually good to not try to compress files that won\(cq\&t compress well, -such as already compressed files. -.IP -The \(dq\&dont compress\(dq\& parameter takes a space\-separated list of -case\-insensitive wildcard patterns. Any source filename matching one -of the patterns will not be compressed during transfer. -.IP -See the \fB\-\-skip\-compress\fP parameter in the \fBrsync\fP(1) manpage for the list -of file suffixes that are not compressed by default. Specifying a value -for the \(dq\&dont compress\(dq\& parameter changes the default when the daemon is -the sender. -.IP -.IP "\fBpre\-xfer exec\fP, \fBpost\-xfer exec\fP" -You may specify a command to be run -before and/or after the transfer. If the \fBpre\-xfer exec\fP command fails, the -transfer is aborted before it begins. Any output from the script on stdout (up -to several KB) will be displayed to the user when aborting, but is NOT -displayed if the script returns success. Any output from the script on stderr -goes to the daemon\(cq\&s stderr, which is typically discarded (though see -\-\-no\-detatch option for a way to see the stderr output, which can assist with -debugging). -.IP -The following environment variables will be set, though some are -specific to the pre\-xfer or the post\-xfer environment: -.IP -.RS -.IP o +.IP +The use of a negated match allows you to fine-tune your refusals after a +wild-card, such as this: +.RS 4 +.IP +.nf +refuse options = delete-* !delete-during +.fi +.RE +.IP +Negated matching can also turn your list of refused options into a list of +accepted options. To do this, begin the list with a "\fB*\fP" (to refuse all +options) and then specify one or more negated matches to accept. For +example: +.RS 4 +.IP +.nf +refuse options = * !a !v !compress* +.fi +.RE +.IP +Don't worry that the "\fB*\fP" will refuse certain vital options such as +\fB\-\-dry-run\fP, \fB\-\-server\fP, \fB\-\-no-iconv\fP, \fB\-\-protect-args\fP, etc. These +important options are not matched by wild-card, so they must be overridden +by their exact name. For instance, if you're forcing iconv transfers you +could use something like this: +.RS 4 +.IP +.nf +refuse options = * no-iconv !a !v +.fi +.RE +.IP +As an additional aid (beginning in 3.2.0), refusing (or "\fB!refusing\fP") the +"a" or "archive" option also affects all the options that the \fB\-\-archive\fP +option implies (\fB\-rdlptgoD\fP), but only if the option is matched explicitly +(not using a wildcard). If you want to do something tricky, you can use +"\fBarchive*\fP" to avoid this side-effect, but keep in mind that no normal +rsync client ever sends the actual archive option to the server. +.IP +As an additional safety feature, the refusal of "delete" also refuses +\fBremove-source-files\fP when the daemon is the sender; if you want the latter +without the former, instead refuse "\fBdelete-*\fP" as that refuses all the +delete modes without affecting \fB\-\-remove-source-files\fP. (Keep in mind that +the client's \fB\-\-delete\fP option typically results in \fB\-\-delete-during\fP.) +.IP +When un-refusing delete options, you should either specify "\fB!delete*\fP" (to +accept all delete options) or specify a limited set that includes "delete", +such as: +.RS 4 +.IP +.nf +refuse options = * !a !delete !delete-during +.fi +.RE +.IP +\&... whereas this accepts any delete option except \fB\-\-delete-after\fP: +.RS 4 +.IP +.nf +refuse options = * !a !delete* delete-after +.fi +.RE +.IP +A note on refusing "compress"\ \-\- it is better to set the "dont compress" +daemon parameter to "\fB*\fP" because that disables compression silently +instead of returning an error that forces the client to remove the \fB\-z\fP +option. +.IP +If you are un-refusing the compress option, you probably want to match +"\fB!compress*\fP" so that you also accept the \fB\-\-compress-level\fP option. +.IP +Note that the "write-devices" option is refused by default, but can be +explicitly accepted with "\fB!write-devices\fP". The options "log-file" and +"log-file-format" are forcibly refused and cannot be accepted. +.IP +Here are all the options that are not matched by wild-cards: +.IP +.RS +.IP o +\fB\-\-server\fP: Required for rsync to even work. +.IP o +\fB\-\-rsh\fP, \fB\-e\fP: Required to convey compatibility flags to the server. +.IP o +\fB\-\-log-format\fP: This is required to convey things like +\fB\-\-itemize-changes\fP to a remote receiver. Is an older name for +\fB\-\-out-format\fP that is still passed to the server for improved backward +compatibility and should not be confused with \fB\-\-log-file-format\fP. +.IP o +\fB\-\-sender\fP: Use "write only" parameter instead of refusing this. +.IP o +\fB\-\-dry-run\fP, \fB\-n\fP: Who would want to disable this? +.IP o +\fB\-\-protect-args\fP, \fB\-n\fP: This actually makes transfers safer. +.IP o +\fB\-\-from0\fP, \fB\-0\fP: Make it easier to accept/refuse \fB\-\-files-from\fP without +affecting this modifier. +.IP o +\fB\-\-iconv\fP: This is auto-disabled based on "charset" parameter. +.IP o +\fB\-\-no-iconv\fP: Most transfers use this option. +.IP o +\fB\-\-checksum-seed\fP: Is a fairly rare, safe option. +.IP o +\fB\-\-write-devices\fP: Is non-wild but also auto-disabled. +.RE +.IP "\fBdont\ compress\fP" +This parameter allows you to select filenames based on wildcard patterns +that should not be compressed when pulling files from the daemon (no +analogous parameter exists to govern the pushing of files to a daemon). +Compression can be expensive in terms of CPU usage, so it is usually good +to not try to compress files that won't compress well, such as already +compressed files. +.IP +The "dont compress" parameter takes a space-separated list of +case-insensitive wildcard patterns. Any source filename matching one of the +patterns will be compressed as little as possible during the transfer. If +the compression algorithm has an "off" level (such as zlib/zlibx) then no +compression occurs for those files. Other algorithms have the level +minimized to reduces the CPU usage as much as possible. +.IP +See the \fB\-\-skip-compress\fP parameter in the \fBrsync\fP(1) manpage for the +list of file suffixes that are not compressed by default. Specifying a +value for the "dont compress" parameter changes the default when the daemon +is the sender. +.IP "\fBearly\ exec\fP, \fBpre-xfer\ exec\fP, \fBpost-xfer\ exec\fP" +You may specify a command to be run in the early stages of the connection, +or right before and/or after the transfer. If the \fBearly\ exec\fP or +\fBpre-xfer\ exec\fP command returns an error code, the transfer is aborted +before it begins. Any output from the \fBpre-xfer\ exec\fP command on stdout +(up to several KB) will be displayed to the user when aborting, but is +\fInot\fP displayed if the script returns success. The other programs cannot +send any text to the user. All output except for the \fBpre-xfer\ exec\fP +stdout goes to the corresponding daemon's stdout/stderr, which is typically +discarded. See the \fB\-\-no-detatch\fP option for a way to see the daemon's +output, which can assist with debugging. +.IP +Note that the \fBearly\ exec\fP command runs before any part of the transfer +request is known except for the module name. This helper script can be +used to setup a disk mount or decrypt some data into a module dir, but you +may need to use \fBlock\ file\fP and \fBmax\ connections\fP to avoid concurrency +issues. +.IP +Note that the \fBpost-xfer\ exec\fP command is still run even if one of the +other scripts returns an error code. The \fBpre-xfer\ exec\fP command will \fInot\fP +be run, however, if the \fBearly\ exec\fP command fails. +.IP +The following environment variables will be set, though some are specific +to the pre-xfer or the post-xfer environment: +.IP +.RS +.IP o \fBRSYNC_MODULE_NAME\fP: The name of the module being accessed. -.IP o +.IP o \fBRSYNC_MODULE_PATH\fP: The path configured for the module. -.IP o -\fBRSYNC_HOST_ADDR\fP: The accessing host\(cq\&s IP address. -.IP o -\fBRSYNC_HOST_NAME\fP: The accessing host\(cq\&s name. -.IP o -\fBRSYNC_USER_NAME\fP: The accessing user\(cq\&s name (empty if no user). -.IP o +.IP o +\fBRSYNC_HOST_ADDR\fP: The accessing host's IP address. +.IP o +\fBRSYNC_HOST_NAME\fP: The accessing host's name. +.IP o +\fBRSYNC_USER_NAME\fP: The accessing user's name (empty if no user). +.IP o \fBRSYNC_PID\fP: A unique number for this transfer. -.IP o -\fBRSYNC_REQUEST\fP: (pre\-xfer only) The module/path info specified -by the user. Note that the user can specify multiple source files, -so the request can be something like \(dq\&mod/path1 mod/path2\(dq\&, etc. -.IP o -\fBRSYNC_ARG#\fP: (pre\-xfer only) The pre\-request arguments are set -in these numbered values. RSYNC_ARG0 is always \(dq\&rsyncd\(dq\&, followed by -the options that were used in RSYNC_ARG1, and so on. There will be a -value of \(dq\&.\(dq\& indicating that the options are done and the path args -are beginning \-\- these contain similar information to RSYNC_REQUEST, -but with values separated and the module name stripped off. -.IP o -\fBRSYNC_EXIT_STATUS\fP: (post\-xfer only) the server side\(cq\&s exit value. -This will be 0 for a successful run, a positive value for an error that the +.IP o +\fBRSYNC_REQUEST\fP: (pre-xfer only) The module/path info specified by the +user. Note that the user can specify multiple source files, so the +request can be something like "mod/path1 mod/path2", etc. +.IP o +\fBRSYNC_ARG#\fP: (pre-xfer only) The pre-request arguments are set in these +numbered values. RSYNC_ARG0 is always "rsyncd", followed by the options +that were used in RSYNC_ARG1, and so on. There will be a value of "." +indicating that the options are done and the path args are beginning\ \-\- +these contain similar information to RSYNC_REQUEST, but with values +separated and the module name stripped off. +.IP o +\fBRSYNC_EXIT_STATUS\fP: (post-xfer only) the server side's exit value. This +will be 0 for a successful run, a positive value for an error that the server generated, or a \-1 if rsync failed to exit properly. Note that an error that occurs on the client side does not currently get sent to the server side, so this is not the final exit status for the whole transfer. -.IP o -\fBRSYNC_RAW_STATUS\fP: (post\-xfer only) the raw exit value from -\f(CWwaitpid()\fP -\&. +.IP o +\fBRSYNC_RAW_STATUS\fP: (post-xfer only) the raw exit value from +\fBwaitpid()\fP. .RE - -.IP +.IP Even though the commands can be associated with a particular module, they are run using the permissions of the user that started the daemon (not the -module\(cq\&s uid/gid setting) without any chroot restrictions. -.IP +module's uid/gid setting) without any chroot restrictions. +.IP +These settings honor 2 environment variables: use RSYNC_SHELL to set a +shell to use when running the command (which otherwise uses your +\fBsystem()\fP call's default shell), and use RSYNC_NO_XFER_EXEC to disable +both options completely. +.P .SH "CONFIG DIRECTIVES" - -.PP +.P There are currently two config directives available that allow a config file to -incorporate the contents of other files: \fB&include\fP and \fB&merge\fP. Both -allow a reference to either a file or a directory. They differ in how -segregated the file\(cq\&s contents are considered to be. -.PP +incorporate the contents of other files: \fB&include\fP and \fB&merge\fP. Both allow +a reference to either a file or a directory. They differ in how segregated the +file's contents are considered to be. +.P The \fB&include\fP directive treats each file as more distinct, with each one -inheriting the defaults of the parent file, starting the parameter parsing -as globals/defaults, and leaving the defaults unchanged for the parsing of -the rest of the parent file. -.PP -The \fB&merge\fP directive, on the other hand, treats the file\(cq\&s contents as -if it were simply inserted in place of the directive, and thus it can set -parameters in a module started in another file, can affect the defaults for -other files, etc. -.PP -When an \fB&include\fP or \fB&merge\fP directive refers to a directory, it will read -in all the \fB*.conf\fP or \fB*.inc\fP files (respectively) that are contained inside -that directory (without any -recursive scanning), with the files sorted into alpha order. So, if you have a -directory named \(dq\&rsyncd.d\(dq\& with the files \(dq\&foo.conf\(dq\&, \(dq\&bar.conf\(dq\&, and -\(dq\&baz.conf\(dq\& inside it, this directive: -.PP -.nf - &include /path/rsyncd.d -.fi - -.PP +inheriting the defaults of the parent file, starting the parameter parsing as +globals/defaults, and leaving the defaults unchanged for the parsing of the +rest of the parent file. +.P +The \fB&merge\fP directive, on the other hand, treats the file's contents as if it +were simply inserted in place of the directive, and thus it can set parameters +in a module started in another file, can affect the defaults for other files, +etc. +.P +When an \fB&include\fP or \fB&merge\fP directive refers to a directory, it will read in +all the \fB*.conf\fP or \fB*.inc\fP files (respectively) that are contained inside that +directory (without any recursive scanning), with the files sorted into alpha +order. So, if you have a directory named "rsyncd.d" with the files "foo.conf", +"bar.conf", and "baz.conf" inside it, this directive: +.RS 4 +.P +.nf +&include /path/rsyncd.d +.fi +.RE +.P would be the same as this set of directives: -.PP -.nf - &include /path/rsyncd.d/bar.conf - &include /path/rsyncd.d/baz.conf - &include /path/rsyncd.d/foo.conf -.fi - -.PP +.RS 4 +.P +.nf +&include /path/rsyncd.d/bar.conf +&include /path/rsyncd.d/baz.conf +&include /path/rsyncd.d/foo.conf +.fi +.RE +.P except that it adjusts as files are added and removed from the directory. -.PP +.P The advantage of the \fB&include\fP directive is that you can define one or more -modules in a separate file without worrying about unintended side\-effects -between the self\-contained module files. -.PP +modules in a separate file without worrying about unintended side-effects +between the self-contained module files. +.P The advantage of the \fB&merge\fP directive is that you can load config snippets that can be included into multiple module definitions, and you can also set -global values that will affect connections (such as \fBmotd file\fP), or globals +global values that will affect connections (such as \fBmotd\ file\fP), or globals that will affect other include files. -.PP +.P For example, this is a useful /etc/rsyncd.conf file: -.PP -.nf - port = 873 - log file = /var/log/rsync.log - pid file = /var/lock/rsync.lock +.RS 4 +.P +.nf +port = 873 +log file = /var/log/rsync.log +pid file = /var/lock/rsync.lock - &merge /etc/rsyncd.d - &include /etc/rsyncd.d -.fi - -.PP -This would merge any /etc/rsyncd.d/*.inc files (for global values that should -stay in effect), and then include any /etc/rsyncd.d/*.conf files (defining -modules without any global\-value cross\-talk). -.PP +&merge /etc/rsyncd.d +&include /etc/rsyncd.d +.fi +.RE +.P +This would merge any \fB/etc/rsyncd.d/*.inc\fP files (for global values that should +stay in effect), and then include any \fB/etc/rsyncd.d/*.conf\fP files (defining +modules without any global-value cross-talk). +.P .SH "AUTHENTICATION STRENGTH" - -.PP -The authentication protocol used in rsync is a 128 bit MD4 based -challenge response system. This is fairly weak protection, though (with -at least one brute\-force hash\-finding algorithm publicly available), so -if you want really top\-quality security, then I recommend that you run -rsync over ssh. (Yes, a future version of rsync will switch over to a -stronger hashing method.) -.PP +.P +The authentication protocol used in rsync is a 128 bit MD4 based challenge +response system. This is fairly weak protection, though (with at least one +brute-force hash-finding algorithm publicly available), so if you want really +top-quality security, then I recommend that you run rsync over ssh. (Yes, a +future version of rsync will switch over to a stronger hashing method.) +.P Also note that the rsync daemon protocol does not currently provide any encryption of the data that is transferred over the connection. Only -authentication is provided. Use ssh as the transport if you want +authentication is provided. Use ssh as the transport if you want encryption. +.P +You can also make use of SSL/TLS encryption if you put rsync behind an +SSL proxy. +.P +.SH "SSL/TLS Daemon Setup" +.P +When setting up an rsync daemon for access via SSL/TLS, you will need to +configure a proxy (such as haproxy or nginx) as the front-end that handles the encryption. -.PP -Future versions of rsync may support SSL for better authentication and -encryption, but that is still being investigated. -.PP -.SH "EXAMPLES" +.P +.IP o +You should limit the access to the backend-rsyncd port to only allow the +proxy to connect. If it is on the same host as the proxy, then configuring +it to only listen on localhost is a good idea. +.IP o +You should consider turning on the \fBproxy\ protocol\fP parameter if your proxy +supports sending that information. The examples below assume that this is +enabled. +.P +An example haproxy setup is as follows: +.RS 4 +.P +.nf +frontend fe_rsync-ssl + bind :::874 ssl crt /etc/letsencrypt/example.com/combined.pem + mode tcp + use_backend be_rsync -.PP -A simple rsyncd.conf file that allow anonymous rsync to a ftp area at -\f(CW/home/ftp\fP would be: -.PP -.nf +backend be_rsync + mode tcp + server local-rsync 127.0.0.1:873 check send-proxy +.fi +.RE +.P +An example nginx proxy setup is as follows: +.RS 4 +.P +.nf +stream { + server { + listen 874 ssl; + listen [::]:874 ssl; + ssl_certificate /etc/letsencrypt/example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem + + proxy_pass localhost:873; + proxy_protocol on; # Requires "proxy protocol = true" + proxy_timeout 1m; + proxy_connect_timeout 5s; + } +} +.fi +.RE +.P +.SH "EXAMPLES" +.P +A simple rsyncd.conf file that allow anonymous rsync to a ftp area at +\fB/home/ftp\fP would be: +.RS 4 +.P +.nf [ftp] path = /home/ftp comment = ftp export area - -.fi - -.PP +.fi +.RE +.P A more sophisticated example would be: -.PP -.nf - +.RS 4 +.P +.nf uid = nobody gid = nobody use chroot = yes @@ -1061,73 +1188,58 @@ pid file = /var/run/rsyncd.pid comment = CVS repository (requires authentication) auth users = tridge, susan secrets file = /etc/rsyncd.secrets - -.fi - -.PP +.fi +.RE +.P The /etc/rsyncd.secrets file would look something like this: -.PP -.RS -\f(CWtridge:mypass\fP -.br -\f(CWsusan:herpass\fP -.br +.RS 4 +.P +.nf +tridge:mypass +susan:herpass +.fi .RE - -.PP +.P .SH "FILES" - -.PP +.P /etc/rsyncd.conf or rsyncd.conf -.PP +.P .SH "SEE ALSO" - -.PP -\fBrsync\fP(1) -.PP -.SH "DIAGNOSTICS" - -.PP +.P +\fBrsync\fP(1), \fBrsync-ssl\fP(1) +.P .SH "BUGS" - -.PP +.P Please report bugs! The rsync bug tracking system is online at -http://rsync.samba.org/ -.PP +http://rsync.samba.org/. +.P .SH "VERSION" - -.PP -This man page is current for version 3.1.3 of rsync. -.PP +.P +This man page is current for version 3.2.0 of rsync. +.P .SH "CREDITS" - -.PP +.P rsync is distributed under the GNU General Public License. See the file COPYING for details. -.PP -The primary ftp site for rsync is -ftp://rsync.samba.org/pub/rsync. -.PP -A WEB site is available at -http://rsync.samba.org/ -.PP +.P +The primary ftp site for rsync is ftp://rsync.samba.org/pub/rsync +.P +A web site is available at http://rsync.samba.org/. +.P We would be delighted to hear from you if you like this program. -.PP -This program uses the zlib compression library written by Jean\-loup -Gailly and Mark Adler. -.PP +.P +This program uses the zlib compression library written by Jean-loup Gailly and +Mark Adler. +.P .SH "THANKS" - -.PP -Thanks to Warren Stanley for his original idea and patch for the rsync -daemon. Thanks to Karsten Thygesen for his many suggestions and -documentation! -.PP +.P +Thanks to Warren Stanley for his original idea and patch for the rsync daemon. +Thanks to Karsten Thygesen for his many suggestions and documentation! +.P .SH "AUTHOR" - -.PP -rsync was written by Andrew Tridgell and Paul Mackerras. -Many people have later contributed to it. -.PP +.P +rsync was written by Andrew Tridgell and Paul Mackerras. Many people have +later contributed to it. +.P Mailing lists for support and development are available at -http://lists.samba.org +http://lists.samba.org/. diff --git a/rsyncd.conf.5.html b/rsyncd.conf.5.html new file mode 100644 index 00000000..130835c6 --- /dev/null +++ b/rsyncd.conf.5.html @@ -0,0 +1,1141 @@ +<html><head> +<title>rsyncd.conf(5) man page</title> +<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet"> +<style> +body { + max-width: 50em; + margin: auto; +} +body, b, strong, u { + font-family: 'Roboto', sans-serif; +} +code { + font-family: 'Roboto Mono', monospace; + font-weight: bold; + white-space: pre; +} +pre code { + display: block; + font-weight: normal; +} +blockquote pre code { + background: #f1f1f1; +} +dd p:first-of-type { + margin-block-start: 0em; +} +</style> +</head><body> +<h1>NAME</h1> +<p>rsyncd.conf -⁠ configuration file for rsync in daemon mode</p> +<h1>SYNOPSIS</h1> +<p>rsyncd.conf</p> +<h1>DESCRIPTION</h1> +<p>The rsyncd.conf file is the runtime configuration file for rsync when run as an +rsync daemon.</p> +<p>The rsyncd.conf file controls authentication, access, logging and available +modules.</p> +<h1>FILE FORMAT</h1> +<p>The file consists of modules and parameters. A module begins with the name of +the module in square brackets and continues until the next module begins. +Modules contain parameters of the form <code>name = value</code>.</p> +<p>The file is line-based -⁠-⁠ that is, each newline-terminated line represents +either a comment, a module name or a parameter.</p> +<p>Only the first equals sign in a parameter is significant. Whitespace before or +after the first equals sign is discarded. Leading, trailing and internal +whitespace in module and parameter names is irrelevant. Leading and trailing +whitespace in a parameter value is discarded. Internal whitespace within a +parameter value is retained verbatim.</p> +<p>Any line <strong>beginning</strong> with a hash (<code>#</code>) is ignored, as are lines containing +only whitespace. (If a hash occurs after anything other than leading +whitespace, it is considered a part of the line's content.)</p> +<p>Any line ending in a <code>\</code> is "continued" on the next line in the customary UNIX +fashion.</p> +<p>The values following the equals sign in parameters are all either a string (no +quotes needed) or a boolean, which may be given as yes/no, 0/1 or true/false. +Case is not significant in boolean values, but is preserved in string values.</p> +<h1>LAUNCHING THE RSYNC DAEMON</h1> +<p>The rsync daemon is launched by specifying the <code>--daemon</code> option to +rsync.</p> +<p>The daemon must run with root privileges if you wish to use chroot, to bind to +a port numbered under 1024 (as is the default 873), or to set file ownership. +Otherwise, it must just have permission to read and write the appropriate data, +log, and lock files.</p> +<p>You can launch it either via inetd, as a stand-alone daemon, or from an rsync +client via a remote shell. If run as a stand-alone daemon then just run the +command "<code>rsync --daemon</code>" from a suitable startup script.</p> +<p>When run via inetd you should add a line like this to /etc/services:</p> +<blockquote> +<pre><code>rsync 873/tcp +</code></pre> +</blockquote> +<p>and a single line something like this to /etc/inetd.conf:</p> +<blockquote> +<pre><code>rsync stream tcp nowait root /usr/bin/rsync rsyncd --daemon +</code></pre> +</blockquote> +<p>Replace "/usr/bin/rsync" with the path to where you have rsync installed on +your system. You will then need to send inetd a HUP signal to tell it to +reread its config file.</p> +<p>Note that you should <strong>not</strong> send the rsync daemon a HUP signal to force it to +reread the <code>rsyncd.conf</code> file. The file is re-read on each client connection.</p> +<h1>GLOBAL PARAMETERS</h1> +<p>The first parameters in the file (before a [module] header) are the global +parameters. Rsync also allows for the use of a "[global]" module name to +indicate the start of one or more global-parameter sections (the name must be +lower case).</p> +<p>You may also include any module parameters in the global part of the config +file in which case the supplied value will override the default for that +parameter.</p> +<p>You may use references to environment variables in the values of parameters. +String parameters will have %VAR% references expanded as late as possible (when +the string is first used in the program), allowing for the use of variables +that rsync sets at connection time, such as RSYNC_USER_NAME. Non-string +parameters (such as true/false settings) are expanded when read from the config +file. If a variable does not exist in the environment, or if a sequence of +characters is not a valid reference (such as an un-paired percent sign), the +raw characters are passed through unchanged. This helps with backward +compatibility and safety (e.g. expanding a non-existent %VAR% to an empty +string in a path could result in a very unsafe path). The safest way to insert +a literal % into a value is to use %%.</p> +<dl> + +<dt><code>motd file</code></dt><dd> +<p>This parameter allows you to specify a "message of the day" to display to +clients on each connect. This usually contains site information and any +legal notices. The default is no motd file. This can be overridden by the +<code>--dparam=motdfile=FILE</code> command-line option when starting the daemon.</p> +</dd> + +<dt><code>pid file</code></dt><dd> +<p>This parameter tells the rsync daemon to write its process ID to that file. +The rsync keeps the file locked so that it can know when it is safe to +overwrite an existing file.</p> +<p>The filename can be overridden by the <code>--dparam=pidfile=FILE</code> command-line +option when starting the daemon.</p> +</dd> + +<dt><code>port</code></dt><dd> +<p>You can override the default port the daemon will listen on by specifying +this value (defaults to 873). This is ignored if the daemon is being run +by inetd, and is superseded by the <code>--port</code> command-line option.</p> +</dd> + +<dt><code>address</code></dt><dd> +<p>You can override the default IP address the daemon will listen on by +specifying this value. This is ignored if the daemon is being run by +inetd, and is superseded by the <code>--address</code> command-line option.</p> +</dd> + +<dt><code>socket options</code></dt><dd> +<p>This parameter can provide endless fun for people who like to tune their +systems to the utmost degree. You can set all sorts of socket options which +may make transfers faster (or slower!). Read the man page for the +<strong>setsockopt()</strong> system call for details on some of the options you may be +able to set. By default no special socket options are set. These settings +can also be specified via the <code>--sockopts</code> command-line option.</p> +</dd> + +<dt><code>listen backlog</code></dt><dd> +<p>You can override the default backlog value when the daemon listens for +connections. It defaults to 5.</p> +</dd> +</dl> +<h1>MODULE PARAMETERS</h1> +<p>After the global parameters you should define a number of modules, each module +exports a directory tree as a symbolic name. Modules are exported by specifying +a module name in square brackets [module] followed by the parameters for that +module. The module name cannot contain a slash or a closing square bracket. +If the name contains whitespace, each internal sequence of whitespace will be +changed into a single space, while leading or trailing whitespace will be +discarded. Also, the name cannot be "global" as that exact name indicates that +global parameters follow (see above).</p> +<p>As with GLOBAL PARAMETERS, you may use references to environment variables in +the values of parameters. See the GLOBAL PARAMETERS section for more details.</p> +<dl> + +<dt><code>comment</code></dt><dd> +<p>This parameter specifies a description string that is displayed next to the +module name when clients obtain a list of available modules. The default is +no comment.</p> +</dd> + +<dt><code>path</code></dt><dd> +<p>This parameter specifies the directory in the daemon's filesystem to make +available in this module. You must specify this parameter for each module +in <code>rsyncd.conf</code>.</p> +<p>You may base the path's value off of an environment variable by surrounding +the variable name with percent signs. You can even reference a variable +that is set by rsync when the user connects. For example, this would use +the authorizing user's name in the path:</p> +<blockquote> +<pre><code>path = /home/%RSYNC_USER_NAME% +</code></pre> +</blockquote> +<p>It is fine if the path includes internal spaces -⁠-⁠ they will be retained +verbatim (which means that you shouldn't try to escape them). If your +final directory has a trailing space (and this is somehow not something you +wish to fix), append a trailing slash to the path to avoid losing the +trailing whitespace.</p> +</dd> + +<dt><code>use chroot</code></dt><dd> +<p>If "use chroot" is true, the rsync daemon will chroot to the "path" before +starting the file transfer with the client. This has the advantage of +extra protection against possible implementation security holes, but it has +the disadvantages of requiring super-user privileges, of not being able to +follow symbolic links that are either absolute or outside of the new root +path, and of complicating the preservation of users and groups by name (see +below).</p> +<p>As an additional safety feature, you can specify a dot-dir in the module's +"path" to indicate the point where the chroot should occur. This allows +rsync to run in a chroot with a non-"/" path for the top of the transfer +hierarchy. Doing this guards against unintended library loading (since +those absolute paths will not be inside the transfer hierarchy unless you +have used an unwise pathname), and lets you setup libraries for the chroot +that are outside of the transfer. For example, specifying +"/var/rsync/./module1" will chroot to the "/var/rsync" directory and set +the inside-chroot path to "/module1". If you had omitted the dot-dir, the +chroot would have used the whole path, and the inside-chroot path would +have been "/".</p> +<p>When both "use chroot" and "daemon chroot" are false, OR the inside-chroot +path of "use chroot" is not "/", rsync will: (1) munge symlinks by default +for security reasons (see "munge symlinks" for a way to turn this off, but +only if you trust your users), (2) substitute leading slashes in absolute +paths with the module's path (so that options such as <code>--backup-dir</code>, +<code>--compare-dest</code>, etc. interpret an absolute path as rooted in the module's +"path" dir), and (3) trim ".." path elements from args if rsync believes +they would escape the module hierarchy. The default for "use chroot" is +true, and is the safer choice (especially if the module is not read-only).</p> +<p>When this parameter is enabled, the "numeric-ids" option will also default +to being enabled (disabling name lookups). See below for what a chroot +needs in order for name lookups to succeed.</p> +<p>If you copy library resources into the module's chroot area, you should +protect them through your OS's normal user/group or ACL settings (to +prevent the rsync module's user from being able to change them), and then +hide them from the user's view via "exclude" (see how in the discussion of +that parameter). At that point it will be safe to enable the mapping of +users and groups by name using the "numeric ids" daemon parameter (see +below).</p> +<p>Note also that you are free to setup custom user/group information in the +chroot area that is different from your normal system. For example, you +could abbreviate the list of users and groups.</p> +</dd> + +<dt><code>daemon chroot</code></dt><dd> +<p>This parameter specifies a path to which the daemon will chroot before +beginning communication with clients. Module paths (and any "use chroot" +settings) will then be related to this one. This lets you choose if you +want the whole daemon to be chrooted (with this setting), just the +transfers to be chrooted (with "use chroot"), or both. Keep in mind that +the "daemon chroot" area may need various OS/lib/etc files installed to +allow the daemon to function. By default the daemon runs without any +chrooting.</p> +</dd> + +<dt><code>proxy protocol</code></dt><dd> +<p>When this parameter is enabled, all incoming connections must start with a +V1 or V2 proxy protocol header. If the header is not found, the connection +is closed.</p> +<p>Setting this to <code>true</code> requires a proxy server to forward source IP +information to rsync, allowing you to log proper IP/host info and make use +of client-oriented IP restrictions. The default of <code>false</code> means that the +IP information comes directly from the socket's metadata. If rsync is not +behind a proxy, this should be disabled.</p> +<p><u>CAUTION</u>: using this option can be dangerous if you do not ensure that +only the proxy is allowed to connect to the rsync port. If any non-proxied +connections are allowed through, the client will be able to use a modified +rsync to spoof any remote IP address that they desire. You can lock this +down using something like iptables <code>-uid-owner root</code> rules (for strict +localhost access), various firewall rules, or you can require password +authorization so that any spoofing by users will not grant extra access.</p> +<p>This setting is global. If you need some modules to require this and not +others, then you will need to setup multiple rsync daemon processes on +different ports.</p> +</dd> + +<dt><code>numeric ids</code></dt><dd> +<p>Enabling this parameter disables the mapping of users and groups by name +for the current daemon module. This prevents the daemon from trying to +load any user/group-related files or libraries. This enabling makes the +transfer behave as if the client had passed the <code>--numeric-ids</code> +command-line option. By default, this parameter is enabled for chroot +modules and disabled for non-chroot modules. Also keep in mind that +uid/gid preservation requires the module to be running as root (see "uid") +or for "fake super" to be configured.</p> +<p>A chroot-enabled module should not have this parameter enabled unless +you've taken steps to ensure that the module has the necessary resources it +needs to translate names, and that it is not possible for a user to change +those resources. That includes being the code being able to call functions +like <strong>getpwuid()</strong>, <strong>getgrgid()</strong>, <strong>getpwname()</strong>, and <strong>getgrnam()</strong>. +You should test what libraries and config files are required for your OS +and get those setup before starting to test name mapping in rsync.</p> +</dd> + +<dt><code>munge symlinks</code></dt><dd> +<p>This parameter tells rsync to modify all symlinks in the same way as the +(non-daemon-affecting) <code>--munge-links</code> command-line option (using a method +described below). This should help protect your files from user trickery +when your daemon module is writable. The default is disabled when +"use chroot" is on with an inside-chroot path of "/", OR if "daemon chroot" +is on, otherwise it is enabled.</p> +<p>If you disable this parameter on a daemon that is not read-only, there are +tricks that a user can play with uploaded symlinks to access +daemon-excluded items (if your module has any), and, if "use chroot" is +off, rsync can even be tricked into showing or changing data that is +outside the module's path (as access-permissions allow).</p> +<p>The way rsync disables the use of symlinks is to prefix each one with the +string "/rsyncd-munged/". This prevents the links from being used as long +as that directory does not exist. When this parameter is enabled, rsync +will refuse to run if that path is a directory or a symlink to a directory. +When using the "munge symlinks" parameter in a chroot area that has an +inside-chroot path of "/", you should add "/rsyncd-munged/" to the exclude +setting for the module so that a user can't try to create it.</p> +<p>Note: rsync makes no attempt to verify that any pre-existing symlinks in +the module's hierarchy are as safe as you want them to be (unless, of +course, it just copied in the whole hierarchy). If you setup an rsync +daemon on a new area or locally add symlinks, you can manually protect your +symlinks from being abused by prefixing "/rsyncd-munged/" to the start of +every symlink's value. There is a perl script in the support directory of +the source code named "munge-symlinks" that can be used to add or remove +this prefix from your symlinks.</p> +<p>When this parameter is disabled on a writable module and "use chroot" is +off (or the inside-chroot path is not "/"), incoming symlinks will be +modified to drop a leading slash and to remove ".." path elements that +rsync believes will allow a symlink to escape the module's hierarchy. +There are tricky ways to work around this, though, so you had better trust +your users if you choose this combination of parameters.</p> +</dd> + +<dt><code>charset</code></dt><dd> +<p>This specifies the name of the character set in which the module's +filenames are stored. If the client uses an <code>--iconv</code> option, the daemon +will use the value of the "charset" parameter regardless of the character +set the client actually passed. This allows the daemon to support charset +conversion in a chroot module without extra files in the chroot area, and +also ensures that name-translation is done in a consistent manner. If the +"charset" parameter is not set, the <code>--iconv</code> option is refused, just as if +"iconv" had been specified via "refuse options".</p> +<p>If you wish to force users to always use <code>--iconv</code> for a particular module, +add "no-iconv" to the "refuse options" parameter. Keep in mind that this +will restrict access to your module to very new rsync clients.</p> +</dd> + +<dt><code>max connections</code></dt><dd> +<p>This parameter allows you to specify the maximum number of simultaneous +connections you will allow. Any clients connecting when the maximum has +been reached will receive a message telling them to try later. The default +is 0, which means no limit. A negative value disables the module. See +also the "lock file" parameter.</p> +</dd> + +<dt><code>log file</code></dt><dd> +<p>When the "log file" parameter is set to a non-empty string, the rsync +daemon will log messages to the indicated file rather than using syslog. +This is particularly useful on systems (such as AIX) where <strong>syslog()</strong> +doesn't work for chrooted programs. The file is opened before <strong>chroot()</strong> +is called, allowing it to be placed outside the transfer. If this value is +set on a per-module basis instead of globally, the global log will still +contain any authorization failures or config-file error messages.</p> +<p>If the daemon fails to open the specified file, it will fall back to using +syslog and output an error about the failure. (Note that the failure to +open the specified log file used to be a fatal error.)</p> +<p>This setting can be overridden by using the <code>--log-file=FILE</code> or +<code>--dparam=logfile=FILE</code> command-line options. The former overrides all the +log-file parameters of the daemon and all module settings. The latter sets +the daemon's log file and the default for all the modules, which still +allows modules to override the default setting.</p> +</dd> + +<dt><code>syslog facility</code></dt><dd> +<p>This parameter allows you to specify the syslog facility name to use when +logging messages from the rsync daemon. You may use any standard syslog +facility name which is defined on your system. Common names are auth, +authpriv, cron, daemon, ftp, kern, lpr, mail, news, security, syslog, user, +uucp, local0, local1, local2, local3, local4, local5, local6 and local7. +The default is daemon. This setting has no effect if the "log file" +setting is a non-empty string (either set in the per-modules settings, or +inherited from the global settings).</p> +</dd> + +<dt><code>syslog tag</code></dt><dd> +<p>This parameter allows you to specify the syslog tag to use when logging +messages from the rsync daemon. The default is "rsyncd". This setting has +no effect if the "log file" setting is a non-empty string (either set in +the per-modules settings, or inherited from the global settings).</p> +<p>For example, if you wanted each authenticated user's name to be included in +the syslog tag, you could do something like this:</p> +<blockquote> +<pre><code>syslog tag = rsyncd.%RSYNC_USER_NAME% +</code></pre> +</blockquote> +</dd> + +<dt><code>max verbosity</code></dt><dd> +<p>This parameter allows you to control the maximum amount of verbose +information that you'll allow the daemon to generate (since the information +goes into the log file). The default is 1, which allows the client to +request one level of verbosity.</p> +<p>This also affects the user's ability to request higher levels of <code>--info</code> +and <code>--debug</code> logging. If the max value is 2, then no info and/or debug +value that is higher than what would be set by <code>-vv</code> will be honored by the +daemon in its logging. To see how high of a verbosity level you need to +accept for a particular info/debug level, refer to <code>rsync --info=help</code> and +<code>rsync --debug=help</code>. For instance, it takes max-verbosity 4 to be able to +output debug TIME2 and FLIST3.</p> +</dd> + +<dt><code>lock file</code></dt><dd> +<p>This parameter specifies the file to use to support the "max connections" +parameter. The rsync daemon uses record locking on this file to ensure that +the max connections limit is not exceeded for the modules sharing the lock +file. The default is <code>/var/run/rsyncd.lock</code>.</p> +</dd> + +<dt><code>read only</code></dt><dd> +<p>This parameter determines whether clients will be able to upload files or +not. If "read only" is true then any attempted uploads will fail. If +"read only" is false then uploads will be possible if file permissions on +the daemon side allow them. The default is for all modules to be read only.</p> +<p>Note that "auth users" can override this setting on a per-user basis.</p> +</dd> + +<dt><code>write only</code></dt><dd> +<p>This parameter determines whether clients will be able to download files or +not. If "write only" is true then any attempted downloads will fail. If +"write only" is false then downloads will be possible if file permissions +on the daemon side allow them. The default is for this parameter to be +disabled.</p> +<p>Helpful hint: you probably want to specify "refuse options = delete" for a +write-only module.</p> +</dd> + +<dt><code>list</code></dt><dd> +<p>This parameter determines whether this module is listed when the client +asks for a listing of available modules. In addition, if this is false, +the daemon will pretend the module does not exist when a client denied by +"hosts allow" or "hosts deny" attempts to access it. Realize that if +"reverse lookup" is disabled globally but enabled for the module, the +resulting reverse lookup to a potentially client-controlled DNS server may +still reveal to the client that it hit an existing module. The default is +for modules to be listable.</p> +</dd> + +<dt><code>uid</code></dt><dd> +<p>This parameter specifies the user name or user ID that file transfers to +and from that module should take place as when the daemon was run as root. +In combination with the "gid" parameter this determines what file +permissions are available. The default when run by a super-user is to +switch to the system's "nobody" user. The default for a non-super-user is +to not try to change the user. See also the "gid" parameter.</p> +<p>The RSYNC_USER_NAME environment variable may be used to request that rsync +run as the authorizing user. For example, if you want a rsync to run as +the same user that was received for the rsync authentication, this setup is +useful:</p> +<blockquote> +<pre><code>uid = %RSYNC_USER_NAME% +gid = * +</code></pre> +</blockquote> +</dd> + +<dt><code>gid</code></dt><dd> +<p>This parameter specifies one or more group names/IDs that will be used when +accessing the module. The first one will be the default group, and any +extra ones be set as supplemental groups. You may also specify a "<code>*</code>" as +the first gid in the list, which will be replaced by all the normal groups +for the transfer's user (see "uid"). The default when run by a super-user +is to switch to your OS's "nobody" (or perhaps "nogroup") group with no +other supplementary groups. The default for a non-super-user is to not +change any group attributes (and indeed, your OS may not allow a +non-super-user to try to change their group settings).</p> +<p>The specified list is normally split into tokens based on spaces and +commas. However, if the list starts with a comma, then the list is only +split on commas, which allows a group name to contain a space. In either +case any leading and/or trailing whitespace is removed from the tokens and +empty tokens are ignored.</p> +</dd> + +<dt><code>daemon uid</code></dt><dd> +<p>This parameter specifies a uid under which the daemon will run. The daemon +usually runs as user root, and when this is left unset the user is left +unchanged. See also the "uid" parameter.</p> +</dd> + +<dt><code>daemon gid</code></dt><dd> +<p>This parameter specifies a gid under which the daemon will run. The daemon +usually runs as group root, and when this is left unset, the group is left +unchanged. See also the "gid" parameter.</p> +</dd> + +<dt><code>fake super</code></dt><dd> +<p>Setting "fake super = yes" for a module causes the daemon side to behave as +if the <code>--fake-super</code> command-line option had been specified. This allows +the full attributes of a file to be stored without having to have the +daemon actually running as root.</p> +</dd> + +<dt><code>filter</code></dt><dd> +<p>The daemon has its own filter chain that determines what files it will let +the client access. This chain is not sent to the client and is independent +of any filters the client may have specified. Files excluded by the daemon +filter chain (<code>daemon-excluded</code> files) are treated as non-existent if the +client tries to pull them, are skipped with an error message if the client +tries to push them (triggering exit code 23), and are never deleted from +the module. You can use daemon filters to prevent clients from downloading +or tampering with private administrative files, such as files you may add +to support uid/gid name translations.</p> +<p>The daemon filter chain is built from the "filter", "include from", +"include", "exclude from", and "exclude" parameters, in that order of +priority. Anchored patterns are anchored at the root of the module. To +prevent access to an entire subtree, for example, "<code>/secret</code>", you <strong>must</strong> +exclude everything in the subtree; the easiest way to do this is with a +triple-star pattern like "<code>/secret/***</code>".</p> +<p>The "filter" parameter takes a space-separated list of daemon filter rules, +though it is smart enough to know not to split a token at an internal space +in a rule (e.g. "<code>- /foo - /bar</code>" is parsed as two rules). You may specify +one or more merge-file rules using the normal syntax. Only one "filter" +parameter can apply to a given module in the config file, so put all the +rules you want in a single parameter. Note that per-directory merge-file +rules do not provide as much protection as global rules, but they can be +used to make <code>--delete</code> work better during a client download operation if +the per-dir merge files are included in the transfer and the client +requests that they be used.</p> +</dd> + +<dt><code>exclude</code></dt><dd> +<p>This parameter takes a space-separated list of daemon exclude patterns. As +with the client <code>--exclude</code> option, patterns can be qualified with "<code>-</code>" or +"<code>+</code>" to explicitly indicate exclude/include. Only one "exclude" parameter +can apply to a given module. See the "filter" parameter for a description +of how excluded files affect the daemon.</p> +</dd> + +<dt><code>include</code></dt><dd> +<p>Use an "include" to override the effects of the "exclude" parameter. Only +one "include" parameter can apply to a given module. See the "filter" +parameter for a description of how excluded files affect the daemon.</p> +</dd> + +<dt><code>exclude from</code></dt><dd> +<p>This parameter specifies the name of a file on the daemon that contains +daemon exclude patterns, one per line. Only one "exclude from" parameter +can apply to a given module; if you have multiple exclude-from files, you +can specify them as a merge file in the "filter" parameter. See the +"filter" parameter for a description of how excluded files affect the +daemon.</p> +</dd> + +<dt><code>include from</code></dt><dd> +<p>Analogue of "exclude from" for a file of daemon include patterns. Only one +"include from" parameter can apply to a given module. See the "filter" +parameter for a description of how excluded files affect the daemon.</p> +</dd> + +<dt><code>incoming chmod</code></dt><dd> +<p>This parameter allows you to specify a set of comma-separated chmod strings +that will affect the permissions of all incoming files (files that are +being received by the daemon). These changes happen after all other +permission calculations, and this will even override destination-default +and/or existing permissions when the client does not specify <code>--perms</code>. +See the description of the <code>--chmod</code> rsync option and the <strong>chmod</strong>(1) +manpage for information on the format of this string.</p> +</dd> + +<dt><code>outgoing chmod</code></dt><dd> +<p>This parameter allows you to specify a set of comma-separated chmod strings +that will affect the permissions of all outgoing files (files that are +being sent out from the daemon). These changes happen first, making the +sent permissions appear to be different than those stored in the filesystem +itself. For instance, you could disable group write permissions on the +server while having it appear to be on to the clients. See the description +of the <code>--chmod</code> rsync option and the <strong>chmod</strong>(1) manpage for information +on the format of this string.</p> +</dd> + +<dt><code>auth users</code></dt><dd> +<p>This parameter specifies a comma and/or space-separated list of +authorization rules. In its simplest form, you list the usernames that +will be allowed to connect to this module. The usernames do not need to +exist on the local system. The rules may contain shell wildcard characters +that will be matched against the username provided by the client for +authentication. If "auth users" is set then the client will be challenged +to supply a username and password to connect to the module. A challenge +response authentication protocol is used for this exchange. The plain text +usernames and passwords are stored in the file specified by the +"secrets file" parameter. The default is for all users to be able to +connect without a password (this is called "anonymous rsync").</p> +<p>In addition to username matching, you can specify groupname matching via a +'@' prefix. When using groupname matching, the authenticating username +must be a real user on the system, or it will be assumed to be a member of +no groups. For example, specifying "@rsync" will match the authenticating +user if the named user is a member of the rsync group.</p> +<p>Finally, options may be specified after a colon (:). The options allow you +to "deny" a user or a group, set the access to "ro" (read-only), or set the +access to "rw" (read/write). Setting an auth-rule-specific ro/rw setting +overrides the module's "read only" setting.</p> +<p>Be sure to put the rules in the order you want them to be matched, because +the checking stops at the first matching user or group, and that is the +only auth that is checked. For example:</p> +<blockquote> +<pre><code>auth users = joe:deny @guest:deny admin:rw @rsync:ro susan joe sam +</code></pre> +</blockquote> +<p>In the above rule, user joe will be denied access no matter what. Any user +that is in the group "guest" is also denied access. The user "admin" gets +access in read/write mode, but only if the admin user is not in group +"guest" (because the admin user-matching rule would never be reached if the +user is in group "guest"). Any other user who is in group "rsync" will get +read-only access. Finally, users susan, joe, and sam get the ro/rw setting +of the module, but only if the user didn't match an earlier group-matching +rule.</p> +<p>If you need to specify a user or group name with a space in it, start your +list with a comma to indicate that the list should only be split on commas +(though leading and trailing whitespace will also be removed, and empty +entries are just ignored). For example:</p> +<blockquote> +<pre><code>auth users = , joe:deny, @Some Group:deny, admin:rw, @RO Group:ro +</code></pre> +</blockquote> +<p>See the description of the secrets file for how you can have per-user +passwords as well as per-group passwords. It also explains how a user can +authenticate using their user password or (when applicable) a group +password, depending on what rule is being authenticated.</p> +<p>See also the section entitled "USING RSYNC-DAEMON FEATURES VIA A REMOTE +SHELL CONNECTION" in <strong>rsync</strong>(1) for information on how handle an +rsyncd.conf-level username that differs from the remote-shell-level +username when using a remote shell to connect to an rsync daemon.</p> +</dd> + +<dt><code>secrets file</code></dt><dd> +<p>This parameter specifies the name of a file that contains the +username:password and/or @groupname:password pairs used for authenticating +this module. This file is only consulted if the "auth users" parameter is +specified. The file is line-based and contains one name:password pair per +line. Any line has a hash (#) as the very first character on the line is +considered a comment and is skipped. The passwords can contain any +characters but be warned that many operating systems limit the length of +passwords that can be typed at the client end, so you may find that +passwords longer than 8 characters don't work.</p> +<p>The use of group-specific lines are only relevant when the module is being +authorized using a matching "@groupname" rule. When that happens, the user +can be authorized via either their "username:password" line or the +"@groupname:password" line for the group that triggered the authentication.</p> +<p>It is up to you what kind of password entries you want to include, either +users, groups, or both. The use of group rules in "auth users" does not +require that you specify a group password if you do not want to use shared +passwords.</p> +<p>There is no default for the "secrets file" parameter, you must choose a +name (such as <code>/etc/rsyncd.secrets</code>). The file must normally not be +readable by "other"; see "strict modes". If the file is not found or is +rejected, no logins for a "user auth" module will be possible.</p> +</dd> + +<dt><code>strict modes</code></dt><dd> +<p>This parameter determines whether or not the permissions on the secrets +file will be checked. If "strict modes" is true, then the secrets file +must not be readable by any user ID other than the one that the rsync +daemon is running under. If "strict modes" is false, the check is not +performed. The default is true. This parameter was added to accommodate +rsync running on the Windows operating system.</p> +</dd> + +<dt><code>hosts allow</code></dt><dd> +<p>This parameter allows you to specify a list of comma- and/or +whitespace-separated patterns that are matched against a connecting +client's hostname and IP address. If none of the patterns match, then the +connection is rejected.</p> +<p>Each pattern can be in one of five forms:</p> +<ul> +<li>a dotted decimal IPv4 address of the form a.b.c.d, or an IPv6 address of +the form a:b:c::d:e:f. In this case the incoming machine's IP address +must match exactly.</li> +<li>an address/mask in the form ipaddr/n where ipaddr is the IP address and n +is the number of one bits in the netmask. All IP addresses which match +the masked IP address will be allowed in.</li> +<li>an address/mask in the form ipaddr/maskaddr where ipaddr is the IP +address and maskaddr is the netmask in dotted decimal notation for IPv4, +or similar for IPv6, e.g. ffff:ffff:ffff:ffff:: instead of /64. All IP +addresses which match the masked IP address will be allowed in.</li> +<li>a hostname pattern using wildcards. If the hostname of the connecting IP +(as determined by a reverse lookup) matches the wildcarded name (using +the same rules as normal unix filename matching), the client is allowed +in. This only works if "reverse lookup" is enabled (the default).</li> +<li>a hostname. A plain hostname is matched against the reverse DNS of the +connecting IP (if "reverse lookup" is enabled), and/or the IP of the +given hostname is matched against the connecting IP (if "forward lookup" +is enabled, as it is by default). Any match will be allowed in.</li> +</ul> +<p>Note IPv6 link-local addresses can have a scope in the address +specification:</p> +<blockquote> +<pre><code>fe80::1%link1 +fe80::%link1/64 +fe80::%link1/ffff:ffff:ffff:ffff:: +</code></pre> +</blockquote> +<p>You can also combine "hosts allow" with a separate "hosts deny" parameter. +If both parameters are specified then the "hosts allow" parameter is +checked first and a match results in the client being able to connect. The +"hosts deny" parameter is then checked and a match means that the host is +rejected. If the host does not match either the "hosts allow" or the +"hosts deny" patterns then it is allowed to connect.</p> +<p>The default is no "hosts allow" parameter, which means all hosts can +connect.</p> +</dd> + +<dt><code>hosts deny</code></dt><dd> +<p>This parameter allows you to specify a list of comma- and/or +whitespace-separated patterns that are matched against a connecting clients +hostname and IP address. If the pattern matches then the connection is +rejected. See the "hosts allow" parameter for more information.</p> +<p>The default is no "hosts deny" parameter, which means all hosts can +connect.</p> +</dd> + +<dt><code>reverse lookup</code></dt><dd> +<p>Controls whether the daemon performs a reverse lookup on the client's IP +address to determine its hostname, which is used for "hosts allow" & +"hosts deny" checks and the "%h" log escape. This is enabled by default, +but you may wish to disable it to save time if you know the lookup will not +return a useful result, in which case the daemon will use the name +"UNDETERMINED" instead.</p> +<p>If this parameter is enabled globally (even by default), rsync performs the +lookup as soon as a client connects, so disabling it for a module will not +avoid the lookup. Thus, you probably want to disable it globally and then +enable it for modules that need the information.</p> +</dd> + +<dt><code>forward lookup</code></dt><dd> +<p>Controls whether the daemon performs a forward lookup on any hostname +specified in an hosts allow/deny setting. By default this is enabled, +allowing the use of an explicit hostname that would not be returned by +reverse DNS of the connecting IP.</p> +</dd> + +<dt><code>ignore errors</code></dt><dd> +<p>This parameter tells rsyncd to ignore I/O errors on the daemon when +deciding whether to run the delete phase of the transfer. Normally rsync +skips the <code>--delete</code> step if any I/O errors have occurred in order to +prevent disastrous deletion due to a temporary resource shortage or other +I/O error. In some cases this test is counter productive so you can use +this parameter to turn off this behavior.</p> +</dd> + +<dt><code>ignore nonreadable</code></dt><dd> +<p>This tells the rsync daemon to completely ignore files that are not +readable by the user. This is useful for public archives that may have some +non-readable files among the directories, and the sysadmin doesn't want +those files to be seen at all.</p> +</dd> + +<dt><code>transfer logging</code></dt><dd> +<p>This parameter enables per-file logging of downloads and uploads in a +format somewhat similar to that used by ftp daemons. The daemon always +logs the transfer at the end, so if a transfer is aborted, no mention will +be made in the log file.</p> +<p>If you want to customize the log lines, see the "log format" parameter.</p> +</dd> + +<dt><code>log format</code></dt><dd> +<p>This parameter allows you to specify the format used for logging file +transfers when transfer logging is enabled. The format is a text string +containing embedded single-character escape sequences prefixed with a +percent (%) character. An optional numeric field width may also be +specified between the percent and the escape letter (e.g. +"<code>%-50n %8l %07p</code>"). In addition, one or more apostrophes may be specified +prior to a numerical escape to indicate that the numerical value should be +made more human-readable. The 3 supported levels are the same as for the +<code>--human-readable</code> command-line option, though the default is for +human-readability to be off. Each added apostrophe increases the level +(e.g. "<code>%''l %'b %f</code>").</p> +<p>The default log format is "<code>%o %h [%a] %m (%u) %f %l</code>", and a "<code>%t [%p]</code>" +is always prefixed when using the "log file" parameter. (A perl script +that will summarize this default log format is included in the rsync source +code distribution in the "support" subdirectory: rsyncstats.)</p> +<p>The single-character escapes that are understood are as follows:</p> +<ul> +<li>%a the remote IP address (only available for a daemon)</li> +<li>%b the number of bytes actually transferred</li> +<li>%B the permission bits of the file (e.g. rwxrwxrwt)</li> +<li>%c the total size of the block checksums received for the basis file +(only when sending)</li> +<li>%C the full-file checksum if it is known for the file. For older rsync +protocols/versions, the checksum was salted, and is thus not a useful +value (and is not displayed when that is the case). For the checksum to +output for a file, either the <code>--checksum</code> option must be in-effect or +the file must have been transferred without a salted checksum being used. +See the <code>--checksum-choice</code> option for a way to choose the algorithm.</li> +<li>%f the filename (long form on sender; no trailing "/")</li> +<li>%G the gid of the file (decimal) or "DEFAULT"</li> +<li>%h the remote host name (only available for a daemon)</li> +<li>%i an itemized list of what is being updated</li> +<li>%l the length of the file in bytes</li> +<li>%L the string "<code>-> SYMLINK</code>", "<code>=> HARDLINK</code>", or "" (where <code>SYMLINK</code> +or <code>HARDLINK</code> is a filename)</li> +<li>%m the module name</li> +<li>%M the last-modified time of the file</li> +<li>%n the filename (short form; trailing "/" on dir)</li> +<li>%o the operation, which is "send", "recv", or "del." (the latter includes +the trailing period)</li> +<li>%p the process ID of this rsync session</li> +<li>%P the module path</li> +<li>%t the current date time</li> +<li>%u the authenticated username or an empty string</li> +<li>%U the uid of the file (decimal)</li> +</ul> +<p>For a list of what the characters mean that are output by "%i", see the +<code>--itemize-changes</code> option in the rsync manpage.</p> +<p>Note that some of the logged output changes when talking with older rsync +versions. For instance, deleted files were only output as verbose messages +prior to rsync 2.6.4.</p> +</dd> + +<dt><code>timeout</code></dt><dd> +<p>This parameter allows you to override the clients choice for I/O timeout +for this module. Using this parameter you can ensure that rsync won't wait +on a dead client forever. The timeout is specified in seconds. A value of +zero means no timeout and is the default. A good choice for anonymous rsync +daemons may be 600 (giving a 10 minute timeout).</p> +</dd> + +<dt><code>refuse options</code></dt><dd> +<p>This parameter allows you to specify a space-separated list of rsync +command-line options that will be refused by your rsync daemon. You may +specify the full option name, its one-letter abbreviation, or a wild-card +string that matches multiple options. Beginning in 3.2.0, you can also +negate a match term by starting it with a "!".</p> +<p>When an option is refused, the daemon prints an error message and exits.</p> +<p>For example, this would refuse <code>--checksum</code> (<code>-c</code>) and all the various +delete options:</p> +<blockquote> +<pre><code>refuse options = c delete +</code></pre> +</blockquote> +<p>The reason the above refuses all delete options is that the options imply +<code>--delete</code>, and implied options are refused just like explicit options.</p> +<p>The use of a negated match allows you to fine-tune your refusals after a +wild-card, such as this:</p> +<blockquote> +<pre><code>refuse options = delete-* !delete-during +</code></pre> +</blockquote> +<p>Negated matching can also turn your list of refused options into a list of +accepted options. To do this, begin the list with a "<code>*</code>" (to refuse all +options) and then specify one or more negated matches to accept. For +example:</p> +<blockquote> +<pre><code>refuse options = * !a !v !compress* +</code></pre> +</blockquote> +<p>Don't worry that the "<code>*</code>" will refuse certain vital options such as +<code>--dry-run</code>, <code>--server</code>, <code>--no-iconv</code>, <code>--protect-args</code>, etc. These +important options are not matched by wild-card, so they must be overridden +by their exact name. For instance, if you're forcing iconv transfers you +could use something like this:</p> +<blockquote> +<pre><code>refuse options = * no-iconv !a !v +</code></pre> +</blockquote> +<p>As an additional aid (beginning in 3.2.0), refusing (or "<code>!refusing</code>") the +"a" or "archive" option also affects all the options that the <code>--archive</code> +option implies (<code>-rdlptgoD</code>), but only if the option is matched explicitly +(not using a wildcard). If you want to do something tricky, you can use +"<code>archive*</code>" to avoid this side-effect, but keep in mind that no normal +rsync client ever sends the actual archive option to the server.</p> +<p>As an additional safety feature, the refusal of "delete" also refuses +<code>remove-source-files</code> when the daemon is the sender; if you want the latter +without the former, instead refuse "<code>delete-*</code>" as that refuses all the +delete modes without affecting <code>--remove-source-files</code>. (Keep in mind that +the client's <code>--delete</code> option typically results in <code>--delete-during</code>.)</p> +<p>When un-refusing delete options, you should either specify "<code>!delete*</code>" (to +accept all delete options) or specify a limited set that includes "delete", +such as:</p> +<blockquote> +<pre><code>refuse options = * !a !delete !delete-during +</code></pre> +</blockquote> +<p>... whereas this accepts any delete option except <code>--delete-after</code>:</p> +<blockquote> +<pre><code>refuse options = * !a !delete* delete-after +</code></pre> +</blockquote> +<p>A note on refusing "compress" -⁠-⁠ it is better to set the "dont compress" +daemon parameter to "<code>*</code>" because that disables compression silently +instead of returning an error that forces the client to remove the <code>-z</code> +option.</p> +<p>If you are un-refusing the compress option, you probably want to match +"<code>!compress*</code>" so that you also accept the <code>--compress-level</code> option.</p> +<p>Note that the "write-devices" option is refused by default, but can be +explicitly accepted with "<code>!write-devices</code>". The options "log-file" and +"log-file-format" are forcibly refused and cannot be accepted.</p> +<p>Here are all the options that are not matched by wild-cards:</p> +<ul> +<li><code>--server</code>: Required for rsync to even work.</li> +<li><code>--rsh</code>, <code>-e</code>: Required to convey compatibility flags to the server.</li> +<li><code>--log-format</code>: This is required to convey things like +<code>--itemize-changes</code> to a remote receiver. Is an older name for +<code>--out-format</code> that is still passed to the server for improved backward +compatibility and should not be confused with <code>--log-file-format</code>.</li> +<li><code>--sender</code>: Use "write only" parameter instead of refusing this.</li> +<li><code>--dry-run</code>, <code>-n</code>: Who would want to disable this?</li> +<li><code>--protect-args</code>, <code>-n</code>: This actually makes transfers safer.</li> +<li><code>--from0</code>, <code>-0</code>: Make it easier to accept/refuse <code>--files-from</code> without +affecting this modifier.</li> +<li><code>--iconv</code>: This is auto-disabled based on "charset" parameter.</li> +<li><code>--no-iconv</code>: Most transfers use this option.</li> +<li><code>--checksum-seed</code>: Is a fairly rare, safe option.</li> +<li><code>--write-devices</code>: Is non-wild but also auto-disabled.</li> +</ul> +</dd> + +<dt><code>dont compress</code></dt><dd> +<p>This parameter allows you to select filenames based on wildcard patterns +that should not be compressed when pulling files from the daemon (no +analogous parameter exists to govern the pushing of files to a daemon). +Compression can be expensive in terms of CPU usage, so it is usually good +to not try to compress files that won't compress well, such as already +compressed files.</p> +<p>The "dont compress" parameter takes a space-separated list of +case-insensitive wildcard patterns. Any source filename matching one of the +patterns will be compressed as little as possible during the transfer. If +the compression algorithm has an "off" level (such as zlib/zlibx) then no +compression occurs for those files. Other algorithms have the level +minimized to reduces the CPU usage as much as possible.</p> +<p>See the <code>--skip-compress</code> parameter in the <strong>rsync</strong>(1) manpage for the +list of file suffixes that are not compressed by default. Specifying a +value for the "dont compress" parameter changes the default when the daemon +is the sender.</p> +</dd> + +<dt><code>early exec</code>, <code>pre-xfer exec</code>, <code>post-xfer exec</code></dt><dd> +<p>You may specify a command to be run in the early stages of the connection, +or right before and/or after the transfer. If the <code>early exec</code> or +<code>pre-xfer exec</code> command returns an error code, the transfer is aborted +before it begins. Any output from the <code>pre-xfer exec</code> command on stdout +(up to several KB) will be displayed to the user when aborting, but is +<u>not</u> displayed if the script returns success. The other programs cannot +send any text to the user. All output except for the <code>pre-xfer exec</code> +stdout goes to the corresponding daemon's stdout/stderr, which is typically +discarded. See the <code>--no-detatch</code> option for a way to see the daemon's +output, which can assist with debugging.</p> +<p>Note that the <code>early exec</code> command runs before any part of the transfer +request is known except for the module name. This helper script can be +used to setup a disk mount or decrypt some data into a module dir, but you +may need to use <code>lock file</code> and <code>max connections</code> to avoid concurrency +issues.</p> +<p>Note that the <code>post-xfer exec</code> command is still run even if one of the +other scripts returns an error code. The <code>pre-xfer exec</code> command will <u>not</u> +be run, however, if the <code>early exec</code> command fails.</p> +<p>The following environment variables will be set, though some are specific +to the pre-xfer or the post-xfer environment:</p> +<ul> +<li><code>RSYNC_MODULE_NAME</code>: The name of the module being accessed.</li> +<li><code>RSYNC_MODULE_PATH</code>: The path configured for the module.</li> +<li><code>RSYNC_HOST_ADDR</code>: The accessing host's IP address.</li> +<li><code>RSYNC_HOST_NAME</code>: The accessing host's name.</li> +<li><code>RSYNC_USER_NAME</code>: The accessing user's name (empty if no user).</li> +<li><code>RSYNC_PID</code>: A unique number for this transfer.</li> +<li><code>RSYNC_REQUEST</code>: (pre-xfer only) The module/path info specified by the +user. Note that the user can specify multiple source files, so the +request can be something like "mod/path1 mod/path2", etc.</li> +<li><code>RSYNC_ARG#</code>: (pre-xfer only) The pre-request arguments are set in these +numbered values. RSYNC_ARG0 is always "rsyncd", followed by the options +that were used in RSYNC_ARG1, and so on. There will be a value of "." +indicating that the options are done and the path args are beginning -⁠-⁠ +these contain similar information to RSYNC_REQUEST, but with values +separated and the module name stripped off.</li> +<li><code>RSYNC_EXIT_STATUS</code>: (post-xfer only) the server side's exit value. This +will be 0 for a successful run, a positive value for an error that the +server generated, or a -⁠1 if rsync failed to exit properly. Note that an +error that occurs on the client side does not currently get sent to the +server side, so this is not the final exit status for the whole transfer.</li> +<li><code>RSYNC_RAW_STATUS</code>: (post-xfer only) the raw exit value from +<strong>waitpid()</strong>.</li> +</ul> +<p>Even though the commands can be associated with a particular module, they +are run using the permissions of the user that started the daemon (not the +module's uid/gid setting) without any chroot restrictions.</p> +<p>These settings honor 2 environment variables: use RSYNC_SHELL to set a +shell to use when running the command (which otherwise uses your +<strong>system()</strong> call's default shell), and use RSYNC_NO_XFER_EXEC to disable +both options completely.</p> +</dd> +</dl> +<h1>CONFIG DIRECTIVES</h1> +<p>There are currently two config directives available that allow a config file to +incorporate the contents of other files: <code>&include</code> and <code>&merge</code>. Both allow +a reference to either a file or a directory. They differ in how segregated the +file's contents are considered to be.</p> +<p>The <code>&include</code> directive treats each file as more distinct, with each one +inheriting the defaults of the parent file, starting the parameter parsing as +globals/defaults, and leaving the defaults unchanged for the parsing of the +rest of the parent file.</p> +<p>The <code>&merge</code> directive, on the other hand, treats the file's contents as if it +were simply inserted in place of the directive, and thus it can set parameters +in a module started in another file, can affect the defaults for other files, +etc.</p> +<p>When an <code>&include</code> or <code>&merge</code> directive refers to a directory, it will read in +all the <code>*.conf</code> or <code>*.inc</code> files (respectively) that are contained inside that +directory (without any recursive scanning), with the files sorted into alpha +order. So, if you have a directory named "rsyncd.d" with the files "foo.conf", +"bar.conf", and "baz.conf" inside it, this directive:</p> +<blockquote> +<pre><code>&include /path/rsyncd.d +</code></pre> +</blockquote> +<p>would be the same as this set of directives:</p> +<blockquote> +<pre><code>&include /path/rsyncd.d/bar.conf +&include /path/rsyncd.d/baz.conf +&include /path/rsyncd.d/foo.conf +</code></pre> +</blockquote> +<p>except that it adjusts as files are added and removed from the directory.</p> +<p>The advantage of the <code>&include</code> directive is that you can define one or more +modules in a separate file without worrying about unintended side-effects +between the self-contained module files.</p> +<p>The advantage of the <code>&merge</code> directive is that you can load config snippets +that can be included into multiple module definitions, and you can also set +global values that will affect connections (such as <code>motd file</code>), or globals +that will affect other include files.</p> +<p>For example, this is a useful /etc/rsyncd.conf file:</p> +<blockquote> +<pre><code>port = 873 +log file = /var/log/rsync.log +pid file = /var/lock/rsync.lock + +&merge /etc/rsyncd.d +&include /etc/rsyncd.d +</code></pre> +</blockquote> +<p>This would merge any <code>/etc/rsyncd.d/*.inc</code> files (for global values that should +stay in effect), and then include any <code>/etc/rsyncd.d/*.conf</code> files (defining +modules without any global-value cross-talk).</p> +<h1>AUTHENTICATION STRENGTH</h1> +<p>The authentication protocol used in rsync is a 128 bit MD4 based challenge +response system. This is fairly weak protection, though (with at least one +brute-force hash-finding algorithm publicly available), so if you want really +top-quality security, then I recommend that you run rsync over ssh. (Yes, a +future version of rsync will switch over to a stronger hashing method.)</p> +<p>Also note that the rsync daemon protocol does not currently provide any +encryption of the data that is transferred over the connection. Only +authentication is provided. Use ssh as the transport if you want encryption.</p> +<p>You can also make use of SSL/TLS encryption if you put rsync behind an +SSL proxy.</p> +<h1>SSL/TLS Daemon Setup</h1> +<p>When setting up an rsync daemon for access via SSL/TLS, you will need to +configure a proxy (such as haproxy or nginx) as the front-end that handles the +encryption.</p> +<ul> +<li>You should limit the access to the backend-rsyncd port to only allow the +proxy to connect. If it is on the same host as the proxy, then configuring +it to only listen on localhost is a good idea.</li> +<li>You should consider turning on the <code>proxy protocol</code> parameter if your proxy +supports sending that information. The examples below assume that this is +enabled.</li> +</ul> +<p>An example haproxy setup is as follows:</p> +<blockquote> +<pre><code>frontend fe_rsync-ssl + bind :::874 ssl crt /etc/letsencrypt/example.com/combined.pem + mode tcp + use_backend be_rsync + +backend be_rsync + mode tcp + server local-rsync 127.0.0.1:873 check send-proxy +</code></pre> +</blockquote> +<p>An example nginx proxy setup is as follows:</p> +<blockquote> +<pre><code>stream { + server { + listen 874 ssl; + listen [::]:874 ssl; + + ssl_certificate /etc/letsencrypt/example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem + + proxy_pass localhost:873; + proxy_protocol on; # Requires "proxy protocol = true" + proxy_timeout 1m; + proxy_connect_timeout 5s; + } +} +</code></pre> +</blockquote> +<h1>EXAMPLES</h1> +<p>A simple rsyncd.conf file that allow anonymous rsync to a ftp area at +<code>/home/ftp</code> would be:</p> +<blockquote> +<pre><code>[ftp] + path = /home/ftp + comment = ftp export area +</code></pre> +</blockquote> +<p>A more sophisticated example would be:</p> +<blockquote> +<pre><code>uid = nobody +gid = nobody +use chroot = yes +max connections = 4 +syslog facility = local5 +pid file = /var/run/rsyncd.pid + +[ftp] + path = /var/ftp/./pub + comment = whole ftp area (approx 6.1 GB) + +[sambaftp] + path = /var/ftp/./pub/samba + comment = Samba ftp area (approx 300 MB) + +[rsyncftp] + path = /var/ftp/./pub/rsync + comment = rsync ftp area (approx 6 MB) + +[sambawww] + path = /public_html/samba + comment = Samba WWW pages (approx 240 MB) + +[cvs] + path = /data/cvs + comment = CVS repository (requires authentication) + auth users = tridge, susan + secrets file = /etc/rsyncd.secrets +</code></pre> +</blockquote> +<p>The /etc/rsyncd.secrets file would look something like this:</p> +<blockquote> +<pre><code>tridge:mypass +susan:herpass +</code></pre> +</blockquote> +<h1>FILES</h1> +<p>/etc/rsyncd.conf or rsyncd.conf</p> +<h1>SEE ALSO</h1> +<p><strong>rsync</strong>(1), <strong>rsync-ssl</strong>(1)</p> +<h1>BUGS</h1> +<p>Please report bugs! The rsync bug tracking system is online at +<a href="http://rsync.samba.org/">http://rsync.samba.org/</a>.</p> +<h1>VERSION</h1> +<p>This man page is current for version 3.2.0 of rsync.</p> +<h1>CREDITS</h1> +<p>rsync is distributed under the GNU General Public License. See the file +COPYING for details.</p> +<p>The primary ftp site for rsync is <a href="ftp://rsync.samba.org/pub/rsync">ftp://rsync.samba.org/pub/rsync</a></p> +<p>A web site is available at <a href="http://rsync.samba.org/">http://rsync.samba.org/</a>.</p> +<p>We would be delighted to hear from you if you like this program.</p> +<p>This program uses the zlib compression library written by Jean-loup Gailly and +Mark Adler.</p> +<h1>THANKS</h1> +<p>Thanks to Warren Stanley for his original idea and patch for the rsync daemon. +Thanks to Karsten Thygesen for his many suggestions and documentation!</p> +<h1>AUTHOR</h1> +<p>rsync was written by Andrew Tridgell and Paul Mackerras. Many people have +later contributed to it.</p> +<p>Mailing lists for support and development are available at +<a href="http://lists.samba.org/">http://lists.samba.org/</a>.</p> +<div style="float: right"><p><i>19 Jun 2020</i></p></div> +</body></html> diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md new file mode 100644 index 00000000..30774222 --- /dev/null +++ b/rsyncd.conf.5.md @@ -0,0 +1,1207 @@ +# NAME + +rsyncd.conf - configuration file for rsync in daemon mode + +# SYNOPSIS + +rsyncd.conf + +# DESCRIPTION + +The rsyncd.conf file is the runtime configuration file for rsync when run as an +rsync daemon. + +The rsyncd.conf file controls authentication, access, logging and available +modules. + +# FILE FORMAT + +The file consists of modules and parameters. A module begins with the name of +the module in square brackets and continues until the next module begins. +Modules contain parameters of the form `name = value`. + +The file is line-based -- that is, each newline-terminated line represents +either a comment, a module name or a parameter. + +Only the first equals sign in a parameter is significant. Whitespace before or +after the first equals sign is discarded. Leading, trailing and internal +whitespace in module and parameter names is irrelevant. Leading and trailing +whitespace in a parameter value is discarded. Internal whitespace within a +parameter value is retained verbatim. + +Any line **beginning** with a hash (`#`) is ignored, as are lines containing +only whitespace. (If a hash occurs after anything other than leading +whitespace, it is considered a part of the line's content.) + +Any line ending in a `\` is "continued" on the next line in the customary UNIX +fashion. + +The values following the equals sign in parameters are all either a string (no +quotes needed) or a boolean, which may be given as yes/no, 0/1 or true/false. +Case is not significant in boolean values, but is preserved in string values. + +# LAUNCHING THE RSYNC DAEMON + +The rsync daemon is launched by specifying the `--daemon` option to +rsync. + +The daemon must run with root privileges if you wish to use chroot, to bind to +a port numbered under 1024 (as is the default 873), or to set file ownership. +Otherwise, it must just have permission to read and write the appropriate data, +log, and lock files. + +You can launch it either via inetd, as a stand-alone daemon, or from an rsync +client via a remote shell. If run as a stand-alone daemon then just run the +command "`rsync --daemon`" from a suitable startup script. + +When run via inetd you should add a line like this to /etc/services: + +> rsync 873/tcp + +and a single line something like this to /etc/inetd.conf: + +> rsync stream tcp nowait root /usr/bin/rsync rsyncd --daemon + +Replace "/usr/bin/rsync" with the path to where you have rsync installed on +your system. You will then need to send inetd a HUP signal to tell it to +reread its config file. + +Note that you should **not** send the rsync daemon a HUP signal to force it to +reread the `rsyncd.conf` file. The file is re-read on each client connection. + +# GLOBAL PARAMETERS + +The first parameters in the file (before a [module] header) are the global +parameters. Rsync also allows for the use of a "[global]" module name to +indicate the start of one or more global-parameter sections (the name must be +lower case). + +You may also include any module parameters in the global part of the config +file in which case the supplied value will override the default for that +parameter. + +You may use references to environment variables in the values of parameters. +String parameters will have %VAR% references expanded as late as possible (when +the string is first used in the program), allowing for the use of variables +that rsync sets at connection time, such as RSYNC_USER_NAME. Non-string +parameters (such as true/false settings) are expanded when read from the config +file. If a variable does not exist in the environment, or if a sequence of +characters is not a valid reference (such as an un-paired percent sign), the +raw characters are passed through unchanged. This helps with backward +compatibility and safety (e.g. expanding a non-existent %VAR% to an empty +string in a path could result in a very unsafe path). The safest way to insert +a literal % into a value is to use %%. + +[comment]: # (An OL starting at 0 is converted into a DL by the parser.) + +0. `motd file` + + This parameter allows you to specify a "message of the day" to display to + clients on each connect. This usually contains site information and any + legal notices. The default is no motd file. This can be overridden by the + `--dparam=motdfile=FILE` command-line option when starting the daemon. + +0. `pid file` + + This parameter tells the rsync daemon to write its process ID to that file. + The rsync keeps the file locked so that it can know when it is safe to + overwrite an existing file. + + The filename can be overridden by the `--dparam=pidfile=FILE` command-line + option when starting the daemon. + +0. `port` + + You can override the default port the daemon will listen on by specifying + this value (defaults to 873). This is ignored if the daemon is being run + by inetd, and is superseded by the `--port` command-line option. + +0. `address` + + You can override the default IP address the daemon will listen on by + specifying this value. This is ignored if the daemon is being run by + inetd, and is superseded by the `--address` command-line option. + +0. `socket options` + + This parameter can provide endless fun for people who like to tune their + systems to the utmost degree. You can set all sorts of socket options which + may make transfers faster (or slower!). Read the man page for the + **setsockopt()** system call for details on some of the options you may be + able to set. By default no special socket options are set. These settings + can also be specified via the `--sockopts` command-line option. + +0. `listen backlog` + + You can override the default backlog value when the daemon listens for + connections. It defaults to 5. + +# MODULE PARAMETERS + +After the global parameters you should define a number of modules, each module +exports a directory tree as a symbolic name. Modules are exported by specifying +a module name in square brackets [module] followed by the parameters for that +module. The module name cannot contain a slash or a closing square bracket. +If the name contains whitespace, each internal sequence of whitespace will be +changed into a single space, while leading or trailing whitespace will be +discarded. Also, the name cannot be "global" as that exact name indicates that +global parameters follow (see above). + +As with GLOBAL PARAMETERS, you may use references to environment variables in +the values of parameters. See the GLOBAL PARAMETERS section for more details. + +0. `comment` + + This parameter specifies a description string that is displayed next to the + module name when clients obtain a list of available modules. The default is + no comment. + +0. `path` + + This parameter specifies the directory in the daemon's filesystem to make + available in this module. You must specify this parameter for each module + in `rsyncd.conf`. + + You may base the path's value off of an environment variable by surrounding + the variable name with percent signs. You can even reference a variable + that is set by rsync when the user connects. For example, this would use + the authorizing user's name in the path: + + > path = /home/%RSYNC_USER_NAME% + + It is fine if the path includes internal spaces -- they will be retained + verbatim (which means that you shouldn't try to escape them). If your + final directory has a trailing space (and this is somehow not something you + wish to fix), append a trailing slash to the path to avoid losing the + trailing whitespace. + +0. `use chroot` + + If "use chroot" is true, the rsync daemon will chroot to the "path" before + starting the file transfer with the client. This has the advantage of + extra protection against possible implementation security holes, but it has + the disadvantages of requiring super-user privileges, of not being able to + follow symbolic links that are either absolute or outside of the new root + path, and of complicating the preservation of users and groups by name (see + below). + + As an additional safety feature, you can specify a dot-dir in the module's + "path" to indicate the point where the chroot should occur. This allows + rsync to run in a chroot with a non-"/" path for the top of the transfer + hierarchy. Doing this guards against unintended library loading (since + those absolute paths will not be inside the transfer hierarchy unless you + have used an unwise pathname), and lets you setup libraries for the chroot + that are outside of the transfer. For example, specifying + "/var/rsync/./module1" will chroot to the "/var/rsync" directory and set + the inside-chroot path to "/module1". If you had omitted the dot-dir, the + chroot would have used the whole path, and the inside-chroot path would + have been "/". + + When both "use chroot" and "daemon chroot" are false, OR the inside-chroot + path of "use chroot" is not "/", rsync will: (1) munge symlinks by default + for security reasons (see "munge symlinks" for a way to turn this off, but + only if you trust your users), (2) substitute leading slashes in absolute + paths with the module's path (so that options such as `--backup-dir`, + `--compare-dest`, etc. interpret an absolute path as rooted in the module's + "path" dir), and (3) trim ".." path elements from args if rsync believes + they would escape the module hierarchy. The default for "use chroot" is + true, and is the safer choice (especially if the module is not read-only). + + When this parameter is enabled, the "numeric-ids" option will also default + to being enabled (disabling name lookups). See below for what a chroot + needs in order for name lookups to succeed. + + If you copy library resources into the module's chroot area, you should + protect them through your OS's normal user/group or ACL settings (to + prevent the rsync module's user from being able to change them), and then + hide them from the user's view via "exclude" (see how in the discussion of + that parameter). At that point it will be safe to enable the mapping of + users and groups by name using the "numeric ids" daemon parameter (see + below). + + Note also that you are free to setup custom user/group information in the + chroot area that is different from your normal system. For example, you + could abbreviate the list of users and groups. + +0. `daemon chroot` + + This parameter specifies a path to which the daemon will chroot before + beginning communication with clients. Module paths (and any "use chroot" + settings) will then be related to this one. This lets you choose if you + want the whole daemon to be chrooted (with this setting), just the + transfers to be chrooted (with "use chroot"), or both. Keep in mind that + the "daemon chroot" area may need various OS/lib/etc files installed to + allow the daemon to function. By default the daemon runs without any + chrooting. + +0. `proxy protocol` + + When this parameter is enabled, all incoming connections must start with a + V1 or V2 proxy protocol header. If the header is not found, the connection + is closed. + + Setting this to `true` requires a proxy server to forward source IP + information to rsync, allowing you to log proper IP/host info and make use + of client-oriented IP restrictions. The default of `false` means that the + IP information comes directly from the socket's metadata. If rsync is not + behind a proxy, this should be disabled. + + _CAUTION_: using this option can be dangerous if you do not ensure that + only the proxy is allowed to connect to the rsync port. If any non-proxied + connections are allowed through, the client will be able to use a modified + rsync to spoof any remote IP address that they desire. You can lock this + down using something like iptables `-uid-owner root` rules (for strict + localhost access), various firewall rules, or you can require password + authorization so that any spoofing by users will not grant extra access. + + This setting is global. If you need some modules to require this and not + others, then you will need to setup multiple rsync daemon processes on + different ports. + +0. `numeric ids` + + Enabling this parameter disables the mapping of users and groups by name + for the current daemon module. This prevents the daemon from trying to + load any user/group-related files or libraries. This enabling makes the + transfer behave as if the client had passed the `--numeric-ids` + command-line option. By default, this parameter is enabled for chroot + modules and disabled for non-chroot modules. Also keep in mind that + uid/gid preservation requires the module to be running as root (see "uid") + or for "fake super" to be configured. + + A chroot-enabled module should not have this parameter enabled unless + you've taken steps to ensure that the module has the necessary resources it + needs to translate names, and that it is not possible for a user to change + those resources. That includes being the code being able to call functions + like **getpwuid()**, **getgrgid()**, **getpwname()**, and **getgrnam()**. + You should test what libraries and config files are required for your OS + and get those setup before starting to test name mapping in rsync. + +0. `munge symlinks` + + This parameter tells rsync to modify all symlinks in the same way as the + (non-daemon-affecting) `--munge-links` command-line option (using a method + described below). This should help protect your files from user trickery + when your daemon module is writable. The default is disabled when + "use chroot" is on with an inside-chroot path of "/", OR if "daemon chroot" + is on, otherwise it is enabled. + + If you disable this parameter on a daemon that is not read-only, there are + tricks that a user can play with uploaded symlinks to access + daemon-excluded items (if your module has any), and, if "use chroot" is + off, rsync can even be tricked into showing or changing data that is + outside the module's path (as access-permissions allow). + + The way rsync disables the use of symlinks is to prefix each one with the + string "/rsyncd-munged/". This prevents the links from being used as long + as that directory does not exist. When this parameter is enabled, rsync + will refuse to run if that path is a directory or a symlink to a directory. + When using the "munge symlinks" parameter in a chroot area that has an + inside-chroot path of "/", you should add "/rsyncd-munged/" to the exclude + setting for the module so that a user can't try to create it. + + Note: rsync makes no attempt to verify that any pre-existing symlinks in + the module's hierarchy are as safe as you want them to be (unless, of + course, it just copied in the whole hierarchy). If you setup an rsync + daemon on a new area or locally add symlinks, you can manually protect your + symlinks from being abused by prefixing "/rsyncd-munged/" to the start of + every symlink's value. There is a perl script in the support directory of + the source code named "munge-symlinks" that can be used to add or remove + this prefix from your symlinks. + + When this parameter is disabled on a writable module and "use chroot" is + off (or the inside-chroot path is not "/"), incoming symlinks will be + modified to drop a leading slash and to remove ".." path elements that + rsync believes will allow a symlink to escape the module's hierarchy. + There are tricky ways to work around this, though, so you had better trust + your users if you choose this combination of parameters. + +0. `charset` + + This specifies the name of the character set in which the module's + filenames are stored. If the client uses an `--iconv` option, the daemon + will use the value of the "charset" parameter regardless of the character + set the client actually passed. This allows the daemon to support charset + conversion in a chroot module without extra files in the chroot area, and + also ensures that name-translation is done in a consistent manner. If the + "charset" parameter is not set, the `--iconv` option is refused, just as if + "iconv" had been specified via "refuse options". + + If you wish to force users to always use `--iconv` for a particular module, + add "no-iconv" to the "refuse options" parameter. Keep in mind that this + will restrict access to your module to very new rsync clients. + +0. `max connections` + + This parameter allows you to specify the maximum number of simultaneous + connections you will allow. Any clients connecting when the maximum has + been reached will receive a message telling them to try later. The default + is 0, which means no limit. A negative value disables the module. See + also the "lock file" parameter. + +0. `log file` + + When the "log file" parameter is set to a non-empty string, the rsync + daemon will log messages to the indicated file rather than using syslog. + This is particularly useful on systems (such as AIX) where **syslog()** + doesn't work for chrooted programs. The file is opened before **chroot()** + is called, allowing it to be placed outside the transfer. If this value is + set on a per-module basis instead of globally, the global log will still + contain any authorization failures or config-file error messages. + + If the daemon fails to open the specified file, it will fall back to using + syslog and output an error about the failure. (Note that the failure to + open the specified log file used to be a fatal error.) + + This setting can be overridden by using the `--log-file=FILE` or + `--dparam=logfile=FILE` command-line options. The former overrides all the + log-file parameters of the daemon and all module settings. The latter sets + the daemon's log file and the default for all the modules, which still + allows modules to override the default setting. + +0. `syslog facility` + + This parameter allows you to specify the syslog facility name to use when + logging messages from the rsync daemon. You may use any standard syslog + facility name which is defined on your system. Common names are auth, + authpriv, cron, daemon, ftp, kern, lpr, mail, news, security, syslog, user, + uucp, local0, local1, local2, local3, local4, local5, local6 and local7. + The default is daemon. This setting has no effect if the "log file" + setting is a non-empty string (either set in the per-modules settings, or + inherited from the global settings). + +0. `syslog tag` + + This parameter allows you to specify the syslog tag to use when logging + messages from the rsync daemon. The default is "rsyncd". This setting has + no effect if the "log file" setting is a non-empty string (either set in + the per-modules settings, or inherited from the global settings). + + For example, if you wanted each authenticated user's name to be included in + the syslog tag, you could do something like this: + + > syslog tag = rsyncd.%RSYNC_USER_NAME% + +0. `max verbosity` + + This parameter allows you to control the maximum amount of verbose + information that you'll allow the daemon to generate (since the information + goes into the log file). The default is 1, which allows the client to + request one level of verbosity. + + This also affects the user's ability to request higher levels of `--info` + and `--debug` logging. If the max value is 2, then no info and/or debug + value that is higher than what would be set by `-vv` will be honored by the + daemon in its logging. To see how high of a verbosity level you need to + accept for a particular info/debug level, refer to `rsync --info=help` and + `rsync --debug=help`. For instance, it takes max-verbosity 4 to be able to + output debug TIME2 and FLIST3. + +0. `lock file` + + This parameter specifies the file to use to support the "max connections" + parameter. The rsync daemon uses record locking on this file to ensure that + the max connections limit is not exceeded for the modules sharing the lock + file. The default is `/var/run/rsyncd.lock`. + +0. `read only` + + This parameter determines whether clients will be able to upload files or + not. If "read only" is true then any attempted uploads will fail. If + "read only" is false then uploads will be possible if file permissions on + the daemon side allow them. The default is for all modules to be read only. + + Note that "auth users" can override this setting on a per-user basis. + +0. `write only` + + This parameter determines whether clients will be able to download files or + not. If "write only" is true then any attempted downloads will fail. If + "write only" is false then downloads will be possible if file permissions + on the daemon side allow them. The default is for this parameter to be + disabled. + + Helpful hint: you probably want to specify "refuse options = delete" for a + write-only module. + +0. `list` + + This parameter determines whether this module is listed when the client + asks for a listing of available modules. In addition, if this is false, + the daemon will pretend the module does not exist when a client denied by + "hosts allow" or "hosts deny" attempts to access it. Realize that if + "reverse lookup" is disabled globally but enabled for the module, the + resulting reverse lookup to a potentially client-controlled DNS server may + still reveal to the client that it hit an existing module. The default is + for modules to be listable. + +0. `uid` + + This parameter specifies the user name or user ID that file transfers to + and from that module should take place as when the daemon was run as root. + In combination with the "gid" parameter this determines what file + permissions are available. The default when run by a super-user is to + switch to the system's "nobody" user. The default for a non-super-user is + to not try to change the user. See also the "gid" parameter. + + The RSYNC_USER_NAME environment variable may be used to request that rsync + run as the authorizing user. For example, if you want a rsync to run as + the same user that was received for the rsync authentication, this setup is + useful: + + > uid = %RSYNC_USER_NAME% + > gid = * + +0. `gid` + + This parameter specifies one or more group names/IDs that will be used when + accessing the module. The first one will be the default group, and any + extra ones be set as supplemental groups. You may also specify a "`*`" as + the first gid in the list, which will be replaced by all the normal groups + for the transfer's user (see "uid"). The default when run by a super-user + is to switch to your OS's "nobody" (or perhaps "nogroup") group with no + other supplementary groups. The default for a non-super-user is to not + change any group attributes (and indeed, your OS may not allow a + non-super-user to try to change their group settings). + + The specified list is normally split into tokens based on spaces and + commas. However, if the list starts with a comma, then the list is only + split on commas, which allows a group name to contain a space. In either + case any leading and/or trailing whitespace is removed from the tokens and + empty tokens are ignored. + +0. `daemon uid` + + This parameter specifies a uid under which the daemon will run. The daemon + usually runs as user root, and when this is left unset the user is left + unchanged. See also the "uid" parameter. + +0. `daemon gid` + + This parameter specifies a gid under which the daemon will run. The daemon + usually runs as group root, and when this is left unset, the group is left + unchanged. See also the "gid" parameter. + +0. `fake super` + + Setting "fake super = yes" for a module causes the daemon side to behave as + if the `--fake-super` command-line option had been specified. This allows + the full attributes of a file to be stored without having to have the + daemon actually running as root. + +0. `filter` + + The daemon has its own filter chain that determines what files it will let + the client access. This chain is not sent to the client and is independent + of any filters the client may have specified. Files excluded by the daemon + filter chain (`daemon-excluded` files) are treated as non-existent if the + client tries to pull them, are skipped with an error message if the client + tries to push them (triggering exit code 23), and are never deleted from + the module. You can use daemon filters to prevent clients from downloading + or tampering with private administrative files, such as files you may add + to support uid/gid name translations. + + The daemon filter chain is built from the "filter", "include from", + "include", "exclude from", and "exclude" parameters, in that order of + priority. Anchored patterns are anchored at the root of the module. To + prevent access to an entire subtree, for example, "`/secret`", you **must** + exclude everything in the subtree; the easiest way to do this is with a + triple-star pattern like "`/secret/***`". + + The "filter" parameter takes a space-separated list of daemon filter rules, + though it is smart enough to know not to split a token at an internal space + in a rule (e.g. "`- /foo - /bar`" is parsed as two rules). You may specify + one or more merge-file rules using the normal syntax. Only one "filter" + parameter can apply to a given module in the config file, so put all the + rules you want in a single parameter. Note that per-directory merge-file + rules do not provide as much protection as global rules, but they can be + used to make `--delete` work better during a client download operation if + the per-dir merge files are included in the transfer and the client + requests that they be used. + +0. `exclude` + + This parameter takes a space-separated list of daemon exclude patterns. As + with the client `--exclude` option, patterns can be qualified with "`- `" or + "`+ `" to explicitly indicate exclude/include. Only one "exclude" parameter + can apply to a given module. See the "filter" parameter for a description + of how excluded files affect the daemon. + +0. `include` + + Use an "include" to override the effects of the "exclude" parameter. Only + one "include" parameter can apply to a given module. See the "filter" + parameter for a description of how excluded files affect the daemon. + +0. `exclude from` + + This parameter specifies the name of a file on the daemon that contains + daemon exclude patterns, one per line. Only one "exclude from" parameter + can apply to a given module; if you have multiple exclude-from files, you + can specify them as a merge file in the "filter" parameter. See the + "filter" parameter for a description of how excluded files affect the + daemon. + +0. `include from` + + Analogue of "exclude from" for a file of daemon include patterns. Only one + "include from" parameter can apply to a given module. See the "filter" + parameter for a description of how excluded files affect the daemon. + +0. `incoming chmod` + + This parameter allows you to specify a set of comma-separated chmod strings + that will affect the permissions of all incoming files (files that are + being received by the daemon). These changes happen after all other + permission calculations, and this will even override destination-default + and/or existing permissions when the client does not specify `--perms`. + See the description of the `--chmod` rsync option and the **chmod**(1) + manpage for information on the format of this string. + +0. `outgoing chmod` + + This parameter allows you to specify a set of comma-separated chmod strings + that will affect the permissions of all outgoing files (files that are + being sent out from the daemon). These changes happen first, making the + sent permissions appear to be different than those stored in the filesystem + itself. For instance, you could disable group write permissions on the + server while having it appear to be on to the clients. See the description + of the `--chmod` rsync option and the **chmod**(1) manpage for information + on the format of this string. + +0. `auth users` + + This parameter specifies a comma and/or space-separated list of + authorization rules. In its simplest form, you list the usernames that + will be allowed to connect to this module. The usernames do not need to + exist on the local system. The rules may contain shell wildcard characters + that will be matched against the username provided by the client for + authentication. If "auth users" is set then the client will be challenged + to supply a username and password to connect to the module. A challenge + response authentication protocol is used for this exchange. The plain text + usernames and passwords are stored in the file specified by the + "secrets file" parameter. The default is for all users to be able to + connect without a password (this is called "anonymous rsync"). + + In addition to username matching, you can specify groupname matching via a + '@' prefix. When using groupname matching, the authenticating username + must be a real user on the system, or it will be assumed to be a member of + no groups. For example, specifying "@rsync" will match the authenticating + user if the named user is a member of the rsync group. + + Finally, options may be specified after a colon (:). The options allow you + to "deny" a user or a group, set the access to "ro" (read-only), or set the + access to "rw" (read/write). Setting an auth-rule-specific ro/rw setting + overrides the module's "read only" setting. + + Be sure to put the rules in the order you want them to be matched, because + the checking stops at the first matching user or group, and that is the + only auth that is checked. For example: + + > auth users = joe:deny @guest:deny admin:rw @rsync:ro susan joe sam + + In the above rule, user joe will be denied access no matter what. Any user + that is in the group "guest" is also denied access. The user "admin" gets + access in read/write mode, but only if the admin user is not in group + "guest" (because the admin user-matching rule would never be reached if the + user is in group "guest"). Any other user who is in group "rsync" will get + read-only access. Finally, users susan, joe, and sam get the ro/rw setting + of the module, but only if the user didn't match an earlier group-matching + rule. + + If you need to specify a user or group name with a space in it, start your + list with a comma to indicate that the list should only be split on commas + (though leading and trailing whitespace will also be removed, and empty + entries are just ignored). For example: + + > auth users = , joe:deny, @Some Group:deny, admin:rw, @RO Group:ro + + See the description of the secrets file for how you can have per-user + passwords as well as per-group passwords. It also explains how a user can + authenticate using their user password or (when applicable) a group + password, depending on what rule is being authenticated. + + See also the section entitled "USING RSYNC-DAEMON FEATURES VIA A REMOTE + SHELL CONNECTION" in **rsync**(1) for information on how handle an + rsyncd.conf-level username that differs from the remote-shell-level + username when using a remote shell to connect to an rsync daemon. + +0. `secrets file` + + This parameter specifies the name of a file that contains the + username:password and/or @groupname:password pairs used for authenticating + this module. This file is only consulted if the "auth users" parameter is + specified. The file is line-based and contains one name:password pair per + line. Any line has a hash (#) as the very first character on the line is + considered a comment and is skipped. The passwords can contain any + characters but be warned that many operating systems limit the length of + passwords that can be typed at the client end, so you may find that + passwords longer than 8 characters don't work. + + The use of group-specific lines are only relevant when the module is being + authorized using a matching "@groupname" rule. When that happens, the user + can be authorized via either their "username:password" line or the + "@groupname:password" line for the group that triggered the authentication. + + It is up to you what kind of password entries you want to include, either + users, groups, or both. The use of group rules in "auth users" does not + require that you specify a group password if you do not want to use shared + passwords. + + There is no default for the "secrets file" parameter, you must choose a + name (such as `/etc/rsyncd.secrets`). The file must normally not be + readable by "other"; see "strict modes". If the file is not found or is + rejected, no logins for a "user auth" module will be possible. + +0. `strict modes` + + This parameter determines whether or not the permissions on the secrets + file will be checked. If "strict modes" is true, then the secrets file + must not be readable by any user ID other than the one that the rsync + daemon is running under. If "strict modes" is false, the check is not + performed. The default is true. This parameter was added to accommodate + rsync running on the Windows operating system. + +0. `hosts allow` + + This parameter allows you to specify a list of comma- and/or + whitespace-separated patterns that are matched against a connecting + client's hostname and IP address. If none of the patterns match, then the + connection is rejected. + + Each pattern can be in one of five forms: + + - a dotted decimal IPv4 address of the form a.b.c.d, or an IPv6 address of + the form a:b:c::d:e:f. In this case the incoming machine's IP address + must match exactly. + - an address/mask in the form ipaddr/n where ipaddr is the IP address and n + is the number of one bits in the netmask. All IP addresses which match + the masked IP address will be allowed in. + - an address/mask in the form ipaddr/maskaddr where ipaddr is the IP + address and maskaddr is the netmask in dotted decimal notation for IPv4, + or similar for IPv6, e.g. ffff:ffff:ffff:ffff:: instead of /64. All IP + addresses which match the masked IP address will be allowed in. + - a hostname pattern using wildcards. If the hostname of the connecting IP + (as determined by a reverse lookup) matches the wildcarded name (using + the same rules as normal unix filename matching), the client is allowed + in. This only works if "reverse lookup" is enabled (the default). + - a hostname. A plain hostname is matched against the reverse DNS of the + connecting IP (if "reverse lookup" is enabled), and/or the IP of the + given hostname is matched against the connecting IP (if "forward lookup" + is enabled, as it is by default). Any match will be allowed in. + + Note IPv6 link-local addresses can have a scope in the address + specification: + + > fe80::1%link1 + > fe80::%link1/64 + > fe80::%link1/ffff:ffff:ffff:ffff:: + + You can also combine "hosts allow" with a separate "hosts deny" parameter. + If both parameters are specified then the "hosts allow" parameter is + checked first and a match results in the client being able to connect. The + "hosts deny" parameter is then checked and a match means that the host is + rejected. If the host does not match either the "hosts allow" or the + "hosts deny" patterns then it is allowed to connect. + + The default is no "hosts allow" parameter, which means all hosts can + connect. + +0. `hosts deny` + + This parameter allows you to specify a list of comma- and/or + whitespace-separated patterns that are matched against a connecting clients + hostname and IP address. If the pattern matches then the connection is + rejected. See the "hosts allow" parameter for more information. + + The default is no "hosts deny" parameter, which means all hosts can + connect. + +0. `reverse lookup` + + Controls whether the daemon performs a reverse lookup on the client's IP + address to determine its hostname, which is used for "hosts allow" & + "hosts deny" checks and the "%h" log escape. This is enabled by default, + but you may wish to disable it to save time if you know the lookup will not + return a useful result, in which case the daemon will use the name + "UNDETERMINED" instead. + + If this parameter is enabled globally (even by default), rsync performs the + lookup as soon as a client connects, so disabling it for a module will not + avoid the lookup. Thus, you probably want to disable it globally and then + enable it for modules that need the information. + +0. `forward lookup` + + Controls whether the daemon performs a forward lookup on any hostname + specified in an hosts allow/deny setting. By default this is enabled, + allowing the use of an explicit hostname that would not be returned by + reverse DNS of the connecting IP. + +0. `ignore errors` + + This parameter tells rsyncd to ignore I/O errors on the daemon when + deciding whether to run the delete phase of the transfer. Normally rsync + skips the `--delete` step if any I/O errors have occurred in order to + prevent disastrous deletion due to a temporary resource shortage or other + I/O error. In some cases this test is counter productive so you can use + this parameter to turn off this behavior. + +0. `ignore nonreadable` + + This tells the rsync daemon to completely ignore files that are not + readable by the user. This is useful for public archives that may have some + non-readable files among the directories, and the sysadmin doesn't want + those files to be seen at all. + +0. `transfer logging` + + This parameter enables per-file logging of downloads and uploads in a + format somewhat similar to that used by ftp daemons. The daemon always + logs the transfer at the end, so if a transfer is aborted, no mention will + be made in the log file. + + If you want to customize the log lines, see the "log format" parameter. + +0. `log format` + + This parameter allows you to specify the format used for logging file + transfers when transfer logging is enabled. The format is a text string + containing embedded single-character escape sequences prefixed with a + percent (%) character. An optional numeric field width may also be + specified between the percent and the escape letter (e.g. + "`%-50n %8l %07p`"). In addition, one or more apostrophes may be specified + prior to a numerical escape to indicate that the numerical value should be + made more human-readable. The 3 supported levels are the same as for the + `--human-readable` command-line option, though the default is for + human-readability to be off. Each added apostrophe increases the level + (e.g. "`%''l %'b %f`"). + + The default log format is "`%o %h [%a] %m (%u) %f %l`", and a "`%t [%p] `" + is always prefixed when using the "log file" parameter. (A perl script + that will summarize this default log format is included in the rsync source + code distribution in the "support" subdirectory: rsyncstats.) + + The single-character escapes that are understood are as follows: + + - %a the remote IP address (only available for a daemon) + - %b the number of bytes actually transferred + - %B the permission bits of the file (e.g. rwxrwxrwt) + - %c the total size of the block checksums received for the basis file + (only when sending) + - %C the full-file checksum if it is known for the file. For older rsync + protocols/versions, the checksum was salted, and is thus not a useful + value (and is not displayed when that is the case). For the checksum to + output for a file, either the `--checksum` option must be in-effect or + the file must have been transferred without a salted checksum being used. + See the `--checksum-choice` option for a way to choose the algorithm. + - %f the filename (long form on sender; no trailing "/") + - %G the gid of the file (decimal) or "DEFAULT" + - %h the remote host name (only available for a daemon) + - %i an itemized list of what is being updated + - %l the length of the file in bytes + - %L the string "` -> SYMLINK`", "` => HARDLINK`", or "" (where `SYMLINK` + or `HARDLINK` is a filename) + - %m the module name + - %M the last-modified time of the file + - %n the filename (short form; trailing "/" on dir) + - %o the operation, which is "send", "recv", or "del." (the latter includes + the trailing period) + - %p the process ID of this rsync session + - %P the module path + - %t the current date time + - %u the authenticated username or an empty string + - %U the uid of the file (decimal) + + For a list of what the characters mean that are output by "%i", see the + `--itemize-changes` option in the rsync manpage. + + Note that some of the logged output changes when talking with older rsync + versions. For instance, deleted files were only output as verbose messages + prior to rsync 2.6.4. + +0. `timeout` + + This parameter allows you to override the clients choice for I/O timeout + for this module. Using this parameter you can ensure that rsync won't wait + on a dead client forever. The timeout is specified in seconds. A value of + zero means no timeout and is the default. A good choice for anonymous rsync + daemons may be 600 (giving a 10 minute timeout). + +0. `refuse options` + + This parameter allows you to specify a space-separated list of rsync + command-line options that will be refused by your rsync daemon. You may + specify the full option name, its one-letter abbreviation, or a wild-card + string that matches multiple options. Beginning in 3.2.0, you can also + negate a match term by starting it with a "!". + + When an option is refused, the daemon prints an error message and exits. + + For example, this would refuse `--checksum` (`-c`) and all the various + delete options: + + > refuse options = c delete + + The reason the above refuses all delete options is that the options imply + `--delete`, and implied options are refused just like explicit options. + + The use of a negated match allows you to fine-tune your refusals after a + wild-card, such as this: + + > refuse options = delete-* !delete-during + + Negated matching can also turn your list of refused options into a list of + accepted options. To do this, begin the list with a "`*`" (to refuse all + options) and then specify one or more negated matches to accept. For + example: + + > refuse options = * !a !v !compress* + + Don't worry that the "`*`" will refuse certain vital options such as + `--dry-run`, `--server`, `--no-iconv`, `--protect-args`, etc. These + important options are not matched by wild-card, so they must be overridden + by their exact name. For instance, if you're forcing iconv transfers you + could use something like this: + + > refuse options = * no-iconv !a !v + + As an additional aid (beginning in 3.2.0), refusing (or "`!refusing`") the + "a" or "archive" option also affects all the options that the `--archive` + option implies (`-rdlptgoD`), but only if the option is matched explicitly + (not using a wildcard). If you want to do something tricky, you can use + "`archive*`" to avoid this side-effect, but keep in mind that no normal + rsync client ever sends the actual archive option to the server. + + As an additional safety feature, the refusal of "delete" also refuses + `remove-source-files` when the daemon is the sender; if you want the latter + without the former, instead refuse "`delete-*`" as that refuses all the + delete modes without affecting `--remove-source-files`. (Keep in mind that + the client's `--delete` option typically results in `--delete-during`.) + + When un-refusing delete options, you should either specify "`!delete*`" (to + accept all delete options) or specify a limited set that includes "delete", + such as: + + > refuse options = * !a !delete !delete-during + + ... whereas this accepts any delete option except `--delete-after`: + + > refuse options = * !a !delete* delete-after + + A note on refusing "compress" -- it is better to set the "dont compress" + daemon parameter to "`*`" because that disables compression silently + instead of returning an error that forces the client to remove the `-z` + option. + + If you are un-refusing the compress option, you probably want to match + "`!compress*`" so that you also accept the `--compress-level` option. + + Note that the "write-devices" option is refused by default, but can be + explicitly accepted with "`!write-devices`". The options "log-file" and + "log-file-format" are forcibly refused and cannot be accepted. + + Here are all the options that are not matched by wild-cards: + + - `--server`: Required for rsync to even work. + - `--rsh`, `-e`: Required to convey compatibility flags to the server. + - `--log-format`: This is required to convey things like + `--itemize-changes` to a remote receiver. Is an older name for + `--out-format` that is still passed to the server for improved backward + compatibility and should not be confused with `--log-file-format`. + - `--sender`: Use "write only" parameter instead of refusing this. + - `--dry-run`, `-n`: Who would want to disable this? + - `--protect-args`, `-n`: This actually makes transfers safer. + - `--from0`, `-0`: Make it easier to accept/refuse `--files-from` without + affecting this modifier. + - `--iconv`: This is auto-disabled based on "charset" parameter. + - `--no-iconv`: Most transfers use this option. + - `--checksum-seed`: Is a fairly rare, safe option. + - `--write-devices`: Is non-wild but also auto-disabled. + +0. `dont compress` + + This parameter allows you to select filenames based on wildcard patterns + that should not be compressed when pulling files from the daemon (no + analogous parameter exists to govern the pushing of files to a daemon). + Compression can be expensive in terms of CPU usage, so it is usually good + to not try to compress files that won't compress well, such as already + compressed files. + + The "dont compress" parameter takes a space-separated list of + case-insensitive wildcard patterns. Any source filename matching one of the + patterns will be compressed as little as possible during the transfer. If + the compression algorithm has an "off" level (such as zlib/zlibx) then no + compression occurs for those files. Other algorithms have the level + minimized to reduces the CPU usage as much as possible. + + See the `--skip-compress` parameter in the **rsync**(1) manpage for the + list of file suffixes that are not compressed by default. Specifying a + value for the "dont compress" parameter changes the default when the daemon + is the sender. + +0. `early exec`, `pre-xfer exec`, `post-xfer exec` + + You may specify a command to be run in the early stages of the connection, + or right before and/or after the transfer. If the `early exec` or + `pre-xfer exec` command returns an error code, the transfer is aborted + before it begins. Any output from the `pre-xfer exec` command on stdout + (up to several KB) will be displayed to the user when aborting, but is + _not_ displayed if the script returns success. The other programs cannot + send any text to the user. All output except for the `pre-xfer exec` + stdout goes to the corresponding daemon's stdout/stderr, which is typically + discarded. See the `--no-detatch` option for a way to see the daemon's + output, which can assist with debugging. + + Note that the `early exec` command runs before any part of the transfer + request is known except for the module name. This helper script can be + used to setup a disk mount or decrypt some data into a module dir, but you + may need to use `lock file` and `max connections` to avoid concurrency + issues. + + Note that the `post-xfer exec` command is still run even if one of the + other scripts returns an error code. The `pre-xfer exec` command will _not_ + be run, however, if the `early exec` command fails. + + The following environment variables will be set, though some are specific + to the pre-xfer or the post-xfer environment: + + - `RSYNC_MODULE_NAME`: The name of the module being accessed. + - `RSYNC_MODULE_PATH`: The path configured for the module. + - `RSYNC_HOST_ADDR`: The accessing host's IP address. + - `RSYNC_HOST_NAME`: The accessing host's name. + - `RSYNC_USER_NAME`: The accessing user's name (empty if no user). + - `RSYNC_PID`: A unique number for this transfer. + - `RSYNC_REQUEST`: (pre-xfer only) The module/path info specified by the + user. Note that the user can specify multiple source files, so the + request can be something like "mod/path1 mod/path2", etc. + - `RSYNC_ARG#`: (pre-xfer only) The pre-request arguments are set in these + numbered values. RSYNC_ARG0 is always "rsyncd", followed by the options + that were used in RSYNC_ARG1, and so on. There will be a value of "." + indicating that the options are done and the path args are beginning -- + these contain similar information to RSYNC_REQUEST, but with values + separated and the module name stripped off. + - `RSYNC_EXIT_STATUS`: (post-xfer only) the server side's exit value. This + will be 0 for a successful run, a positive value for an error that the + server generated, or a -1 if rsync failed to exit properly. Note that an + error that occurs on the client side does not currently get sent to the + server side, so this is not the final exit status for the whole transfer. + - `RSYNC_RAW_STATUS`: (post-xfer only) the raw exit value from + **waitpid()**. + + Even though the commands can be associated with a particular module, they + are run using the permissions of the user that started the daemon (not the + module's uid/gid setting) without any chroot restrictions. + + These settings honor 2 environment variables: use RSYNC_SHELL to set a + shell to use when running the command (which otherwise uses your + **system()** call's default shell), and use RSYNC_NO_XFER_EXEC to disable + both options completely. + +# CONFIG DIRECTIVES + +There are currently two config directives available that allow a config file to +incorporate the contents of other files: `&include` and `&merge`. Both allow +a reference to either a file or a directory. They differ in how segregated the +file's contents are considered to be. + +The `&include` directive treats each file as more distinct, with each one +inheriting the defaults of the parent file, starting the parameter parsing as +globals/defaults, and leaving the defaults unchanged for the parsing of the +rest of the parent file. + +The `&merge` directive, on the other hand, treats the file's contents as if it +were simply inserted in place of the directive, and thus it can set parameters +in a module started in another file, can affect the defaults for other files, +etc. + +When an `&include` or `&merge` directive refers to a directory, it will read in +all the `*.conf` or `*.inc` files (respectively) that are contained inside that +directory (without any recursive scanning), with the files sorted into alpha +order. So, if you have a directory named "rsyncd.d" with the files "foo.conf", +"bar.conf", and "baz.conf" inside it, this directive: + +> &include /path/rsyncd.d + +would be the same as this set of directives: + +> &include /path/rsyncd.d/bar.conf +> &include /path/rsyncd.d/baz.conf +> &include /path/rsyncd.d/foo.conf + +except that it adjusts as files are added and removed from the directory. + +The advantage of the `&include` directive is that you can define one or more +modules in a separate file without worrying about unintended side-effects +between the self-contained module files. + +The advantage of the `&merge` directive is that you can load config snippets +that can be included into multiple module definitions, and you can also set +global values that will affect connections (such as `motd file`), or globals +that will affect other include files. + +For example, this is a useful /etc/rsyncd.conf file: + +> port = 873 +> log file = /var/log/rsync.log +> pid file = /var/lock/rsync.lock +> +> &merge /etc/rsyncd.d +> &include /etc/rsyncd.d + +This would merge any `/etc/rsyncd.d/*.inc` files (for global values that should +stay in effect), and then include any `/etc/rsyncd.d/*.conf` files (defining +modules without any global-value cross-talk). + +# AUTHENTICATION STRENGTH + +The authentication protocol used in rsync is a 128 bit MD4 based challenge +response system. This is fairly weak protection, though (with at least one +brute-force hash-finding algorithm publicly available), so if you want really +top-quality security, then I recommend that you run rsync over ssh. (Yes, a +future version of rsync will switch over to a stronger hashing method.) + +Also note that the rsync daemon protocol does not currently provide any +encryption of the data that is transferred over the connection. Only +authentication is provided. Use ssh as the transport if you want encryption. + +You can also make use of SSL/TLS encryption if you put rsync behind an +SSL proxy. + +# SSL/TLS Daemon Setup + +When setting up an rsync daemon for access via SSL/TLS, you will need to +configure a proxy (such as haproxy or nginx) as the front-end that handles the +encryption. + +- You should limit the access to the backend-rsyncd port to only allow the + proxy to connect. If it is on the same host as the proxy, then configuring + it to only listen on localhost is a good idea. +- You should consider turning on the `proxy protocol` parameter if your proxy + supports sending that information. The examples below assume that this is + enabled. + +An example haproxy setup is as follows: + +> ``` +> frontend fe_rsync-ssl +> bind :::874 ssl crt /etc/letsencrypt/example.com/combined.pem +> mode tcp +> use_backend be_rsync +> +> backend be_rsync +> mode tcp +> server local-rsync 127.0.0.1:873 check send-proxy +> ``` + +An example nginx proxy setup is as follows: + +> ``` +> stream { +> server { +> listen 874 ssl; +> listen [::]:874 ssl; +> +> ssl_certificate /etc/letsencrypt/example.com/fullchain.pem; +> ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem +> +> proxy_pass localhost:873; +> proxy_protocol on; # Requires "proxy protocol = true" +> proxy_timeout 1m; +> proxy_connect_timeout 5s; +> } +> } +> ``` + +# EXAMPLES + +A simple rsyncd.conf file that allow anonymous rsync to a ftp area at +`/home/ftp` would be: + +> ``` +> [ftp] +> path = /home/ftp +> comment = ftp export area +> ``` + +A more sophisticated example would be: + +> ``` +> uid = nobody +> gid = nobody +> use chroot = yes +> max connections = 4 +> syslog facility = local5 +> pid file = /var/run/rsyncd.pid +> +> [ftp] +> path = /var/ftp/./pub +> comment = whole ftp area (approx 6.1 GB) +> +> [sambaftp] +> path = /var/ftp/./pub/samba +> comment = Samba ftp area (approx 300 MB) +> +> [rsyncftp] +> path = /var/ftp/./pub/rsync +> comment = rsync ftp area (approx 6 MB) +> +> [sambawww] +> path = /public_html/samba +> comment = Samba WWW pages (approx 240 MB) +> +> [cvs] +> path = /data/cvs +> comment = CVS repository (requires authentication) +> auth users = tridge, susan +> secrets file = /etc/rsyncd.secrets +> ``` + +The /etc/rsyncd.secrets file would look something like this: + +> tridge:mypass +> susan:herpass + +# FILES + +/etc/rsyncd.conf or rsyncd.conf + +# SEE ALSO + +**rsync**(1), **rsync-ssl**(1) + +# BUGS + +Please report bugs! The rsync bug tracking system is online at +<http://rsync.samba.org/>. + +# VERSION + +This man page is current for version @VERSION@ of rsync. + +# CREDITS + +rsync is distributed under the GNU General Public License. See the file +COPYING for details. + +The primary ftp site for rsync is <ftp://rsync.samba.org/pub/rsync> + +A web site is available at <http://rsync.samba.org/>. + +We would be delighted to hear from you if you like this program. + +This program uses the zlib compression library written by Jean-loup Gailly and +Mark Adler. + +# THANKS + +Thanks to Warren Stanley for his original idea and patch for the rsync daemon. +Thanks to Karsten Thygesen for his many suggestions and documentation! + +# AUTHOR + +rsync was written by Andrew Tridgell and Paul Mackerras. Many people have +later contributed to it. + +Mailing lists for support and development are available at +<http://lists.samba.org/>. diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo deleted file mode 100644 index 7326b42d..00000000 --- a/rsyncd.conf.yo +++ /dev/null @@ -1,988 +0,0 @@ -mailto(rsync-bugs@samba.org) -manpage(rsyncd.conf)(5)(28 Jan 2018)()() -manpagename(rsyncd.conf)(configuration file for rsync in daemon mode) -manpagesynopsis() - -rsyncd.conf - -manpagedescription() - -The rsyncd.conf file is the runtime configuration file for rsync when -run as an rsync daemon. - -The rsyncd.conf file controls authentication, access, logging and -available modules. - -manpagesection(FILE FORMAT) - -The file consists of modules and parameters. A module begins with the -name of the module in square brackets and continues until the next -module begins. Modules contain parameters of the form "name = value". - -The file is line-based -- that is, each newline-terminated line represents -either a comment, a module name or a parameter. - -Only the first equals sign in a parameter is significant. Whitespace before -or after the first equals sign is discarded. Leading, trailing and internal -whitespace in module and parameter names is irrelevant. Leading and -trailing whitespace in a parameter value is discarded. Internal whitespace -within a parameter value is retained verbatim. - -Any line bf(beginning) with a hash (#) is ignored, as are lines containing -only whitespace. (If a hash occurs after anything other than leading -whitespace, it is considered a part of the line's content.) - -Any line ending in a \ is "continued" on the next line in the -customary UNIX fashion. - -The values following the equals sign in parameters are all either a string -(no quotes needed) or a boolean, which may be given as yes/no, 0/1 or -true/false. Case is not significant in boolean values, but is preserved -in string values. - -manpagesection(LAUNCHING THE RSYNC DAEMON) - -The rsync daemon is launched by specifying the bf(--daemon) option to -rsync. - -The daemon must run with root privileges if you wish to use chroot, to -bind to a port numbered under 1024 (as is the default 873), or to set -file ownership. Otherwise, it must just have permission to read and -write the appropriate data, log, and lock files. - -You can launch it either via inetd, as a stand-alone daemon, or from -an rsync client via a remote shell. If run as a stand-alone daemon then -just run the command "bf(rsync --daemon)" from a suitable startup script. - -When run via inetd you should add a line like this to /etc/services: - -verb( rsync 873/tcp) - -and a single line something like this to /etc/inetd.conf: - -verb( rsync stream tcp nowait root /usr/bin/rsync rsyncd --daemon) - -Replace "/usr/bin/rsync" with the path to where you have rsync installed on -your system. You will then need to send inetd a HUP signal to tell it to -reread its config file. - -Note that you should bf(not) send the rsync daemon a HUP signal to force -it to reread the tt(rsyncd.conf) file. The file is re-read on each client -connection. - -manpagesection(GLOBAL PARAMETERS) - -The first parameters in the file (before a [module] header) are the -global parameters. -Rsync also allows for the use of a "[global]" module name to indicate the -start of one or more global-parameter sections (the name must be lower case). - -You may also include any module parameters in the global part of the -config file in which case the supplied value will override the -default for that parameter. - -You may use references to environment variables in the values of parameters. -String parameters will have %VAR% references expanded as late as possible (when -the string is used in the program), allowing for the use of variables that -rsync sets at connection time, such as RSYNC_USER_NAME. Non-string parameters -(such as true/false settings) are expanded when read from the config file. If -a variable does not exist in the environment, or if a sequence of characters is -not a valid reference (such as an un-paired percent sign), the raw characters -are passed through unchanged. This helps with backward compatibility and -safety (e.g. expanding a non-existent %VAR% to an empty string in a path could -result in a very unsafe path). The safest way to insert a literal % into a -value is to use %%. - -description( - -dit(bf(motd file)) This parameter allows you to specify a -"message of the day" to display to clients on each connect. This -usually contains site information and any legal notices. The default -is no motd file. -This can be overridden by the bf(--dparam=motdfile=FILE) -command-line option when starting the daemon. - -dit(bf(pid file)) This parameter tells the rsync daemon to write -its process ID to that file. If the file already exists, the rsync -daemon will abort rather than overwrite the file. -This can be overridden by the bf(--dparam=pidfile=FILE) -command-line option when starting the daemon. - -dit(bf(port)) You can override the default port the daemon will listen on -by specifying this value (defaults to 873). This is ignored if the daemon -is being run by inetd, and is superseded by the bf(--port) command-line option. - -dit(bf(address)) You can override the default IP address the daemon -will listen on by specifying this value. This is ignored if the daemon is -being run by inetd, and is superseded by the bf(--address) command-line option. - -dit(bf(socket options)) This parameter can provide endless fun for people -who like to tune their systems to the utmost degree. You can set all -sorts of socket options which may make transfers faster (or -slower!). Read the man page for the code(setsockopt()) system call for -details on some of the options you may be able to set. By default no -special socket options are set. These settings can also be specified -via the bf(--sockopts) command-line option. - -dit(bf(listen backlog)) You can override the default backlog value when the -daemon listens for connections. It defaults to 5. - -) - -manpagesection(MODULE PARAMETERS) - -After the global parameters you should define a number of modules, each -module exports a directory tree as a symbolic name. Modules are -exported by specifying a module name in square brackets [module] -followed by the parameters for that module. -The module name cannot contain a slash or a closing square bracket. If the -name contains whitespace, each internal sequence of whitespace will be -changed into a single space, while leading or trailing whitespace will be -discarded. -Also, the name cannot be "global" as that exact name indicates that -global parameters follow (see above). - -As with GLOBAL PARAMETERS, you may use references to environment variables in -the values of parameters. See the GLOBAL PARAMETERS section for more details. - -description( - -dit(bf(comment)) This parameter specifies a description string -that is displayed next to the module name when clients obtain a list -of available modules. The default is no comment. - -dit(bf(path)) This parameter specifies the directory in the daemon's -filesystem to make available in this module. You must specify this parameter -for each module in tt(rsyncd.conf). - -You may base the path's value off of an environment variable by surrounding -the variable name with percent signs. You can even reference a variable -that is set by rsync when the user connects. -For example, this would use the authorizing user's name in the path: - -verb( path = /home/%RSYNC_USER_NAME% ) - -It is fine if the path includes internal spaces -- they will be retained -verbatim (which means that you shouldn't try to escape them). If your final -directory has a trailing space (and this is somehow not something you wish to -fix), append a trailing slash to the path to avoid losing the trailing -whitespace. - -dit(bf(use chroot)) If "use chroot" is true, the rsync daemon will chroot -to the "path" before starting the file transfer with the client. This has -the advantage of extra protection against possible implementation security -holes, but it has the disadvantages of requiring super-user privileges, -of not being able to follow symbolic links that are either absolute or outside -of the new root path, and of complicating the preservation of users and groups -by name (see below). - -As an additional safety feature, you can specify a dot-dir in the module's -"path" to indicate the point where the chroot should occur. This allows rsync -to run in a chroot with a non-"/" path for the top of the transfer hierarchy. -Doing this guards against unintended library loading (since those absolute -paths will not be inside the transfer hierarchy unless you have used an unwise -pathname), and lets you setup libraries for the chroot that are outside of the -transfer. For example, specifying "/var/rsync/./module1" will chroot to the -"/var/rsync" directory and set the inside-chroot path to "/module1". If you -had omitted the dot-dir, the chroot would have used the whole path, and the -inside-chroot path would have been "/". - -When both "use chroot" and "daemon chroot" are false, OR the inside-chroot path -of "use chroot" is not "/", rsync will: (1) munge symlinks by -default for security reasons (see "munge symlinks" for a way to turn this -off, but only if you trust your users), (2) substitute leading slashes in -absolute paths with the module's path (so that options such as -bf(--backup-dir), bf(--compare-dest), etc. interpret an absolute path as -rooted in the module's "path" dir), and (3) trim ".." path elements from -args if rsync believes they would escape the module hierarchy. -The default for "use chroot" is true, and is the safer choice (especially -if the module is not read-only). - -When this parameter is enabled, the "numeric-ids" option will also default to -being enabled (disabling name lookups). See below for what a chroot needs in -order for name lookups to succeed. - -If you copy library resources into the module's chroot area, you -should protect them through your OS's normal user/group or ACL settings (to -prevent the rsync module's user from being able to change them), and then -hide them from the user's view via "exclude" (see how in the discussion of -that parameter). At that point it will be safe to enable the mapping of users -and groups by name using the "numeric ids" daemon parameter (see below). - -Note also that you are free to setup custom user/group information in the -chroot area that is different from your normal system. For example, you -could abbreviate the list of users and groups. - -dit(bf(daemon chroot)) This parameter specifies a path to which the daemon will -chroot before beginning communication with clients. Module paths (and any "use -chroot" settings) will then be related to this one. This lets you choose if you -want the whole daemon to be chrooted (with this setting), just the transfers to -be chrooted (with "use chroot"), or both. Keep in mind that the "daemon chroot" -area may need various OS/lib/etc files installed to allow the daemon to function. -By default the daemon runs without any chrooting. - -dit(bf(numeric ids)) Enabling this parameter disables the mapping -of users and groups by name for the current daemon module. This prevents -the daemon from trying to load any user/group-related files or libraries. -This enabling makes the transfer behave as if the client had passed -the bf(--numeric-ids) command-line option. By default, this parameter is -enabled for chroot modules and disabled for non-chroot modules. -Also keep in mind that uid/gid preservation requires the module to be -running as root (see "uid") or for "fake super" to be configured. - -A chroot-enabled module should not have this parameter enabled unless you've -taken steps to ensure that the module has the necessary resources it needs -to translate names, and that it is not possible for a user to change those -resources. That includes being the code being able to call functions like -code(getpwuid()), code(getgrgid()), code(getpwname()), and code(getgrnam()). -You should test what libraries and config files are required for your OS -and get those setup before starting to test name mapping in rsync. - -dit(bf(munge symlinks)) This parameter tells rsync to modify -all symlinks in the same way as the (non-daemon-affecting) -bf(--munge-links) command-line option (using a method described below). -This should help protect your files from user trickery when -your daemon module is writable. The default is disabled when "use chroot" -is on with an inside-chroot path of "/", OR if "daemon chroot" is on, -otherwise it is enabled. - -If you disable this parameter on a daemon that is not read-only, there -are tricks that a user can play with uploaded symlinks to access -daemon-excluded items (if your module has any), and, if "use chroot" -is off, rsync can even be tricked into showing or changing data that -is outside the module's path (as access-permissions allow). - -The way rsync disables the use of symlinks is to prefix each one with -the string "/rsyncd-munged/". This prevents the links from being used -as long as that directory does not exist. When this parameter is enabled, -rsync will refuse to run if that path is a directory or a symlink to -a directory. When using the "munge symlinks" parameter in a chroot area -that has an inside-chroot path of "/", you should add "/rsyncd-munged/" -to the exclude setting for the module so that -a user can't try to create it. - -Note: rsync makes no attempt to verify that any pre-existing symlinks in -the module's hierarchy are as safe as you want them to be (unless, of -course, it just copied in the whole hierarchy). If you setup an rsync -daemon on a new area or locally add symlinks, you can manually protect your -symlinks from being abused by prefixing "/rsyncd-munged/" to the start of -every symlink's value. There is a perl script in the support directory -of the source code named "munge-symlinks" that can be used to add or remove -this prefix from your symlinks. - -When this parameter is disabled on a writable module and "use chroot" is off -(or the inside-chroot path is not "/"), -incoming symlinks will be modified to drop a leading slash and to remove ".." -path elements that rsync believes will allow a symlink to escape the module's -hierarchy. There are tricky ways to work around this, though, so you had -better trust your users if you choose this combination of parameters. - -dit(bf(charset)) This specifies the name of the character set in which the -module's filenames are stored. If the client uses an bf(--iconv) option, -the daemon will use the value of the "charset" parameter regardless of the -character set the client actually passed. This allows the daemon to -support charset conversion in a chroot module without extra files in the -chroot area, and also ensures that name-translation is done in a consistent -manner. If the "charset" parameter is not set, the bf(--iconv) option is -refused, just as if "iconv" had been specified via "refuse options". - -If you wish to force users to always use bf(--iconv) for a particular -module, add "no-iconv" to the "refuse options" parameter. Keep in mind -that this will restrict access to your module to very new rsync clients. - -dit(bf(max connections)) This parameter allows you to -specify the maximum number of simultaneous connections you will allow. -Any clients connecting when the maximum has been reached will receive a -message telling them to try later. The default is 0, which means no limit. -A negative value disables the module. -See also the "lock file" parameter. - -dit(bf(log file)) When the "log file" parameter is set to a non-empty -string, the rsync daemon will log messages to the indicated file rather -than using syslog. This is particularly useful on systems (such as AIX) -where code(syslog()) doesn't work for chrooted programs. The file is -opened before code(chroot()) is called, allowing it to be placed outside -the transfer. If this value is set on a per-module basis instead of -globally, the global log will still contain any authorization failures -or config-file error messages. - -If the daemon fails to open the specified file, it will fall back to -using syslog and output an error about the failure. (Note that the -failure to open the specified log file used to be a fatal error.) - -This setting can be overridden by using the bf(--log-file=FILE) or -bf(--dparam=logfile=FILE) command-line options. The former overrides -all the log-file parameters of the daemon and all module settings. -The latter sets the daemon's log file and the default for all the -modules, which still allows modules to override the default setting. - -dit(bf(syslog facility)) This parameter allows you to -specify the syslog facility name to use when logging messages from the -rsync daemon. You may use any standard syslog facility name which is -defined on your system. Common names are auth, authpriv, cron, daemon, -ftp, kern, lpr, mail, news, security, syslog, user, uucp, local0, -local1, local2, local3, local4, local5, local6 and local7. The default -is daemon. This setting has no effect if the "log file" setting is a -non-empty string (either set in the per-modules settings, or inherited -from the global settings). - -dit(bf(syslog tag)) This parameter allows you to specify the syslog -tag to use when logging messages from the rsync daemon. The default is -"rsyncd". This setting has no effect if the "log file" setting is a -non-empty string (either set in the per-modules settings, or inherited -from the global settings). - -For example, if you wanted each authenticated user's name to be -included in the syslog tag, you could do something like this: - -verb( syslog tag = rsyncd.%RSYNC_USER_NAME%) - -dit(bf(max verbosity)) This parameter allows you to control -the maximum amount of verbose information that you'll allow the daemon to -generate (since the information goes into the log file). The default is 1, -which allows the client to request one level of verbosity. - -This also affects the user's ability to request higher levels of bf(--info) and -bf(--debug) logging. If the max value is 2, then no info and/or debug value -that is higher than what would be set by bf(-vv) will be honored by the daemon -in its logging. To see how high of a verbosity level you need to accept for a -particular info/debug level, refer to "rsync --info=help" and "rsync --debug=help". -For instance, it takes max-verbosity 4 to be able to output debug TIME2 and FLIST3. - -dit(bf(lock file)) This parameter specifies the file to use to -support the "max connections" parameter. The rsync daemon uses record -locking on this file to ensure that the max connections limit is not -exceeded for the modules sharing the lock file. -The default is tt(/var/run/rsyncd.lock). - -dit(bf(read only)) This parameter determines whether clients -will be able to upload files or not. If "read only" is true then any -attempted uploads will fail. If "read only" is false then uploads will -be possible if file permissions on the daemon side allow them. The default -is for all modules to be read only. - -Note that "auth users" can override this setting on a per-user basis. - -dit(bf(write only)) This parameter determines whether clients -will be able to download files or not. If "write only" is true then any -attempted downloads will fail. If "write only" is false then downloads -will be possible if file permissions on the daemon side allow them. The -default is for this parameter to be disabled. - -Helpful hint: you probably want to specify "refuse options = delete" for a -write-only module. - -) -description( - -dit(bf(list)) This parameter determines whether this module is -listed when the client asks for a listing of available modules. In addition, -if this is false, the daemon will pretend the module does not exist -when a client denied by "hosts allow" or "hosts deny" attempts to access it. -Realize that if "reverse lookup" is disabled globally but enabled for the -module, the resulting reverse lookup to a potentially client-controlled DNS -server may still reveal to the client that it hit an existing module. -The default is for modules to be listable. - -dit(bf(uid)) This parameter specifies the user name or user ID that -file transfers to and from that module should take place as when the daemon -was run as root. In combination with the "gid" parameter this determines what -file permissions are available. The default when run by a super-user is to -switch to the system's "nobody" user. The default for a non-super-user is to -not try to change the user. See also the "gid" parameter. - -The RSYNC_USER_NAME environment variable may be used to request that rsync run -as the authorizing user. For example, if you want a rsync to run as the same -user that was received for the rsync authentication, this setup is useful: - -verb( uid = %RSYNC_USER_NAME% - gid = * ) - -dit(bf(gid)) This parameter specifies one or more group names/IDs that will be -used when accessing the module. The first one will be the default group, and -any extra ones be set as supplemental groups. You may also specify a "*" as -the first gid in the list, which will be replaced by all the normal groups for -the transfer's user (see "uid"). The default when run by a super-user is to -switch to your OS's "nobody" (or perhaps "nogroup") group with no other -supplementary groups. The default for a non-super-user is to not change any -group attributes (and indeed, your OS may not allow a non-super-user to try to -change their group settings). - -dit(bf(daemon uid)) This parameter specifies a uid under which the daemon will -run. The daemon usually runs as user root, and when this is left unset the user -is left unchanged. See also the "uid" parameter. - -dit(bf(daemon gid)) This parameter specifies a gid under which the daemon will -run. The daemon usually runs as group root, and when this is left unset, the -group is left unchanged. See also the "gid" parameter. - -dit(bf(fake super)) Setting "fake super = yes" for a module causes the -daemon side to behave as if the bf(--fake-super) command-line option had -been specified. This allows the full attributes of a file to be stored -without having to have the daemon actually running as root. - -dit(bf(filter)) The daemon has its own filter chain that determines what files -it will let the client access. This chain is not sent to the client and is -independent of any filters the client may have specified. Files excluded by -the daemon filter chain (bf(daemon-excluded) files) are treated as non-existent -if the client tries to pull them, are skipped with an error message if the -client tries to push them (triggering exit code 23), and are never deleted from -the module. You can use daemon filters to prevent clients from downloading or -tampering with private administrative files, such as files you may add to -support uid/gid name translations. - -The daemon filter chain is built from the "filter", "include from", "include", -"exclude from", and "exclude" parameters, in that order of priority. Anchored -patterns are anchored at the root of the module. To prevent access to an -entire subtree, for example, "/secret", you em(must) exclude everything in the -subtree; the easiest way to do this is with a triple-star pattern like -"/secret/***". - -The "filter" parameter takes a space-separated list of daemon filter rules, -though it is smart enough to know not to split a token at an internal space in -a rule (e.g. "- /foo - /bar" is parsed as two rules). You may specify one or -more merge-file rules using the normal syntax. Only one "filter" parameter can -apply to a given module in the config file, so put all the rules you want in a -single parameter. Note that per-directory merge-file rules do not provide as -much protection as global rules, but they can be used to make bf(--delete) work -better during a client download operation if the per-dir merge files are -included in the transfer and the client requests that they be used. - -dit(bf(exclude)) This parameter takes a space-separated list of daemon -exclude patterns. As with the client bf(--exclude) option, patterns can be -qualified with "- " or "+ " to explicitly indicate exclude/include. Only one -"exclude" parameter can apply to a given module. See the "filter" parameter -for a description of how excluded files affect the daemon. - -dit(bf(include)) Use an "include" to override the effects of the "exclude" -parameter. Only one "include" parameter can apply to a given module. See the -"filter" parameter for a description of how excluded files affect the daemon. - -dit(bf(exclude from)) This parameter specifies the name of a file -on the daemon that contains daemon exclude patterns, one per line. Only one -"exclude from" parameter can apply to a given module; if you have multiple -exclude-from files, you can specify them as a merge file in the "filter" -parameter. See the "filter" parameter for a description of how excluded files -affect the daemon. - -dit(bf(include from)) Analogue of "exclude from" for a file of daemon include -patterns. Only one "include from" parameter can apply to a given module. See -the "filter" parameter for a description of how excluded files affect the -daemon. - -dit(bf(incoming chmod)) This parameter allows you to specify a set of -comma-separated chmod strings that will affect the permissions of all -incoming files (files that are being received by the daemon). These -changes happen after all other permission calculations, and this will -even override destination-default and/or existing permissions when the -client does not specify bf(--perms). -See the description of the bf(--chmod) rsync option and the bf(chmod)(1) -manpage for information on the format of this string. - -dit(bf(outgoing chmod)) This parameter allows you to specify a set of -comma-separated chmod strings that will affect the permissions of all -outgoing files (files that are being sent out from the daemon). These -changes happen first, making the sent permissions appear to be different -than those stored in the filesystem itself. For instance, you could -disable group write permissions on the server while having it appear to -be on to the clients. -See the description of the bf(--chmod) rsync option and the bf(chmod)(1) -manpage for information on the format of this string. - -dit(bf(auth users)) This parameter specifies a comma and/or space-separated -list of authorization rules. In its simplest form, you list the usernames -that will be allowed to connect to -this module. The usernames do not need to exist on the local -system. The rules may contain shell wildcard characters that will be matched -against the username provided by the client for authentication. If -"auth users" is set then the client will be challenged to supply a -username and password to connect to the module. A challenge response -authentication protocol is used for this exchange. The plain text -usernames and passwords are stored in the file specified by the -"secrets file" parameter. The default is for all users to be able to -connect without a password (this is called "anonymous rsync"). - -In addition to username matching, you can specify groupname matching via a '@' -prefix. When using groupname matching, the authenticating username must be a -real user on the system, or it will be assumed to be a member of no groups. -For example, specifying "@rsync" will match the authenticating user if the -named user is a member of the rsync group. - -Finally, options may be specified after a colon (:). The options allow you to -"deny" a user or a group, set the access to "ro" (read-only), or set the access -to "rw" (read/write). Setting an auth-rule-specific ro/rw setting overrides -the module's "read only" setting. - -Be sure to put the rules in the order you want them to be matched, because the -checking stops at the first matching user or group, and that is the only auth -that is checked. For example: - -verb( auth users = joe:deny @guest:deny admin:rw @rsync:ro susan joe sam ) - -In the above rule, user joe will be denied access no matter what. Any user -that is in the group "guest" is also denied access. The user "admin" gets -access in read/write mode, but only if the admin user is not in group "guest" -(because the admin user-matching rule would never be reached if the user is in -group "guest"). Any other user who is in group "rsync" will get read-only -access. Finally, users susan, joe, and sam get the ro/rw setting of the -module, but only if the user didn't match an earlier group-matching rule. - -If you need to specify a user or group name with a space in it, start your list -with a comma to indicate that the list should only be split on commas (though -leading and trailing whitespace will also be removed, and empty entries are -just ignored). For example: - -verb( auth users = , joe:deny, @Some Group:deny, admin:rw, @RO Group:ro ) - -See the description of the secrets file for how you can have per-user passwords -as well as per-group passwords. It also explains how a user can authenticate -using their user password or (when applicable) a group password, depending on -what rule is being authenticated. - -See also the section entitled "USING RSYNC-DAEMON FEATURES VIA A REMOTE -SHELL CONNECTION" in bf(rsync)(1) for information on how handle an -rsyncd.conf-level username that differs from the remote-shell-level -username when using a remote shell to connect to an rsync daemon. - -dit(bf(secrets file)) This parameter specifies the name of a file that contains -the username:password and/or @groupname:password pairs used for authenticating -this module. This file is only consulted if the "auth users" parameter is -specified. The file is line-based and contains one name:password pair per -line. Any line has a hash (#) as the very first character on the line is -considered a comment and is skipped. The passwords can contain any characters -but be warned that many operating systems limit the length of passwords that -can be typed at the client end, so you may find that passwords longer than 8 -characters don't work. - -The use of group-specific lines are only relevant when the module is being -authorized using a matching "@groupname" rule. When that happens, the user -can be authorized via either their "username:password" line or the -"@groupname:password" line for the group that triggered the authentication. - -It is up to you what kind of password entries you want to include, either -users, groups, or both. The use of group rules in "auth users" does not -require that you specify a group password if you do not want to use shared -passwords. - -There is no default for the "secrets file" parameter, you must choose a name -(such as tt(/etc/rsyncd.secrets)). The file must normally not be readable -by "other"; see "strict modes". If the file is not found or is rejected, no -logins for a "user auth" module will be possible. - -dit(bf(strict modes)) This parameter determines whether or not -the permissions on the secrets file will be checked. If "strict modes" is -true, then the secrets file must not be readable by any user ID other -than the one that the rsync daemon is running under. If "strict modes" is -false, the check is not performed. The default is true. This parameter -was added to accommodate rsync running on the Windows operating system. - -) -description( - -dit(bf(hosts allow)) This parameter allows you to specify a list of comma- -and/or whitespace-separated patterns that are matched against a connecting -client's hostname and IP address. If none of the patterns match, then the -connection is rejected. - -Each pattern can be in one of five forms: - -quote(itemization( - it() a dotted decimal IPv4 address of the form a.b.c.d, or an IPv6 address - of the form a:b:c::d:e:f. In this case the incoming machine's IP address - must match exactly. - it() an address/mask in the form ipaddr/n where ipaddr is the IP address - and n is the number of one bits in the netmask. All IP addresses which - match the masked IP address will be allowed in. - it() an address/mask in the form ipaddr/maskaddr where ipaddr is the - IP address and maskaddr is the netmask in dotted decimal notation for IPv4, - or similar for IPv6, e.g. ffff:ffff:ffff:ffff:: instead of /64. All IP - addresses which match the masked IP address will be allowed in. - it() a hostname pattern using wildcards. If the hostname of the connecting IP - (as determined by a reverse lookup) matches the wildcarded name (using the - same rules as normal unix filename matching), the client is allowed in. This - only works if "reverse lookup" is enabled (the default). - it() a hostname. A plain hostname is matched against the reverse DNS of the - connecting IP (if "reverse lookup" is enabled), and/or the IP of the given - hostname is matched against the connecting IP (if "forward lookup" is - enabled, as it is by default). Any match will be allowed in. -)) - -Note IPv6 link-local addresses can have a scope in the address specification: - -quote( -tt( fe80::1%link1)nl() -tt( fe80::%link1/64)nl() -tt( fe80::%link1/ffff:ffff:ffff:ffff::)nl() -) - -You can also combine "hosts allow" with a separate "hosts deny" -parameter. If both parameters are specified then the "hosts allow" parameter is -checked first and a match results in the client being able to -connect. The "hosts deny" parameter is then checked and a match means -that the host is rejected. If the host does not match either the -"hosts allow" or the "hosts deny" patterns then it is allowed to -connect. - -The default is no "hosts allow" parameter, which means all hosts can connect. - -dit(bf(hosts deny)) This parameter allows you to specify a list of comma- -and/or whitespace-separated patterns that are matched against a connecting -clients hostname and IP address. If the pattern matches then the connection is -rejected. See the "hosts allow" parameter for more information. - -The default is no "hosts deny" parameter, which means all hosts can connect. - -dit(bf(reverse lookup)) Controls whether the daemon performs a reverse lookup -on the client's IP address to determine its hostname, which is used for -"hosts allow"/"hosts deny" checks and the "%h" log escape. This is enabled by -default, but you may wish to disable it to save time if you know the lookup will -not return a useful result, in which case the daemon will use the name -"UNDETERMINED" instead. - -If this parameter is enabled globally (even by default), rsync performs the -lookup as soon as a client connects, so disabling it for a module will not -avoid the lookup. Thus, you probably want to disable it globally and then -enable it for modules that need the information. - -dit(bf(forward lookup)) Controls whether the daemon performs a forward lookup -on any hostname specified in an hosts allow/deny setting. By default this is -enabled, allowing the use of an explicit hostname that would not be returned -by reverse DNS of the connecting IP. - -dit(bf(ignore errors)) This parameter tells rsyncd to -ignore I/O errors on the daemon when deciding whether to run the delete -phase of the transfer. Normally rsync skips the bf(--delete) step if any -I/O errors have occurred in order to prevent disastrous deletion due -to a temporary resource shortage or other I/O error. In some cases this -test is counter productive so you can use this parameter to turn off this -behavior. - -dit(bf(ignore nonreadable)) This tells the rsync daemon to completely -ignore files that are not readable by the user. This is useful for -public archives that may have some non-readable files among the -directories, and the sysadmin doesn't want those files to be seen at all. - -dit(bf(transfer logging)) This parameter enables per-file -logging of downloads and uploads in a format somewhat similar to that -used by ftp daemons. The daemon always logs the transfer at the end, so -if a transfer is aborted, no mention will be made in the log file. - -If you want to customize the log lines, see the "log format" parameter. - -dit(bf(log format)) This parameter allows you to specify the -format used for logging file transfers when transfer logging is enabled. -The format is a text string containing embedded single-character escape -sequences prefixed with a percent (%) character. An optional numeric -field width may also be specified between the percent and the escape -letter (e.g. "bf(%-50n %8l %07p)"). -In addition, one or more apostrophes may be specified prior to a numerical -escape to indicate that the numerical value should be made more human-readable. -The 3 supported levels are the same as for the bf(--human-readable) -command-line option, though the default is for human-readability to be off. -Each added apostrophe increases the level (e.g. "bf(%''l %'b %f)"). - -The default log format is "%o %h [%a] %m (%u) %f %l", and a "%t [%p] " -is always prefixed when using the "log file" parameter. -(A perl script that will summarize this default log format is included -in the rsync source code distribution in the "support" subdirectory: -rsyncstats.) - -The single-character escapes that are understood are as follows: - -quote(itemization( - it() %a the remote IP address (only available for a daemon) - it() %b the number of bytes actually transferred - it() %B the permission bits of the file (e.g. rwxrwxrwt) - it() %c the total size of the block checksums received for the basis file (only when sending) - it() %C the full-file checksum if it is known for the file. For older rsync protocols/versions, the checksum was salted, and is thus not a useful value (and is not displayed when that is the case). For the checksum to output for a file, either the bf(--checksum) option must be in-effect or the file must have been transferred without a salted checksum being used. See the bf(--checksum-choice) option for a way to choose the algorithm. - it() %f the filename (long form on sender; no trailing "/") - it() %G the gid of the file (decimal) or "DEFAULT" - it() %h the remote host name (only available for a daemon) - it() %i an itemized list of what is being updated - it() %l the length of the file in bytes - it() %L the string " -> SYMLINK", " => HARDLINK", or "" (where bf(SYMLINK) or bf(HARDLINK) is a filename) - it() %m the module name - it() %M the last-modified time of the file - it() %n the filename (short form; trailing "/" on dir) - it() %o the operation, which is "send", "recv", or "del." (the latter includes the trailing period) - it() %p the process ID of this rsync session - it() %P the module path - it() %t the current date time - it() %u the authenticated username or an empty string - it() %U the uid of the file (decimal) -)) - -For a list of what the characters mean that are output by "%i", see the -bf(--itemize-changes) option in the rsync manpage. - -Note that some of the logged output changes when talking with older -rsync versions. For instance, deleted files were only output as verbose -messages prior to rsync 2.6.4. - -) -description( - -dit(bf(timeout)) This parameter allows you to override the -clients choice for I/O timeout for this module. Using this parameter you -can ensure that rsync won't wait on a dead client forever. The timeout -is specified in seconds. A value of zero means no timeout and is the -default. A good choice for anonymous rsync daemons may be 600 (giving -a 10 minute timeout). - -dit(bf(refuse options)) This parameter allows you to -specify a space-separated list of rsync command line options that will -be refused by your rsync daemon. -You may specify the full option name, its one-letter abbreviation, or a -wild-card string that matches multiple options. -For example, this would refuse bf(--checksum) (bf(-c)) and all the various -delete options: - -quote(tt( refuse options = c delete)) - -The reason the above refuses all delete options is that the options imply -bf(--delete), and implied options are refused just like explicit options. -As an additional safety feature, the refusal of "delete" also refuses -bf(remove-source-files) when the daemon is the sender; if you want the latter -without the former, instead refuse "delete-*" -- that refuses all the -delete modes without affecting bf(--remove-source-files). - -When an option is refused, the daemon prints an error message and exits. -To prevent all compression when serving files, -you can use "dont compress = *" (see below) -instead of "refuse options = compress" to avoid returning an error to a -client that requests compression. - -dit(bf(dont compress)) This parameter allows you to select -filenames based on wildcard patterns that should not be compressed -when pulling files from the daemon (no analogous parameter exists to -govern the pushing of files to a daemon). -Compression is expensive in terms of CPU usage, so it -is usually good to not try to compress files that won't compress well, -such as already compressed files. - -The "dont compress" parameter takes a space-separated list of -case-insensitive wildcard patterns. Any source filename matching one -of the patterns will not be compressed during transfer. - -See the bf(--skip-compress) parameter in the bf(rsync)(1) manpage for the list -of file suffixes that are not compressed by default. Specifying a value -for the "dont compress" parameter changes the default when the daemon is -the sender. - -) -description( - -dit(bf(pre-xfer exec), bf(post-xfer exec)) You may specify a command to be run -before and/or after the transfer. If the bf(pre-xfer exec) command fails, the -transfer is aborted before it begins. Any output from the script on stdout (up -to several KB) will be displayed to the user when aborting, but is NOT -displayed if the script returns success. Any output from the script on stderr -goes to the daemon's stderr, which is typically discarded (though see ---no-detatch option for a way to see the stderr output, which can assist with -debugging). - -The following environment variables will be set, though some are -specific to the pre-xfer or the post-xfer environment: - -quote(itemization( - it() bf(RSYNC_MODULE_NAME): The name of the module being accessed. - it() bf(RSYNC_MODULE_PATH): The path configured for the module. - it() bf(RSYNC_HOST_ADDR): The accessing host's IP address. - it() bf(RSYNC_HOST_NAME): The accessing host's name. - it() bf(RSYNC_USER_NAME): The accessing user's name (empty if no user). - it() bf(RSYNC_PID): A unique number for this transfer. - it() bf(RSYNC_REQUEST): (pre-xfer only) The module/path info specified - by the user. Note that the user can specify multiple source files, - so the request can be something like "mod/path1 mod/path2", etc. - it() bf(RSYNC_ARG#): (pre-xfer only) The pre-request arguments are set - in these numbered values. RSYNC_ARG0 is always "rsyncd", followed by - the options that were used in RSYNC_ARG1, and so on. There will be a - value of "." indicating that the options are done and the path args - are beginning -- these contain similar information to RSYNC_REQUEST, - but with values separated and the module name stripped off. - it() bf(RSYNC_EXIT_STATUS): (post-xfer only) the server side's exit value. - This will be 0 for a successful run, a positive value for an error that the - server generated, or a -1 if rsync failed to exit properly. Note that an - error that occurs on the client side does not currently get sent to the - server side, so this is not the final exit status for the whole transfer. - it() bf(RSYNC_RAW_STATUS): (post-xfer only) the raw exit value from code(waitpid()). -)) - -Even though the commands can be associated with a particular module, they -are run using the permissions of the user that started the daemon (not the -module's uid/gid setting) without any chroot restrictions. - -) - -manpagesection(CONFIG DIRECTIVES) - -There are currently two config directives available that allow a config file to -incorporate the contents of other files: bf(&include) and bf(&merge). Both -allow a reference to either a file or a directory. They differ in how -segregated the file's contents are considered to be. - -The bf(&include) directive treats each file as more distinct, with each one -inheriting the defaults of the parent file, starting the parameter parsing -as globals/defaults, and leaving the defaults unchanged for the parsing of -the rest of the parent file. - -The bf(&merge) directive, on the other hand, treats the file's contents as -if it were simply inserted in place of the directive, and thus it can set -parameters in a module started in another file, can affect the defaults for -other files, etc. - -When an bf(&include) or bf(&merge) directive refers to a directory, it will read -in all the bf(*.conf) or bf(*.inc) files (respectively) that are contained inside -that directory (without any -recursive scanning), with the files sorted into alpha order. So, if you have a -directory named "rsyncd.d" with the files "foo.conf", "bar.conf", and -"baz.conf" inside it, this directive: - -verb( &include /path/rsyncd.d ) - -would be the same as this set of directives: - -verb( &include /path/rsyncd.d/bar.conf - &include /path/rsyncd.d/baz.conf - &include /path/rsyncd.d/foo.conf ) - -except that it adjusts as files are added and removed from the directory. - -The advantage of the bf(&include) directive is that you can define one or more -modules in a separate file without worrying about unintended side-effects -between the self-contained module files. - -The advantage of the bf(&merge) directive is that you can load config snippets -that can be included into multiple module definitions, and you can also set -global values that will affect connections (such as bf(motd file)), or globals -that will affect other include files. - -For example, this is a useful /etc/rsyncd.conf file: - -verb( port = 873 - log file = /var/log/rsync.log - pid file = /var/lock/rsync.lock - - &merge /etc/rsyncd.d - &include /etc/rsyncd.d ) - -This would merge any /etc/rsyncd.d/*.inc files (for global values that should -stay in effect), and then include any /etc/rsyncd.d/*.conf files (defining -modules without any global-value cross-talk). - -manpagesection(AUTHENTICATION STRENGTH) - -The authentication protocol used in rsync is a 128 bit MD4 based -challenge response system. This is fairly weak protection, though (with -at least one brute-force hash-finding algorithm publicly available), so -if you want really top-quality security, then I recommend that you run -rsync over ssh. (Yes, a future version of rsync will switch over to a -stronger hashing method.) - -Also note that the rsync daemon protocol does not currently provide any -encryption of the data that is transferred over the connection. Only -authentication is provided. Use ssh as the transport if you want -encryption. - -Future versions of rsync may support SSL for better authentication and -encryption, but that is still being investigated. - -manpagesection(EXAMPLES) - -A simple rsyncd.conf file that allow anonymous rsync to a ftp area at -tt(/home/ftp) would be: - -verb( -[ftp] - path = /home/ftp - comment = ftp export area -) - -A more sophisticated example would be: - -verb( -uid = nobody -gid = nobody -use chroot = yes -max connections = 4 -syslog facility = local5 -pid file = /var/run/rsyncd.pid - -[ftp] - path = /var/ftp/./pub - comment = whole ftp area (approx 6.1 GB) - -[sambaftp] - path = /var/ftp/./pub/samba - comment = Samba ftp area (approx 300 MB) - -[rsyncftp] - path = /var/ftp/./pub/rsync - comment = rsync ftp area (approx 6 MB) - -[sambawww] - path = /public_html/samba - comment = Samba WWW pages (approx 240 MB) - -[cvs] - path = /data/cvs - comment = CVS repository (requires authentication) - auth users = tridge, susan - secrets file = /etc/rsyncd.secrets -) - -The /etc/rsyncd.secrets file would look something like this: - -quote( -tt(tridge:mypass)nl() -tt(susan:herpass)nl() -) - -manpagefiles() - -/etc/rsyncd.conf or rsyncd.conf - -manpageseealso() - -bf(rsync)(1) - -manpagediagnostics() - -manpagebugs() - -Please report bugs! The rsync bug tracking system is online at -url(http://rsync.samba.org/)(http://rsync.samba.org/) - -manpagesection(VERSION) - -This man page is current for version 3.1.3 of rsync. - -manpagesection(CREDITS) - -rsync is distributed under the GNU General Public License. See the file -COPYING for details. - -The primary ftp site for rsync is -url(ftp://rsync.samba.org/pub/rsync)(ftp://rsync.samba.org/pub/rsync). - -A WEB site is available at -url(http://rsync.samba.org/)(http://rsync.samba.org/) - -We would be delighted to hear from you if you like this program. - -This program uses the zlib compression library written by Jean-loup -Gailly and Mark Adler. - -manpagesection(THANKS) - -Thanks to Warren Stanley for his original idea and patch for the rsync -daemon. Thanks to Karsten Thygesen for his many suggestions and -documentation! - -manpageauthor() - -rsync was written by Andrew Tridgell and Paul Mackerras. -Many people have later contributed to it. - -Mailing lists for support and development are available at -url(http://lists.samba.org)(lists.samba.org) diff --git a/runtests.sh b/runtests.sh index 42cd866e..536543f8 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,7 +1,7 @@ #! /bin/sh # Copyright (C) 2001, 2002 by Martin Pool <mbp@samba.org> -# Copyright (C) 2003, 2004, 2005, 2006 Wayne Davison +# Copyright (C) 2003-2020 Wayne Davison # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version @@ -11,16 +11,22 @@ # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# ------------------------------------------------------------------------- + # rsync top-level test script -- this invokes all the other more # detailed tests in order. This script can either be called by `make # check' or `make installcheck'. `check' runs against the copies of # the program and other files in the build directory, and -# `installcheck' against the installed copy of the program. +# `installcheck' against the installed copy of the program. + +# It can also be called on a single test file using a run like this: +# +# preserve_scratch=yes whichtests=itemize.test ./runtests.sh # In either case we need to also be able to find the source directory, # since we read test scripts and possibly other information from @@ -47,7 +53,7 @@ # build scripts. At the moment we assume we are invoked from the # source directory. -# This script must be invoked from the build directory. +# This script must be invoked from the build directory. # A scratch directory, 'testtmp', is used in the build directory to # hold per-test subdirectories. @@ -57,10 +63,11 @@ # Make command line. It's also set by the build farm to give more # detail for failing builds. +# ------------------------------------------------------------------------- # NOTES FOR TEST CASES: -# Each test case runs in its own shell. +# Each test case runs in its own shell. # Exit codes from tests: @@ -75,6 +82,7 @@ # rsync.fns contains some general setup functions and definitions. +# ------------------------------------------------------------------------- # NOTES ON PORTABILITY: @@ -103,6 +111,7 @@ # Don't rely on grep -q, as that doesn't work everywhere -- just redirect # stdout to /dev/null to keep it quiet. +# ------------------------------------------------------------------------- # STILL TO DO: @@ -141,7 +150,7 @@ if [ "x$loglevel" != x ] && [ "$loglevel" -gt 8 ]; then fi fi -POSIXLY_CORRECT=1 +POSIXLY_CORRECT=1 if test x"$TOOLDIR" = x; then TOOLDIR=`pwd` fi @@ -191,7 +200,7 @@ if [ "x$preserve_scratch" = xyes ]; then echo " preserve_scratch=yes" else echo " preserve_scratch=no" -fi +fi # Check if setacl/setfacl is around and if it supports the -k or -s option. if setacl -k u::7,g::5,o:5 testsuite 2>/dev/null; then @@ -301,7 +310,7 @@ do maybe_discard_scratch ;; 78) - # It failed, but we expected that. don't dump out error logs, + # It failed, but we expected that. don't dump out error logs, # because most users won't want to see them. But do leave # the working directory around. echo "XFAIL $testbase" @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -32,6 +32,7 @@ extern int logfile_format_has_i; extern int want_xattr_optim; extern int csum_length; extern int append_mode; +extern int copy_links; extern int io_error; extern int flist_eof; extern int allowed_lull; @@ -41,9 +42,11 @@ extern int remove_source_files; extern int updating_basis_file; extern int make_backups; extern int inplace; +extern int inplace_partial; extern int batch_fd; extern int write_batch; extern int file_old_total; +extern BOOL want_progress_now; extern struct stats stats; extern struct file_list *cur_flist, *first_flist, *dir_flist; @@ -138,17 +141,16 @@ void successful_send(int ndx) return; f_name(file, fname); - if (do_lstat(fname, &st) < 0) { + if ((copy_links ? do_stat(fname, &st) : do_lstat(fname, &st)) < 0) { failed_op = "re-lstat"; goto failed; } - if (S_ISREG(file->mode) /* Symlinks & devices don't need this check: */ - && (st.st_size != F_LENGTH(file) || st.st_mtime != file->modtime + if (st.st_size != F_LENGTH(file) || st.st_mtime != file->modtime #ifdef ST_MTIME_NSEC || (NSEC_BUMP(file) && (uint32)st.ST_MTIME_NSEC != F_MOD_NSEC(file)) #endif - )) { + ) { rprintf(FERROR_XFER, "ERROR: Skipping sender remove for changed file: %s\n", fname); return; } @@ -206,6 +208,8 @@ void send_files(int f_in, int f_out) if (DEBUG_GTE(SEND, 1)) rprintf(FINFO, "send_files starting\n"); + progress_init(); + while (1) { if (inc_recurse) { send_extra_file_list(f_out, MIN_FILECNT_LOOKAHEAD); @@ -218,9 +222,10 @@ void send_files(int f_in, int f_out) extra_flist_sending_enabled = False; if (ndx == NDX_DONE) { - if (!am_server && INFO_GTE(PROGRESS, 2) && cur_flist) { + if (!am_server && cur_flist) { set_current_file_index(NULL, 0); - end_progress(0); + if (INFO_GTE(PROGRESS, 2)) + end_progress(0); } if (inc_recurse && first_flist) { file_old_total -= first_flist->used; @@ -268,8 +273,7 @@ void send_files(int f_in, int f_out) if (!(iflags & ITEM_TRANSFER)) { maybe_log_item(file, iflags, itemizing, xname); - write_ndx_and_attrs(f_out, ndx, iflags, fname, file, - fnamecmp_type, xname, xlen); + write_ndx_and_attrs(f_out, ndx, iflags, fname, file, fnamecmp_type, xname, xlen); if (iflags & ITEM_IS_NEW) { stats.created_files++; if (S_ISREG(file->mode)) { @@ -312,10 +316,10 @@ void send_files(int f_in, int f_out) stats.created_files++; } - updating_basis_file = inplace && (protocol_version >= 29 - ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0); + updating_basis_file = (inplace_partial && fnamecmp_type == FNAMECMP_PARTIAL_DIR) + || (inplace && (protocol_version >= 29 ? fnamecmp_type == FNAMECMP_FNAME : make_backups <= 0)); - if (!am_server && INFO_GTE(PROGRESS, 1)) + if (!am_server) set_current_file_index(file, ndx); stats.xferred_files++; stats.total_transferred_size += F_LENGTH(file); @@ -324,8 +328,7 @@ void send_files(int f_in, int f_out) if (!do_xfers) { /* log the transfer */ log_item(FCLIENT, file, iflags, NULL); - write_ndx_and_attrs(f_out, ndx, iflags, fname, file, - fnamecmp_type, xname, xlen); + write_ndx_and_attrs(f_out, ndx, iflags, fname, file, fnamecmp_type, xname, xlen); continue; } @@ -338,9 +341,7 @@ void send_files(int f_in, int f_out) fd = do_open(fname, O_RDONLY, 0); if (fd == -1) { if (errno == ENOENT) { - enum logcode c = am_daemon - && protocol_version < 28 ? FERROR - : FWARNING; + enum logcode c = am_daemon && protocol_version < 28 ? FERROR : FWARNING; io_error |= IOERR_VANISHED; rprintf(c, "file has vanished: %s\n", full_fname(fname)); @@ -376,8 +377,7 @@ void send_files(int f_in, int f_out) path,slash,fname, big_num(st.st_size)); } - write_ndx_and_attrs(f_out, ndx, iflags, fname, file, - fnamecmp_type, xname, xlen); + write_ndx_and_attrs(f_out, ndx, iflags, fname, file, fnamecmp_type, xname, xlen); write_sum_head(f_xfer, s); if (DEBUG_GTE(DELTASUM, 2)) @@ -393,6 +393,8 @@ void send_files(int f_in, int f_out) match_sums(f_xfer, s, mbuf, st.st_size); if (INFO_GTE(PROGRESS, 1)) end_progress(st.st_size); + else if (want_progress_now) + instant_progress(fname); log_item(log_code, file, iflags, NULL); diff --git a/simd-checksum-x86_64.cpp b/simd-checksum-x86_64.cpp new file mode 100644 index 00000000..a1f5c502 --- /dev/null +++ b/simd-checksum-x86_64.cpp @@ -0,0 +1,543 @@ +/* + * SSE2/SSSE3/AVX2-optimized routines to support checksumming of bytes. + * + * Copyright (C) 1996 Andrew Tridgell + * Copyright (C) 1996 Paul Mackerras + * Copyright (C) 2004-2020 Wayne Davison + * Copyright (C) 2020 Jorrit Jongma + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ +/* + * Optimization target for get_checksum1() was the Intel Atom D2700, the + * slowest CPU in the test set and the most likely to be CPU limited during + * transfers. The combination of intrinsics was chosen specifically for the + * most gain on that CPU, other combinations were occasionally slightly + * faster on the others. + * + * While on more modern CPUs transfers are less likely to be CPU limited + * (at least by this specific function), lower CPU usage is always better. + * Improvements may still be seen when matching chunks from NVMe storage + * even on newer CPUs. + * + * Benchmarks (in MB/s) C SSE2 SSSE3 AVX2 + * - Intel Atom D2700 550 750 1000 N/A + * - Intel i7-7700hq 1850 2550 4050 6200 + * - AMD ThreadRipper 2950x 2900 5600 8950 8100 + * + * Curiously the AMD is slower with AVX2 than SSSE3, while the Intel is + * significantly faster. AVX2 is kept because it's more likely to relieve + * the bottleneck on the slower CPU. + * + * This optimization for get_checksum1() is intentionally limited to x86-64 + * as no 32-bit CPU was available for testing. As 32-bit CPUs only have half + * the available xmm registers, this optimized version may not be faster than + * the pure C version anyway. Note that all x86-64 CPUs support at least SSE2. + * + * This file is compiled using GCC 4.8+/clang 6+'s C++ front end to allow the + * use of the target attribute, selecting the fastest code path based on + * dispatch priority (GCC 5) or runtime detection of CPU capabilities (GCC 6+). + * GCC 4.x are not supported to ease configure.ac logic. + */ + +#ifdef __x86_64__ +#ifdef __cplusplus + +#include "rsync.h" + +#ifdef HAVE_SIMD + +#include <immintrin.h> + +/* Some clang versions don't like it when you use static with multi-versioned functions: linker errors */ +#ifdef __clang__ +#define MVSTATIC +#else +#define MVSTATIC static +#endif + +// Missing from the headers on gcc 6 and older, clang 8 and older +typedef long long __m128i_u __attribute__((__vector_size__(16), __may_alias__, __aligned__(1))); +typedef long long __m256i_u __attribute__((__vector_size__(32), __may_alias__, __aligned__(1))); + +/* Compatibility macros to let our SSSE3 algorithm run with only SSE2. + These used to be neat individual functions with target attributes switching between SSE2 and SSSE3 implementations + as needed, but though this works perfectly with GCC, clang fails to inline those properly leading to a near 50% + performance drop - combined with static and inline modifiers gets you linker errors and even compiler crashes... +*/ + +#define SSE2_INTERLEAVE_ODD_EPI16(a, b) _mm_packs_epi32(_mm_srai_epi32(a, 16), _mm_srai_epi32(b, 16)) +#define SSE2_INTERLEAVE_EVEN_EPI16(a, b) SSE2_INTERLEAVE_ODD_EPI16(_mm_slli_si128(a, 2), _mm_slli_si128(b, 2)) +#define SSE2_MULU_ODD_EPI8(a, b) _mm_mullo_epi16(_mm_srli_epi16(a, 8), _mm_srai_epi16(b, 8)) +#define SSE2_MULU_EVEN_EPI8(a, b) _mm_mullo_epi16(_mm_and_si128(a, _mm_set1_epi16(0xFF)), _mm_srai_epi16(_mm_slli_si128(b, 1), 8)) + +#define SSE2_HADDS_EPI16(a, b) _mm_adds_epi16(SSE2_INTERLEAVE_EVEN_EPI16(a, b), SSE2_INTERLEAVE_ODD_EPI16(a, b)) +#define SSE2_MADDUBS_EPI16(a, b) _mm_adds_epi16(SSE2_MULU_EVEN_EPI8(a, b), SSE2_MULU_ODD_EPI8(a, b)) + +__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; } +__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; } +__attribute__ ((target("default"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { return i; } + +/* + Original loop per 4 bytes: + s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + 10*CHAR_OFFSET; + s1 += buf[i] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET; + + SSE2/SSSE3 loop per 32 bytes: + int16 t1[8]; + int16 t2[8]; + for (int j = 0; j < 8; j++) { + t1[j] = buf[j*4 + i] + buf[j*4 + i+1] + buf[j*4 + i+2] + buf[j*4 + i+3]; + t2[j] = 4*buf[j*4 + i] + 3*buf[j*4 + i+1] + 2*buf[j*4 + i+2] + buf[j*4 + i+3]; + } + s2 += 32*s1 + (uint32)( + 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6] + + t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + ) + 528*CHAR_OFFSET; + s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7]) + + 32*CHAR_OFFSET; + */ +__attribute__ ((target("ssse3"))) MVSTATIC int32 get_checksum1_ssse3_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) +{ + if (len > 32) { + int aligned = ((uintptr_t)buf & 15) == 0; + + uint32 x[4] = {0}; + x[0] = *ps1; + __m128i ss1 = _mm_loadu_si128((__m128i_u*)x); + x[0] = *ps2; + __m128i ss2 = _mm_loadu_si128((__m128i_u*)x); + + const int16 mul_t1_buf[8] = {28, 24, 20, 16, 12, 8, 4, 0}; + __m128i mul_t1 = _mm_loadu_si128((__m128i_u*)mul_t1_buf); + + for (; i < (len-32); i+=32) { + // Load ... 2*[int8*16] + __m128i in8_1, in8_2; + if (!aligned) { + // Synonymous with _mm_loadu_si128 on all but a handful of old CPUs + in8_1 = _mm_lddqu_si128((__m128i_u*)&buf[i]); + in8_2 = _mm_lddqu_si128((__m128i_u*)&buf[i + 16]); + } else { + in8_1 = _mm_load_si128((__m128i_u*)&buf[i]); + in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]); + } + + // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8] + // Fastest, even though multiply by 1 + __m128i mul_one = _mm_set1_epi8(1); + __m128i add16_1 = _mm_maddubs_epi16(mul_one, in8_1); + __m128i add16_2 = _mm_maddubs_epi16(mul_one, in8_2); + + // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8] + __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24)); + __m128i mul_add16_1 = _mm_maddubs_epi16(mul_const, in8_1); + __m128i mul_add16_2 = _mm_maddubs_epi16(mul_const, in8_2); + + // s2 += 32*s1 + ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5)); + + // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16 + // Shifting left, then shifting right again and shuffling (rather than just + // shifting right as with mul32 below) to cheaply end up with the correct sign + // extension as we go from int16 to int32. + __m128i sum_add32 = _mm_add_epi16(add16_1, add16_2); + sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 2)); + sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 4)); + sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 8)); + sum_add32 = _mm_srai_epi32(sum_add32, 16); + sum_add32 = _mm_shuffle_epi32(sum_add32, 3); + + // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16 + __m128i sum_mul_add32 = _mm_add_epi16(mul_add16_1, mul_add16_2); + sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 2)); + sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 4)); + sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 8)); + sum_mul_add32 = _mm_srai_epi32(sum_mul_add32, 16); + sum_mul_add32 = _mm_shuffle_epi32(sum_mul_add32, 3); + + // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + ss1 = _mm_add_epi32(ss1, sum_add32); + + // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + ss2 = _mm_add_epi32(ss2, sum_mul_add32); + + // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8] + // We could've combined this with generating sum_add32 above and + // save an instruction but benchmarking shows that as being slower + __m128i add16 = _mm_hadds_epi16(add16_1, add16_2); + + // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4] + __m128i mul32 = _mm_madd_epi16(add16, mul_t1); + + // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32 + mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 4)); + mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 8)); + + // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6] + ss2 = _mm_add_epi32(ss2, mul32); + +#if CHAR_OFFSET != 0 + // s1 += 32*CHAR_OFFSET + __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET); + ss1 = _mm_add_epi32(ss1, char_offset_multiplier); + + // s2 += 528*CHAR_OFFSET + char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET); + ss2 = _mm_add_epi32(ss2, char_offset_multiplier); +#endif + } + + _mm_store_si128((__m128i_u*)x, ss1); + *ps1 = x[0]; + _mm_store_si128((__m128i_u*)x, ss2); + *ps2 = x[0]; + } + return i; +} + +/* + Same as SSSE3 version, but using macros defined above to emulate SSSE3 calls that are not available with SSE2. + For GCC-only the SSE2 and SSSE3 versions could be a single function calling other functions with the right + target attributes to emulate SSSE3 calls on SSE2 if needed, but clang doesn't inline those properly leading + to a near 50% performance drop. + */ +__attribute__ ((target("sse2"))) MVSTATIC int32 get_checksum1_sse2_32(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) +{ + if (len > 32) { + int aligned = ((uintptr_t)buf & 15) == 0; + + uint32 x[4] = {0}; + x[0] = *ps1; + __m128i ss1 = _mm_loadu_si128((__m128i_u*)x); + x[0] = *ps2; + __m128i ss2 = _mm_loadu_si128((__m128i_u*)x); + + const int16 mul_t1_buf[8] = {28, 24, 20, 16, 12, 8, 4, 0}; + __m128i mul_t1 = _mm_loadu_si128((__m128i_u*)mul_t1_buf); + + for (; i < (len-32); i+=32) { + // Load ... 2*[int8*16] + __m128i in8_1, in8_2; + if (!aligned) { + in8_1 = _mm_loadu_si128((__m128i_u*)&buf[i]); + in8_2 = _mm_loadu_si128((__m128i_u*)&buf[i + 16]); + } else { + in8_1 = _mm_load_si128((__m128i_u*)&buf[i]); + in8_2 = _mm_load_si128((__m128i_u*)&buf[i + 16]); + } + + // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*8] + // Fastest, even though multiply by 1 + __m128i mul_one = _mm_set1_epi8(1); + __m128i add16_1 = SSE2_MADDUBS_EPI16(mul_one, in8_1); + __m128i add16_2 = SSE2_MADDUBS_EPI16(mul_one, in8_2); + + // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*8] + __m128i mul_const = _mm_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24)); + __m128i mul_add16_1 = SSE2_MADDUBS_EPI16(mul_const, in8_1); + __m128i mul_add16_2 = SSE2_MADDUBS_EPI16(mul_const, in8_2); + + // s2 += 32*s1 + ss2 = _mm_add_epi32(ss2, _mm_slli_epi32(ss1, 5)); + + // [sum(t1[0]..t1[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16 + // Shifting left, then shifting right again and shuffling (rather than just + // shifting right as with mul32 below) to cheaply end up with the correct sign + // extension as we go from int16 to int32. + __m128i sum_add32 = _mm_add_epi16(add16_1, add16_2); + sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 2)); + sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 4)); + sum_add32 = _mm_add_epi16(sum_add32, _mm_slli_si128(sum_add32, 8)); + sum_add32 = _mm_srai_epi32(sum_add32, 16); + sum_add32 = _mm_shuffle_epi32(sum_add32, 3); + + // [sum(t2[0]..t2[7]), X, X, X] [int32*4]; faster than multiple _mm_hadds_epi16 + __m128i sum_mul_add32 = _mm_add_epi16(mul_add16_1, mul_add16_2); + sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 2)); + sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 4)); + sum_mul_add32 = _mm_add_epi16(sum_mul_add32, _mm_slli_si128(sum_mul_add32, 8)); + sum_mul_add32 = _mm_srai_epi32(sum_mul_add32, 16); + sum_mul_add32 = _mm_shuffle_epi32(sum_mul_add32, 3); + + // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + ss1 = _mm_add_epi32(ss1, sum_add32); + + // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + ss2 = _mm_add_epi32(ss2, sum_mul_add32); + + // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*8] + // We could've combined this with generating sum_add32 above and + // save an instruction but benchmarking shows that as being slower + __m128i add16 = SSE2_HADDS_EPI16(add16_1, add16_2); + + // [t1[0], t1[1], ...] -> [t1[0]*28 + t1[1]*24, ...] [int32*4] + __m128i mul32 = _mm_madd_epi16(add16, mul_t1); + + // [sum(mul32), X, X, X] [int32*4]; faster than multiple _mm_hadd_epi32 + mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 4)); + mul32 = _mm_add_epi32(mul32, _mm_srli_si128(mul32, 8)); + + // s2 += 28*t1[0] + 24*t1[1] + 20*t1[2] + 16*t1[3] + 12*t1[4] + 8*t1[5] + 4*t1[6] + ss2 = _mm_add_epi32(ss2, mul32); + +#if CHAR_OFFSET != 0 + // s1 += 32*CHAR_OFFSET + __m128i char_offset_multiplier = _mm_set1_epi32(32 * CHAR_OFFSET); + ss1 = _mm_add_epi32(ss1, char_offset_multiplier); + + // s2 += 528*CHAR_OFFSET + char_offset_multiplier = _mm_set1_epi32(528 * CHAR_OFFSET); + ss2 = _mm_add_epi32(ss2, char_offset_multiplier); +#endif + } + + _mm_store_si128((__m128i_u*)x, ss1); + *ps1 = x[0]; + _mm_store_si128((__m128i_u*)x, ss2); + *ps2 = x[0]; + } + return i; +} + +/* + AVX2 loop per 64 bytes: + int16 t1[16]; + int16 t2[16]; + for (int j = 0; j < 16; j++) { + t1[j] = buf[j*4 + i] + buf[j*4 + i+1] + buf[j*4 + i+2] + buf[j*4 + i+3]; + t2[j] = 4*buf[j*4 + i] + 3*buf[j*4 + i+1] + 2*buf[j*4 + i+2] + buf[j*4 + i+3]; + } + s2 += 64*s1 + (uint32)( + 60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14] + + t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15] + ) + 2080*CHAR_OFFSET; + s1 += (uint32)(t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15]) + + 64*CHAR_OFFSET; + */ +__attribute__ ((target("avx2"))) MVSTATIC int32 get_checksum1_avx2_64(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) +{ + if (len > 64) { + // Instructions reshuffled compared to SSE2 for slightly better performance + int aligned = ((uintptr_t)buf & 31) == 0; + + uint32 x[8] = {0}; + x[0] = *ps1; + __m256i ss1 = _mm256_lddqu_si256((__m256i_u*)x); + x[0] = *ps2; + __m256i ss2 = _mm256_lddqu_si256((__m256i_u*)x); + + // The order gets shuffled compared to SSE2 + const int16 mul_t1_buf[16] = {60, 56, 52, 48, 28, 24, 20, 16, 44, 40, 36, 32, 12, 8, 4, 0}; + __m256i mul_t1 = _mm256_lddqu_si256((__m256i_u*)mul_t1_buf); + + for (; i < (len-64); i+=64) { + // Load ... 2*[int8*32] + __m256i in8_1, in8_2; + if (!aligned) { + in8_1 = _mm256_lddqu_si256((__m256i_u*)&buf[i]); + in8_2 = _mm256_lddqu_si256((__m256i_u*)&buf[i + 32]); + } else { + in8_1 = _mm256_load_si256((__m256i_u*)&buf[i]); + in8_2 = _mm256_load_si256((__m256i_u*)&buf[i + 32]); + } + + // Prefetch for next loops. This has no observable effect on the + // tested AMD but makes as much as 20% difference on the Intel. + // Curiously that same Intel sees no benefit from this with SSE2 + // or SSSE3. + _mm_prefetch(&buf[i + 64], _MM_HINT_T0); + _mm_prefetch(&buf[i + 96], _MM_HINT_T0); + _mm_prefetch(&buf[i + 128], _MM_HINT_T0); + _mm_prefetch(&buf[i + 160], _MM_HINT_T0); + + // (1*buf[i] + 1*buf[i+1]), (1*buf[i+2], 1*buf[i+3]), ... 2*[int16*16] + // Fastest, even though multiply by 1 + __m256i mul_one = _mm256_set1_epi8(1); + __m256i add16_1 = _mm256_maddubs_epi16(mul_one, in8_1); + __m256i add16_2 = _mm256_maddubs_epi16(mul_one, in8_2); + + // (4*buf[i] + 3*buf[i+1]), (2*buf[i+2], buf[i+3]), ... 2*[int16*16] + __m256i mul_const = _mm256_set1_epi32(4 + (3 << 8) + (2 << 16) + (1 << 24)); + __m256i mul_add16_1 = _mm256_maddubs_epi16(mul_const, in8_1); + __m256i mul_add16_2 = _mm256_maddubs_epi16(mul_const, in8_2); + + // s2 += 64*s1 + ss2 = _mm256_add_epi32(ss2, _mm256_slli_epi32(ss1, 6)); + + // [t1[0] + t1[1], t1[2] + t1[3] ...] [int16*16] + __m256i add16 = _mm256_hadds_epi16(add16_1, add16_2); + + // [t1[0], t1[1], ...] -> [t1[0]*60 + t1[1]*56, ...] [int32*8] + __m256i mul32 = _mm256_madd_epi16(add16, mul_t1); + + // [sum(t1[0]..t1[15]), X, X, X, X, X, X, X] [int32*8] + __m256i sum_add32 = _mm256_add_epi16(add16_1, add16_2); + sum_add32 = _mm256_add_epi16(sum_add32, _mm256_permute4x64_epi64(sum_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6))); + sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 2)); + sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 4)); + sum_add32 = _mm256_add_epi16(sum_add32, _mm256_slli_si256(sum_add32, 8)); + sum_add32 = _mm256_srai_epi32(sum_add32, 16); + sum_add32 = _mm256_shuffle_epi32(sum_add32, 3); + + // s1 += t1[0] + t1[1] + t1[2] + t1[3] + t1[4] + t1[5] + t1[6] + t1[7] + t1[8] + t1[9] + t1[10] + t1[11] + t1[12] + t1[13] + t1[14] + t1[15] + ss1 = _mm256_add_epi32(ss1, sum_add32); + + // [sum(t2[0]..t2[15]), X, X, X, X, X, X, X] [int32*8] + __m256i sum_mul_add32 = _mm256_add_epi16(mul_add16_1, mul_add16_2); + sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_permute4x64_epi64(sum_mul_add32, 2 + (3 << 2) + (0 << 4) + (1 << 6))); + sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 2)); + sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 4)); + sum_mul_add32 = _mm256_add_epi16(sum_mul_add32, _mm256_slli_si256(sum_mul_add32, 8)); + sum_mul_add32 = _mm256_srai_epi32(sum_mul_add32, 16); + sum_mul_add32 = _mm256_shuffle_epi32(sum_mul_add32, 3); + + // s2 += t2[0] + t2[1] + t2[2] + t2[3] + t2[4] + t2[5] + t2[6] + t2[7] + t2[8] + t2[9] + t2[10] + t2[11] + t2[12] + t2[13] + t2[14] + t2[15] + ss2 = _mm256_add_epi32(ss2, sum_mul_add32); + + // [sum(mul32), X, X, X, X, X, X, X] [int32*8] + mul32 = _mm256_add_epi32(mul32, _mm256_permute2x128_si256(mul32, mul32, 1)); + mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 4)); + mul32 = _mm256_add_epi32(mul32, _mm256_srli_si256(mul32, 8)); + + // s2 += 60*t1[0] + 56*t1[1] + 52*t1[2] + 48*t1[3] + 44*t1[4] + 40*t1[5] + 36*t1[6] + 32*t1[7] + 28*t1[8] + 24*t1[9] + 20*t1[10] + 16*t1[11] + 12*t1[12] + 8*t1[13] + 4*t1[14] + ss2 = _mm256_add_epi32(ss2, mul32); + +#if CHAR_OFFSET != 0 + // s1 += 64*CHAR_OFFSET + __m256i char_offset_multiplier = _mm256_set1_epi32(64 * CHAR_OFFSET); + ss1 = _mm256_add_epi32(ss1, char_offset_multiplier); + + // s2 += 2080*CHAR_OFFSET + char_offset_multiplier = _mm256_set1_epi32(2080 * CHAR_OFFSET); + ss2 = _mm256_add_epi32(ss2, char_offset_multiplier); +#endif + } + + _mm256_store_si256((__m256i_u*)x, ss1); + *ps1 = x[0]; + _mm256_store_si256((__m256i_u*)x, ss2); + *ps2 = x[0]; + } + return i; +} + +static int32 get_checksum1_default_1(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) +{ + uint32 s1 = *ps1; + uint32 s2 = *ps2; + for (; i < (len-4); i+=4) { + s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + 10*CHAR_OFFSET; + s1 += (buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET); + } + for (; i < len; i++) { + s1 += (buf[i]+CHAR_OFFSET); s2 += s1; + } + *ps1 = s1; + *ps2 = s2; + return i; +} + +/* With GCC 10 putting this implementation inside 'extern "C"' causes an + assembler error. That worked fine on GCC 5-9 and clang 6-10... + */ +static inline uint32 get_checksum1_cpp(char *buf1, int32 len) +{ + int32 i = 0; + uint32 s1 = 0; + uint32 s2 = 0; + + // multiples of 64 bytes using AVX2 (if available) + i = get_checksum1_avx2_64((schar*)buf1, len, i, &s1, &s2); + + // multiples of 32 bytes using SSSE3 (if available) + i = get_checksum1_ssse3_32((schar*)buf1, len, i, &s1, &s2); + + // multiples of 32 bytes using SSE2 (if available) + i = get_checksum1_sse2_32((schar*)buf1, len, i, &s1, &s2); + + // whatever is left + i = get_checksum1_default_1((schar*)buf1, len, i, &s1, &s2); + + return (s1 & 0xffff) + (s2 << 16); +} + +extern "C" { + +uint32 get_checksum1(char *buf1, int32 len) +{ + return get_checksum1_cpp(buf1, len); +} + +} // extern "C" + +#ifdef BENCHMARK_SIMD_CHECKSUM1 +#pragma clang optimize off +#pragma GCC push_options +#pragma GCC optimize ("O0") + +#define ROUNDS 1024 +#define BLOCK_LEN 1024*1024 + +#ifndef CLOCK_MONOTONIC_RAW +#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC +#endif + +static void benchmark(const char* desc, int32 (*func)(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2), schar* buf, int32 len) { + struct timespec start, end; + uint64_t us; + uint32_t cs, s1, s2; + int i, next; + + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + for (i = 0; i < ROUNDS; i++) { + s1 = s2 = 0; + next = func((schar*)buf, len, 0, &s1, &s2); + get_checksum1_default_1((schar*)buf, len, next, &s1, &s2); + } + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + us = next == 0 ? 0 : (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_nsec - start.tv_nsec) / 1000; + cs = next == 0 ? 0 : (s1 & 0xffff) + (s2 << 16); + printf("%-5s :: %5.0f MB/s :: %08x\n", desc, us ? (float)(len / (1024 * 1024) * ROUNDS) / ((float)us / 1000000.0f) : 0, cs); +} + +static int32 get_checksum1_auto(schar* buf, int32 len, int32 i, uint32* ps1, uint32* ps2) { + uint32 cs = get_checksum1((char*)buf, len); + *ps1 = cs & 0xffff; + *ps2 = cs >> 16; + return len; +} + +int main() { + int i; + unsigned char* buf = (unsigned char*)malloc(BLOCK_LEN); + for (i = 0; i < BLOCK_LEN; i++) buf[i] = (i + (i % 3) + (i % 11)) % 256; + + benchmark("Auto", get_checksum1_auto, (schar*)buf, BLOCK_LEN); + benchmark("Raw-C", get_checksum1_default_1, (schar*)buf, BLOCK_LEN); + benchmark("SSE2", get_checksum1_sse2_32, (schar*)buf, BLOCK_LEN); + benchmark("SSSE3", get_checksum1_ssse3_32, (schar*)buf, BLOCK_LEN); + benchmark("AVX2", get_checksum1_avx2_64, (schar*)buf, BLOCK_LEN); + + free(buf); + return 0; +} + +#pragma GCC pop_options +#pragma clang optimize on +#endif /* BENCHMARK_SIMD_CHECKSUM1 */ + +#endif /* HAVE_SIMD */ +#endif /* __cplusplus */ +#endif /* __x86_64__ */ @@ -3,7 +3,7 @@ * * Copyright (C) 1992-2001 Andrew Tridgell <tridge@samba.org> * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -38,6 +38,7 @@ extern char *bind_address; extern char *sockopts; extern int default_af_hint; extern int connect_timeout; +extern int pid_file_fd; #ifdef HAVE_SIGACTION static struct sigaction sigact; @@ -48,8 +49,7 @@ static int sock_exec(const char *prog); /* Establish a proxy connection on an open socket to a web proxy by using the * CONNECT method. If proxy_user and proxy_pass are not NULL, they are used to * authenticate to the proxy using the "Basic" proxy-authorization protocol. */ -static int establish_proxy_connection(int fd, char *host, int port, - char *proxy_user, char *proxy_pass) +static int establish_proxy_connection(int fd, char *host, int port, char *proxy_user, char *proxy_pass) { char *cp, buffer[1024]; char *authhdr, authbuf[1024]; @@ -73,9 +73,8 @@ static int establish_proxy_connection(int fd, char *host, int port, authhdr = ""; } - snprintf(buffer, sizeof buffer, "CONNECT %s:%d HTTP/1.0%s%s\r\n\r\n", - host, port, authhdr, authbuf); - len = strlen(buffer); + len = snprintf(buffer, sizeof buffer, "CONNECT %s:%d HTTP/1.0%s%s\r\n\r\n", host, port, authhdr, authbuf); + assert(len > 0 && len < (int)sizeof buffer); if (write(fd, buffer, len) != len) { rsyserr(FERROR, errno, "failed to write to proxy"); return -1; @@ -182,8 +181,7 @@ static void contimeout_handler(UNUSED(int val)) * bind_addr: local address to use. Normally NULL to bind the wildcard address. * * af_hint: address family, e.g. AF_INET or AF_INET6. */ -int open_socket_out(char *host, int port, const char *bind_addr, - int af_hint) +int open_socket_out(char *host, int port, const char *bind_addr, int af_hint) { int type = SOCK_STREAM; int error, s, j, addr_cnt, *errnos; @@ -195,7 +193,7 @@ int open_socket_out(char *host, int port, const char *bind_addr, char *proxy_user = NULL, *proxy_pass = NULL; /* if we have a RSYNC_PROXY env variable then redirect our - * connetcion via a web proxy at the given address. */ + * connection via a web proxy at the given address. */ h = getenv("RSYNC_PROXY"); proxied = h != NULL && *h != '\0'; @@ -294,9 +292,7 @@ int open_socket_out(char *host, int port, const char *bind_addr, continue; } - if (proxied - && establish_proxy_connection(s, host, port, - proxy_user, proxy_pass) != 0) { + if (proxied && establish_proxy_connection(s, host, port, proxy_user, proxy_pass) != 0) { close(s); s = -1; continue; @@ -340,8 +336,7 @@ int open_socket_out(char *host, int port, const char *bind_addr, * This is based on the Samba LIBSMB_PROG feature. * * bind_addr: local address to use. Normally NULL to get the stack default. */ -int open_socket_out_wrapped(char *host, int port, const char *bind_addr, - int af_hint) +int open_socket_out_wrapped(char *host, int port, const char *bind_addr, int af_hint) { char *prog = getenv("RSYNC_CONNECT_PROG"); @@ -458,9 +453,8 @@ static int *open_socket_in(int type, int port, const char *bind_addr, #ifdef IPV6_V6ONLY if (resp->ai_family == AF_INET6) { - if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, - (char *)&one, sizeof one) < 0 - && default_af_hint != AF_INET6) { + if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&one, sizeof one) < 0 + && default_af_hint != AF_INET6) { close(s); continue; } @@ -561,8 +555,7 @@ void start_accept_loop(int port, int (*fn)(int, int)) rsyserr(FERROR, errno, "listen() on socket failed"); #ifdef INET6 if (errno == EADDRINUSE && i > 0) { - rprintf(FINFO, - "Try using --ipv4 or --ipv6 to avoid this listen() error.\n"); + rprintf(FINFO, "Try using --ipv4 or --ipv6 to avoid this listen() error.\n"); } #endif exit_cleanup(RERR_SOCKETIO); @@ -597,8 +590,7 @@ void start_accept_loop(int port, int (*fn)(int, int)) for (i = 0, fd = -1; sp[i] >= 0; i++) { if (FD_ISSET(sp[i], &fds)) { - fd = accept(sp[i], (struct sockaddr *)&addr, - &addrlen); + fd = accept(sp[i], (struct sockaddr *)&addr, &addrlen); break; } } @@ -610,6 +602,8 @@ void start_accept_loop(int port, int (*fn)(int, int)) if ((pid = fork()) == 0) { int ret; + if (pid_file_fd >= 0) + close(pid_file_fd); for (i = 0; sp[i] >= 0; i++) close(sp[i]); /* Re-open log file in child before possibly giving @@ -797,8 +791,7 @@ static int socketpair_tcp(int fd[2]) set_blocking(fd[1]); if (connect_done == 0) { - if (connect(fd[1], (struct sockaddr *)&sock, sizeof sock) != 0 - && errno != EISCONN) + if (connect(fd[1], (struct sockaddr *)&sock, sizeof sock) != 0 && errno != EISCONN) goto failed; } @@ -820,7 +813,7 @@ static int socketpair_tcp(int fd[2]) * stdout. This is used to fake a connection to a daemon for testing -- not * for the normal case of running SSH. * - * Retruns a socket which is attached to a subprocess running "prog". stdin and + * Returns a socket which is attached to a subprocess running "prog". stdin and * stdout are attached. stderr is left attached to the original stderr. */ static int sock_exec(const char *prog) { @@ -847,7 +840,7 @@ static int sock_exec(const char *prog) fprintf(stderr, "Failed to run \"%s\"\n", prog); exit(1); } - exit(system(prog)); + exit(shell_exec(prog)); } close(fd[1]); diff --git a/stunnel-rsync.in b/stunnel-rsync.in deleted file mode 100755 index fdd8bfd1..00000000 --- a/stunnel-rsync.in +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# This must be called as (note the trailing dot): -# -# stunnel-rsync HOSTNAME rsync --server --daemon . -# -# ... which is typically done via the rsync-ssl script, which results in something like this: -# -# rsync --rsh=stunnel-rsync -aiv HOSTNAME::module [ARGS] -# -# This SSL setup based on the files by: http://dozzie.jarowit.net/trac/wiki/RsyncSSL -# Note that this requires at least version 4.x of stunnel. - -# The current environment can override using the RSYNC_SSL_* values: -if [ x"$RSYNC_SSL_CERT" = x ]; then - cert="" -else - cert="cert = $RSYNC_SSL_CERT" -fi -if [ x"$RSYNC_SSL_CA_CERT" ]; then - cafile="" - verify=0 -else - cafile="CAfile = $RSYNC_SSL_CA_CERT" - verify=3 -fi -port=${RSYNC_SSL_PORT:-874} - -# If the user specified USER@HOSTNAME::module, then rsync passes us -# the -l USER option too, so we must be prepared to ignore it. -if [ x"$1" = x"-l" ]; then - shift 2 -fi - -hostname=$1 -shift - -if [ x"$hostname" = x -o x"$1" != x"rsync" -o x"$2" != x"--server" -o x"$3" != x"--daemon" ]; then - echo "Usage: stunnel-rsync HOSTNAME rsync --server --daemon ." 1>&2 - exit 1 -fi - -# devzero@web.de came up with this no-tmpfile calling syntax: -@stunnel4@ -fd 10 11<&0 <<EOF 10<&0 0<&11 11<&- -foreground = yes -debug = crit -connect = $hostname:$port -client = yes -TIMEOUTclose = 0 -verify = $verify -$cert -$cafile -EOF diff --git a/stunnel-rsyncd.conf.in b/stunnel-rsyncd.conf.in index fd28d5d4..b3fd240e 100644 --- a/stunnel-rsyncd.conf.in +++ b/stunnel-rsyncd.conf.in @@ -4,7 +4,7 @@ foreground = no pid = /var/run/stunnel-rsyncd.pid socket = l:TCP_NODELAY=1 socket = r:TCP_NODELAY=1 -compression = rle +#compression = rle # This must be root for rsync to use chroot -- rsync will drop permissions: setuid = root setgid = root @@ -18,7 +18,7 @@ client = no # To allow anyone to try an ssl connection, use this: verify = 0 -CAfile = /etc/ssl/ca-bundle.pem +CAfile = /etc/ssl/certs/ca-certificates.crt # To allow only cert-authorized clients, use something like this instead of the above: #verify = 3 diff --git a/support/atomic-rsync b/support/atomic-rsync index 9e620cfe..0346fb49 100755 --- a/support/atomic-rsync +++ b/support/atomic-rsync @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # # This script lets you update a hierarchy of files in an atomic way by # first creating a new hierarchy using rsync's --link-dest option, and @@ -59,7 +59,7 @@ if (system($RSYNC_PROG, "--link-dest=$dest_dir", @ARGV)) { } else { printf "child exited with value %d\n", $? >> 8; } - exit $?; + exit 1; } if (!defined $old_dir) { diff --git a/support/cvs2includes b/support/cvs2includes index 33981de7..fc7f78fb 100755 --- a/support/cvs2includes +++ b/support/cvs2includes @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # # This script finds all CVS/Entries files in the current directory and below # and creates a local .cvsinclude file with non-inherited rules including each diff --git a/support/deny-rsync b/support/deny-rsync index 1993e3aa..bd4da9e7 100755 --- a/support/deny-rsync +++ b/support/deny-rsync @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Send an error message via the rsync-protocol to a non-daemon client rsync. # # Usage: deny-rsync "message" diff --git a/support/file-attr-restore b/support/file-attr-restore index a053e1d9..2e4a21b7 100755 --- a/support/file-attr-restore +++ b/support/file-attr-restore @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This script will parse the output of "find ARG [ARG...] -ls" and # apply (at your discretion) the permissions, owner, and group info # it reads onto any existing files and dirs (it doesn't try to affect diff --git a/support/files-to-excludes b/support/files-to-excludes index 5fb13b08..a28955cb 100755 --- a/support/files-to-excludes +++ b/support/files-to-excludes @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This script takes an input of filenames and outputs a set of # include/exclude directives that can be used by rsync to copy # just the indicated files using an --exclude-from=FILE option. diff --git a/support/git-set-file-times b/support/git-set-file-times index 077ac0e0..b4c98c5d 100755 --- a/support/git-set-file-times +++ b/support/git-set-file-times @@ -1,37 +1,93 @@ -#!/usr/bin/perl -use strict; -use warnings; - -# Sets mtime and atime of files to the latest commit time in git. -# -# This is useful after the first clone of the rsync repository BEFORE you -# do any building. It is also safe if you have done a "make distclean". - -my %ls; -my $commit_time; -my $prefix = @ARGV && $ARGV[0] =~ s/^--prefix=// ? shift : ''; - -$/ = "\0"; -open FH, 'git ls-files -z|' or die $!; -while (<FH>) { - chomp; - $ls{$_} = $_; -} -close FH; - -$/ = "\n"; -open FH, "git log -r --name-only --no-color --pretty=raw -z @ARGV |" or die $!; -while (<FH>) { - chomp; - if (/^committer .*? (\d+) (?:[\-\+]\d+)$/) { - $commit_time = $1; - } elsif (s/\0\0commit [a-f0-9]{40}$// or s/\0$//) { - my @files = delete @ls{split(/\0/, $_)}; - @files = grep { defined $_ } @files; - next unless @files; - map { s/^/$prefix/ } @files; - utime $commit_time, $commit_time, @files; - } - last unless %ls; -} -close FH; +#!/usr/bin/env python3 + +import os, re, argparse, subprocess +from datetime import datetime + +NULL_COMMIT_RE = re.compile(r'\0\0commit [a-f0-9]{40}$|\0$') + +def main(): + if not args.git_dir: + cmd = 'git rev-parse --show-toplevel 2>/dev/null || echo .' + top_dir = subprocess.check_output(cmd, shell=True, encoding='utf-8').strip() + args.git_dir = os.path.join(top_dir, '.git') + if not args.prefix: + os.chdir(top_dir) + + git = [ 'git', '--git-dir=' + args.git_dir ] + + if args.tree: + cmd = git + 'ls-tree -z -r --name-only'.split() + [ args.tree ] + else: + cmd = git + 'ls-files -z'.split() + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8') + out = proc.communicate()[0] + ls = set(out.split('\0')) + ls.discard('') + + if not args.tree: + # All modified files keep their current mtime. + proc = subprocess.Popen(git + 'ls-files -m -z'.split(), stdout=subprocess.PIPE, encoding='utf-8') + out = proc.communicate()[0] + for fn in out.split('\0'): + if fn == '': + continue + if args.list: + mtime = os.lstat(fn).st_mtime + print_line(fn, mtime, mtime) + ls.discard(fn) + + cmd = git + 'log -r --name-only --no-color --pretty=raw --no-renames -z'.split() + if args.tree: + cmd.append(args.tree) + cmd += ['--'] + args.files + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, encoding='utf-8') + for line in proc.stdout: + line = line.strip() + m = re.match(r'^committer .*? (\d+) [-+]\d+$', line) + if m: + commit_time = int(m[1]) + elif NULL_COMMIT_RE.search(line): + line = NULL_COMMIT_RE.sub('', line) + files = set(fn for fn in line.split('\0') if fn in ls) + if not files: + continue + for fn in files: + if args.prefix: + fn = args.prefix + fn + mtime = os.lstat(fn).st_mtime + if args.list: + print_line(fn, mtime, commit_time) + elif mtime != commit_time: + if not args.quiet: + print(f"Setting {fn}") + os.utime(fn, (commit_time, commit_time), follow_symlinks = False) + ls -= files + if not ls: + break + proc.communicate() + + +def print_line(fn, mtime, commit_time): + if args.list > 1: + ts = str(commit_time).rjust(10) + else: + ts = datetime.utcfromtimestamp(commit_time).strftime("%Y-%m-%d %H:%M:%S") + chg = '.' if mtime == commit_time else '*' + print(chg, ts, fn) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Set the times of the current git checkout to their last-changed time.", add_help=False) + parser.add_argument('--git-dir', metavar='GIT_DIR', help="The git dir to query (defaults to affecting the current git checkout).") + parser.add_argument('--tree', metavar='TREE-ISH', help="The tree-ish to query (defaults to the current branch).") + parser.add_argument('--prefix', metavar='PREFIX_STR', help="Prepend the PREFIX_STR to each filename we tweak.") + parser.add_argument('--quiet', '-q', action='store_true', help="Don't output the changed-file information.") + parser.add_argument('--list', '-l', action='count', help="List files & times instead of changing them. Repeat for Unix timestamp instead of human readable.") + parser.add_argument('files', metavar='FILE', nargs='*', help="Specify a subset of checked-out files to tweak.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/support/instant-rsyncd b/support/instant-rsyncd index e9efb1ab..8bcfd001 100755 --- a/support/instant-rsyncd +++ b/support/instant-rsyncd @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # instant-rsyncd lets you quickly set up and start a simple, unprivileged rsync # daemon with a single module in the current directory. I've found it diff --git a/support/logfilter b/support/logfilter index 282914aa..29cfe690 100755 --- a/support/logfilter +++ b/support/logfilter @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Filter the rsync daemon log messages by module name. The log file can be # in either syslog format or rsync's own log-file format. Note that the # MODULE_NAME parameter is used in a regular-expression match in order to diff --git a/support/lsh b/support/lsh index 2f64d311..40f70c88 100755 --- a/support/lsh +++ b/support/lsh @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This script can be used as a "remote shell" command that is only # capable of pretending to connect to "localhost". This is useful # for testing or for running a local copy where the sender and the @@ -26,7 +26,9 @@ my $host = shift; if ($host =~ s/^([^@]+)\@//) { $login_name = $1; } -if ($host ne 'localhost') { +if ($host eq 'lh') { + $no_chdir = 1; +} elsif ($host ne 'localhost') { die "lsh: unable to connect to host $host\n"; } @@ -77,5 +79,8 @@ sub usage { die <<EOT; Usage: lsh [-l user] [--sudo] [--no-cd] localhost COMMAND [...] + +Note that if you pass hostname "lh" instead of "localhost" that +the --no-cd option is implied. EOT } diff --git a/support/lsh.sh b/support/lsh.sh index b63444e3..f74a8ef4 100755 --- a/support/lsh.sh +++ b/support/lsh.sh @@ -7,7 +7,7 @@ # command. user='' -do_cd=y # Default path is user's home dir, just like ssh. +do_cd=y # Default path is user's home dir (just like ssh) unless host is "lh". while : ; do case "$1" in @@ -16,6 +16,7 @@ while : ; do --no-cd) do_cd=n; shift ;; -*) shift ;; localhost) shift; break ;; + lh) do_cd=n; shift; break ;; *) echo "lsh: unable to connect to host $1" 1>&2; exit 1 ;; esac done diff --git a/support/mapfrom b/support/mapfrom index ca3b8173..88946bc0 100755 --- a/support/mapfrom +++ b/support/mapfrom @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This helper script makes it easy to use a passwd or group file to map # values in a LOCAL transfer. For instance, if you mount a backup that # does not have the same passwd setup as the local machine, you can do diff --git a/support/mapto b/support/mapto index a1dbe3c1..95887523 100755 --- a/support/mapto +++ b/support/mapto @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This helper script makes it easy to use a passwd or group file to map # values in a LOCAL transfer. For instance, if you mount a backup that # does not have the same passwd setup as the local machine, you can do diff --git a/support/mnt-excl b/support/mnt-excl index b2926db4..ed7b49ba 100755 --- a/support/mnt-excl +++ b/support/mnt-excl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This script takes a command-line arg of a source directory # that will be passed to rsync, and generates a set of excludes # that will exclude all mount points from the list. This is diff --git a/support/munge-symlinks b/support/munge-symlinks index 2aa27f36..3e5f3ad2 100755 --- a/support/munge-symlinks +++ b/support/munge-symlinks @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # This script will either prefix all symlink values with the string # "/rsyncd-munged/" or remove that prefix. diff --git a/support/rrsync b/support/rrsync index 9195aa2f..34e62300 100644 --- a/support/rrsync +++ b/support/rrsync @@ -1,8 +1,8 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Name: /usr/local/bin/rrsync (should also have a symlink in /usr/bin) # Purpose: Restricts rsync to subdirectory declared in .ssh/authorized_keys # Author: Joe Smith <js-cgi@inwap.com> 30-Sep-2004 -# Modified by: Wayne Davison <wayned@samba.org> +# Modified by: Wayne Davison <wayne@opencoder.net> use strict; use Socket; @@ -62,8 +62,8 @@ die "$0 reading from write-only server not allowed\n" if $only eq 'w' && $am_sen # To disable a short-named option, add its letter to this string: our $short_disabled = 's'; -our $short_no_arg = 'ACDEHIJKLORSWXbcdgklmnoprstuvxyz'; # DO NOT REMOVE ANY -our $short_with_num = 'B'; # DO NOT REMOVE ANY +our $short_no_arg = 'ACDEHIJKLORSUWXbcdgklmnopqrstuvxyz'; # DO NOT REMOVE ANY +our $short_with_num = '@B'; # DO NOT REMOVE ANY # To disable a long-named option, change its value to a -1. The values mean: # 0 = the option has no arg; 1 = the arg doesn't need any checking; 2 = only @@ -73,8 +73,10 @@ our %long_opt = ( 'backup-dir' => 2, 'block-size' => 1, 'bwlimit' => 1, + 'checksum-choice' => 1, 'checksum-seed' => 1, 'compare-dest' => 2, + 'compress-choice' => 1, 'compress-level' => 1, 'copy-dest' => 2, 'copy-unsafe-links' => 0, @@ -107,7 +109,7 @@ our %long_opt = ( 'link-dest' => 2, 'links' => 0, 'list-only' => 0, - 'log-file' => 3, + 'log-file' => $only eq 'r' ? -1 : 3, 'log-format' => 1, 'max-delete' => 1, 'max-size' => 1, @@ -119,8 +121,10 @@ our %long_opt = ( 'no-relative' => 0, 'no-specials' => 0, 'numeric-ids' => 0, + 'old-compress' => 0, 'one-file-system' => 0, 'only-write-batch' => 1, + 'open-noatime' => 0, 'owner' => 0, 'partial' => 0, 'partial-dir' => 2, @@ -130,7 +134,7 @@ our %long_opt = ( 'remove-sent-files' => $only eq 'r' ? -1 : 0, 'remove-source-files' => $only eq 'r' ? -1 : 0, 'safe-links' => 0, - 'sender' => 0, + 'sender' => $only eq 'w' ? -1 : 0, 'server' => 0, 'size-only' => 0, 'skip-compress' => 1, @@ -143,6 +147,7 @@ our %long_opt = ( 'times' => 0, 'use-qsort' => 0, 'usermap' => 1, + 'write-devices' => -1, ); ### END of options data produced by the cull_options script. ### @@ -207,13 +212,16 @@ while ($command =~ /((?:[^\s\\]+|\\.[^\s\\]*)+)/g) { s{//+}{/}g; s{^/}{}; s{^$}{.}; - die "$0: do not use .. in any path!\n" if m{(^|/)\\?\.\\?\.(\\?/|$)}; } push(@args, bsd_glob($_, GLOB_LIMIT|GLOB_NOCHECK|GLOB_BRACE|GLOB_QUOTE)); } } die "$0: invalid rsync-command syntax or options\n" if $in_options; +if ($subdir ne '/') { + die "$0: do not use .. in any path!\n" if grep m{(^|/)\.\.(/|$)}, @args; +} + @args = ( '.' ) if !@args; if ($write_log) { @@ -227,7 +235,7 @@ if ($write_log) { } # Note: This assumes that the rsync protocol will not be maliciously hijacked. -exec(RSYNC, @opts, @args) or die "exec(rsync @opts @args) failed: $? $!"; +exec(RSYNC, @opts, '--', @args) or die "exec(rsync @opts -- @args) failed: $? $!"; sub check_arg { diff --git a/support/rsync-no-vanished b/support/rsync-no-vanished index 1cce75cb..cc3c29ce 100755 --- a/support/rsync-no-vanished +++ b/support/rsync-no-vanished @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash IGNOREEXIT=24 IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' diff --git a/support/rsync-slash-strip b/support/rsync-slash-strip index 43b82bf4..2869e45c 100755 --- a/support/rsync-slash-strip +++ b/support/rsync-slash-strip @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script can be used as an rsync command-line filter that strips a single # trailing slash from each arg. That treats "src/" the same as "src", thus # you need to use "src/." or "src//" for just the contents of the "src" dir. diff --git a/support/rsyncstats b/support/rsyncstats index ab7246d3..99fd5457 100755 --- a/support/rsyncstats +++ b/support/rsyncstats @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # # This script parses the default logfile format produced by rsync when running # as a daemon with transfer logging enabled. It also parses a slightly tweaked @@ -4,7 +4,7 @@ * * Copyright (C) 1998 Andrew Tridgell * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -42,6 +42,7 @@ extern int inplace; extern int preallocate_files; extern int preserve_perms; extern int preserve_executability; +extern int open_noatime; #ifndef S_BLKSIZE # if defined hpux || defined __hpux__ || defined __hpux @@ -120,11 +121,11 @@ ssize_t do_readlink(const char *path, char *buf, size_t bufsiz) #endif #ifdef HAVE_LINK -int do_link(const char *fname1, const char *fname2) +int do_link(const char *old_path, const char *new_path) { if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - return link(fname1, fname2); + return link(old_path, new_path); } #endif @@ -170,8 +171,8 @@ int do_mknod(const char *pathname, mode_t mode, dev_t dev) saddr.sun_family = AF_UNIX; if ((sock = socket(PF_UNIX, SOCK_STREAM, 0)) < 0 - || (unlink(pathname) < 0 && errno != ENOENT) - || (bind(sock, (struct sockaddr*)&saddr, sizeof saddr)) < 0) + || (unlink(pathname) < 0 && errno != ENOENT) + || (bind(sock, (struct sockaddr*)&saddr, sizeof saddr)) < 0) return -1; close(sock); #ifdef HAVE_CHMOD @@ -202,6 +203,11 @@ int do_open(const char *pathname, int flags, mode_t mode) RETURN_ERROR_IF_RO_OR_LO; } +#ifdef O_NOATIME + if (open_noatime) + flags |= O_NOATIME; +#endif + return open(pathname, flags | O_BINARY, mode); } @@ -235,11 +241,11 @@ int do_chmod(const char *path, mode_t mode) } #endif -int do_rename(const char *fname1, const char *fname2) +int do_rename(const char *old_path, const char *new_path) { if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - return rename(fname1, fname2); + return rename(old_path, new_path); } #ifdef HAVE_FTRUNCATE @@ -361,7 +367,7 @@ OFF_T do_lseek(int fd, OFF_T offset, int whence) } #ifdef HAVE_SETATTRLIST -int do_setattrlist_times(const char *fname, time_t modtime, uint32 mod_nsec) +int do_setattrlist_times(const char *fname, STRUCT_STAT *stp) { struct attrlist attrList; struct timespec ts; @@ -369,8 +375,8 @@ int do_setattrlist_times(const char *fname, time_t modtime, uint32 mod_nsec) if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - ts.tv_sec = modtime; - ts.tv_nsec = mod_nsec; + ts.tv_sec = stp->st_mtime; + ts.tv_nsec = stp->ST_MTIME_NSEC; memset(&attrList, 0, sizeof attrList); attrList.bitmapcount = ATTR_BIT_MAP_COUNT; @@ -380,54 +386,78 @@ int do_setattrlist_times(const char *fname, time_t modtime, uint32 mod_nsec) #endif #ifdef HAVE_UTIMENSAT -int do_utimensat(const char *fname, time_t modtime, uint32 mod_nsec) +int do_utimensat(const char *fname, STRUCT_STAT *stp) { struct timespec t[2]; if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - t[0].tv_sec = 0; - t[0].tv_nsec = UTIME_NOW; - t[1].tv_sec = modtime; - t[1].tv_nsec = mod_nsec; + t[0].tv_sec = stp->st_atime; +#ifdef ST_ATIME_NSEC + t[0].tv_nsec = stp->ST_ATIME_NSEC; +#else + t[0].tv_nsec = 0; +#endif + t[1].tv_sec = stp->st_mtime; +#ifdef ST_MTIME_NSEC + t[1].tv_nsec = stp->ST_MTIME_NSEC; +#else + t[1].tv_nsec = 0; +#endif return utimensat(AT_FDCWD, fname, t, AT_SYMLINK_NOFOLLOW); } #endif #ifdef HAVE_LUTIMES -int do_lutimes(const char *fname, time_t modtime, uint32 mod_nsec) +int do_lutimes(const char *fname, STRUCT_STAT *stp) { struct timeval t[2]; if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - t[0].tv_sec = time(NULL); + t[0].tv_sec = stp->st_atime; +#ifdef ST_ATIME_NSEC + t[0].tv_usec = stp->ST_ATIME_NSEC / 1000; +#else t[0].tv_usec = 0; - t[1].tv_sec = modtime; - t[1].tv_usec = mod_nsec / 1000; +#endif + t[1].tv_sec = stp->st_mtime; +#ifdef ST_MTIME_NSEC + t[1].tv_usec = stp->ST_MTIME_NSEC / 1000; +#else + t[1].tv_usec = 0; +#endif return lutimes(fname, t); } #endif #ifdef HAVE_UTIMES -int do_utimes(const char *fname, time_t modtime, uint32 mod_nsec) +int do_utimes(const char *fname, STRUCT_STAT *stp) { struct timeval t[2]; if (dry_run) return 0; RETURN_ERROR_IF_RO_OR_LO; - t[0].tv_sec = time(NULL); + t[0].tv_sec = stp->st_atime; +#ifdef ST_ATIME_NSEC + t[0].tv_usec = stp->ST_ATIME_NSEC / 1000; +#else t[0].tv_usec = 0; - t[1].tv_sec = modtime; - t[1].tv_usec = mod_nsec / 1000; +#endif + t[1].tv_sec = stp->st_mtime; +#ifdef ST_MTIME_NSEC + t[1].tv_usec = stp->ST_MTIME_NSEC / 1000; +#else + t[1].tv_usec = 0; +#endif return utimes(fname, t); } #elif defined HAVE_UTIME -int do_utime(const char *fname, time_t modtime, UNUSED(uint32 mod_nsec)) +int do_utime(const char *fname, STRUCT_STAT *stp) { #ifdef HAVE_STRUCT_UTIMBUF struct utimbuf tbuf; @@ -439,12 +469,12 @@ int do_utime(const char *fname, time_t modtime, UNUSED(uint32 mod_nsec)) RETURN_ERROR_IF_RO_OR_LO; # ifdef HAVE_STRUCT_UTIMBUF - tbuf.actime = time(NULL); - tbuf.modtime = modtime; + tbuf.actime = stp->st_atime; + tbuf.modtime = stp->st_mtime; return utime(fname, &tbuf); # else - t[0] = time(NULL); - t[1] = modtime; + t[0] = stp->st_atime; + t[1] = stp->st_mtime; return utime(fname, t); # endif } @@ -462,7 +492,7 @@ int do_utime(const char *fname, time_t modtime, UNUSED(uint32 mod_nsec)) OFF_T do_fallocate(int fd, OFF_T offset, OFF_T length) { - int opts = inplace || preallocate_files ? 0 : DO_FALLOC_OPTIONS; + int opts = inplace || preallocate_files ? DO_FALLOC_OPTIONS : 0; int ret; RETURN_ERROR_IF(dry_run, 0); RETURN_ERROR_IF_RO_OR_LO; @@ -493,7 +523,7 @@ OFF_T do_fallocate(int fd, OFF_T offset, OFF_T length) /* Punch a hole at pos for len bytes. The current file position must be at pos and will be * changed to be at pos + len. */ -int do_punch_hole(int fd, UNUSED(OFF_T pos), int len) +int do_punch_hole(int fd, UNUSED(OFF_T pos), OFF_T len) { #ifdef HAVE_FALLOCATE # ifdef HAVE_FALLOC_FL_PUNCH_HOLE @@ -3,7 +3,7 @@ * functions, so that module test harnesses can run standalone. * * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -28,9 +28,11 @@ int protect_args = 0; int module_id = -1; int relative_paths = 0; int module_dirlen = 0; -int preserve_acls = 0; int preserve_times = 0; int preserve_xattrs = 0; +int preserve_perms = 0; +int preserve_executability = 0; +int open_noatime = 0; char *partial_dir; char *module_dir; filter_rule_list daemon_filter_list; @@ -102,3 +104,8 @@ filter_rule_list daemon_filter_list; { return cst || !flg ? 16 : 1; } + + int canonical_checksum(int cst) +{ + return cst ? 0 : 0; +} @@ -2,7 +2,7 @@ * Test harness for unsafe_symlink(). Not linked into rsync itself. * * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -28,9 +28,6 @@ int am_root = 0; int am_sender = 1; int read_only = 0; int list_only = 0; -int human_readable = 0; -int preserve_perms = 0; -int preserve_executability = 0; short info_levels[COUNT_INFO], debug_levels[COUNT_DEBUG]; int @@ -41,8 +38,7 @@ main(int argc, char **argv) return 1; } - printf("%s\n", - unsafe_symlink(argv[1], argv[2]) ? "unsafe" : "safe"); + printf("%s\n", unsafe_symlink(argv[1], argv[2]) ? "unsafe" : "safe"); return 0; } diff --git a/testhelp/maketree.py b/testhelp/maketree.py index d71779ac..19ae71d1 100644 --- a/testhelp/maketree.py +++ b/testhelp/maketree.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2.2 +#!/usr/bin/env python2 # Copyright (C) 2002 by Martin Pool <mbp@samba.org> diff --git a/testsuite/acls.test b/testsuite/acls.test index 40490a67..5728cd57 100644 --- a/testsuite/acls.test +++ b/testsuite/acls.test @@ -7,7 +7,7 @@ . $suitedir/rsync.fns -$RSYNC --version | grep ", ACLs" >/dev/null || test_skipped "Rsync is configured without ACL support" +$RSYNC --version | grep "[, ] ACLs" >/dev/null || test_skipped "Rsync is configured without ACL support" makepath "$fromdir/foo" echo something >"$fromdir/file1" diff --git a/testsuite/atimes.test b/testsuite/atimes.test new file mode 100644 index 00000000..bd3f2927 --- /dev/null +++ b/testsuite/atimes.test @@ -0,0 +1,17 @@ +#! /bin/sh + +# Test rsync copying atimes + +. "$suitedir/rsync.fns" + +mkdir "$fromdir" + +touch "$fromdir/foo" +touch -a -t 200102031717.42 "$fromdir/foo" + +TLS_ARGS=--atimes + +checkit "$RSYNC -rtUgvvv \"$fromdir/\" \"$todir/\"" "$fromdir" "$todir" + +# The script would have aborted on error, so getting here means we've won. +exit 0 diff --git a/testsuite/backup.test b/testsuite/backup.test index e28dd541..87db33e5 100644 --- a/testsuite/backup.test +++ b/testsuite/backup.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2004 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2004-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/chmod-temp-dir.test b/testsuite/chmod-temp-dir.test index bce544c3..d31a1bdf 100644 --- a/testsuite/chmod-temp-dir.test +++ b/testsuite/chmod-temp-dir.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2004 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2004-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). @@ -13,21 +13,16 @@ hands_setup -tmpdir2=$RSYNC_TEST_TMP -if [ x"$tmpdir2" = x ]; then - tmpdir2=/tmp -fi sdev=`$TOOLDIR/getfsdev $scratchdir` -tdev=`$TOOLDIR/getfsdev $tmpdir2` -if [ x$sdev = x$tdev ]; then - tmpdir2=/var/tmp - if [ -d $tmpdir2 ]; then - tdev=`$TOOLDIR/getfsdev $tmpdir2` - else - tdev="$sdev" - fi - [ x$sdev = x$tdev ] && test_skipped "Can't find a tmp dir on a different file system" -fi +tdev=$sdev + +for tmpdir2 in "${RSYNC_TEST_TMP:-/override-tmp-not-specified}" /run/shm /var/tmp /tmp; do + [ -d "$tmpdir2" -a -w "$tmpdir2" ] || continue + tdev=`$TOOLDIR/getfsdev "$tmpdir2"` + [ x$sdev != x$tdev ] && break +done + +[ x$sdev = x$tdev ] && test_skipped "Can't find a tmp dir on a different file system" chmod 440 "$fromdir/text" chmod 500 "$fromdir/dir/text" diff --git a/testsuite/chmod.test b/testsuite/chmod.test index ad4aeff0..f2dd4399 100644 --- a/testsuite/chmod.test +++ b/testsuite/chmod.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2004 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2004-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/chown.test b/testsuite/chown.test index 30a3a94d..aa4958d8 100644 --- a/testsuite/chown.test +++ b/testsuite/chown.test @@ -15,7 +15,7 @@ case $0 in *fake*) - $RSYNC --version | grep ", xattrs" >/dev/null || test_skipped "Rsync needs xattrs for fake device tests" + $RSYNC --version | grep "[, ] xattrs" >/dev/null || test_skipped "Rsync needs xattrs for fake device tests" RSYNC="$RSYNC --fake-super" TLS_ARGS="$TLS_ARGS --fake-super" case "$HOST_OS" in diff --git a/testsuite/compare-dest.test b/testsuite/compare-dest.test index d9eea8e9..7f193c58 100644 --- a/testsuite/compare-dest.test +++ b/testsuite/compare-dest.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2004 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2004-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/daemon.test b/testsuite/daemon.test index 06728e26..a8acd04e 100644 --- a/testsuite/daemon.test +++ b/testsuite/daemon.test @@ -27,7 +27,7 @@ outfile="$scratchdir/rsync.out" SSH="src/support/lsh.sh --no-cd" FILE_REPL='s/^\([^d][^ ]*\) *\(..........[0-9]\) /\1 \2 /' DIR_REPL='s/^\(d[^ ]*\) *[0-9][.,0-9]* /\1 DIR /' -LS_REPL='s;[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9];####/##/## ##:##:##;' +LS_REPL='s;[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] ;####/##/## ##:##:## ;g' build_rsyncd_conf @@ -91,3 +91,12 @@ drwxr-xr-x DIR ####/##/## ##:##:## foo EOT diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed" +$RSYNC -rU localhost::test-from/f* \ + | sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \ + | tee "$outfile" +cat <<EOT >"$chkfile" +drwxr-xr-x DIR ####/##/## ##:##:## foo +-rw-r--r-- 4 ####/##/## ##:##:## ####/##/## ##:##:## foo/one +EOT +diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed" + diff --git a/testsuite/default-acls.test b/testsuite/default-acls.test index eb41a6ff..17893f93 100644 --- a/testsuite/default-acls.test +++ b/testsuite/default-acls.test @@ -7,7 +7,7 @@ . $suitedir/rsync.fns -$RSYNC --version | grep ", ACLs" >/dev/null || test_skipped "Rsync is configured without ACL support" +$RSYNC --version | grep "[, ] ACLs" >/dev/null || test_skipped "Rsync is configured without ACL support" case "$setfacl_nodef" in true) test_skipped "I don't know how to use your setfacl command" ;; diff --git a/testsuite/delete.test b/testsuite/delete.test index 8fa6035a..8c19827b 100644 --- a/testsuite/delete.test +++ b/testsuite/delete.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2005 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2005-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/devices.test b/testsuite/devices.test index a9245138..0af0428b 100644 --- a/testsuite/devices.test +++ b/testsuite/devices.test @@ -16,7 +16,7 @@ outfile="$scratchdir/rsync.out" case $0 in *fake*) - $RSYNC --version | grep ", xattrs" >/dev/null || test_skipped "Rsync needs xattrs for fake device tests" + $RSYNC --version | grep "[, ] xattrs" >/dev/null || test_skipped "Rsync needs xattrs for fake device tests" RSYNC="$RSYNC --fake-super" TLS_ARGS="$TLS_ARGS --fake-super" case "$HOST_OS" in diff --git a/testsuite/exclude.test b/testsuite/exclude.test index 099344f2..fad4fd72 100644 --- a/testsuite/exclude.test +++ b/testsuite/exclude.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2003, 2004, 2005 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2003-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). @@ -12,6 +12,9 @@ . "$suitedir/rsync.fns" +chkfile="$scratchdir/rsync.chk" +outfile="$scratchdir/rsync.out" + CVSIGNORE='*.junk' export CVSIGNORE @@ -113,6 +116,11 @@ rm -rf "$todir" # Add a directory symlink. ln -s too "$fromdir/bar/down/to/foo/sym" +# Start to prep an --update test dir +mkdir "$scratchdir/up1" "$scratchdir/up2" +touch "$scratchdir/up1/older" "$scratchdir/up2/newer" +touch "$scratchdir/up1/extra-src" "$scratchdir/up2/extra-dest" + # Create chkdir with what we expect to be excluded. checkit "$RSYNC -avv '$fromdir/' '$chkdir/'" "$fromdir" "$chkdir" sleep 1 # Ensures that the rm commands will tweak the directory times. @@ -124,6 +132,9 @@ rm "$chkdir"/foo/file[235-9] rm "$chkdir"/bar/down/to/foo/to "$chkdir"/bar/down/to/foo/file[235-9] rm "$chkdir"/mid/for/foo/extra +# Finish prep for the --update test (run last) +touch "$scratchdir/up1/newer" "$scratchdir/up2/older" + # Un-tweak the directory times in our first (weak) exclude test (though # it's a good test of the --existing option). $RSYNC -av --existing --include='*/' --exclude='*' "$fromdir/" "$chkdir/" @@ -215,5 +226,14 @@ $RSYNC -av $relative_opts --existing --filter='-! */' "$fromdir/foo" "$chkdir/" checkit "$RSYNC -avv $relative_opts --exclude='$fromdir/foo/down' \ '$fromdir/foo' '$todir'" "$chkdir$fromdir/foo" "$todir$fromdir/foo" +# Now we'll test the --update option. +$RSYNC -aiO --update touch "$scratchdir/up1/" "$scratchdir/up2/" \ + | tee "$outfile" +cat <<EOT >"$chkfile" +>f$all_plus extra-src +>f..t.$dots newer +EOT +diff $diffopt "$chkfile" "$outfile" || test_fail "--update test failed" + # The script would have aborted on error, so getting here means we've won. exit 0 diff --git a/testsuite/files-from.test b/testsuite/files-from.test index f7181613..207eab5f 100644 --- a/testsuite/files-from.test +++ b/testsuite/files-from.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2008 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2008-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/fuzzy.test b/testsuite/fuzzy.test index 1abfab52..39726a03 100644 --- a/testsuite/fuzzy.test +++ b/testsuite/fuzzy.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2005 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2005-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/itemize.test b/testsuite/itemize.test index 78637ca4..553d0cef 100644 --- a/testsuite/itemize.test +++ b/testsuite/itemize.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2005 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2005-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). @@ -40,7 +40,7 @@ case "$RSYNC" in T=.T ;; *) - if $RSYNC --version | grep ", symtimes" >/dev/null; then + if $RSYNC --version | grep "[, ] symtimes" >/dev/null; then T=.t else T=.T diff --git a/testsuite/merge.test b/testsuite/merge.test index 4e76102d..9b20f73d 100644 --- a/testsuite/merge.test +++ b/testsuite/merge.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2004 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2004-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/missing.test b/testsuite/missing.test index 008c0f4f..c6c8e576 100644 --- a/testsuite/missing.test +++ b/testsuite/missing.test @@ -17,9 +17,16 @@ if grep 'not creating new.*subdir/file' "$scratchdir/out" >/dev/null; then test_fail 'test 1 failed' fi -# Test 2: Attempt to make a fuzzy dirlist for a dir not created on a dry run -$RSYNC -n -r -R --no-implied-dirs -y "$fromdir/./subdir/file" "$todir/" \ - || test_fail 'test 2 failed' +case "$RSYNC" in +*protocol=29*) # FIXME can we get past the new flist sanity check in protocol 29? + echo "Skipped test 2 for protocol 29." + ;; +*) + # Test 2: Attempt to make a fuzzy dirlist for a dir not created on a dry run + $RSYNC -n -r -R --no-implied-dirs -y "$fromdir/./subdir/file" "$todir/" \ + || test_fail 'test 2 failed' + ;; +esac # Test 3: --delete-after pass skipped when last dir is dry-missing $RSYNC -n -r --delete-after -i "$fromdir/" "$todir/" | tee "$scratchdir/out" diff --git a/testsuite/relative.test b/testsuite/relative.test index 686399e4..89e4ce17 100644 --- a/testsuite/relative.test +++ b/testsuite/relative.test @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (C) 2005 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2005-2020 Wayne Davison # # This program is distributable under the terms of the GNU GPL (see COPYING) diff --git a/testsuite/rsync.fns b/testsuite/rsync.fns index ca05297f..65d071ca 100644 --- a/testsuite/rsync.fns +++ b/testsuite/rsync.fns @@ -183,7 +183,7 @@ makepath() { for p in "${@}"; do (echo " makepath $p" - # Absolut Unix. + # Absolute Unix path. if echo $p | grep '^/' >/dev/null then cd / @@ -219,6 +219,14 @@ checkit() { # We can just write everything to stdout/stderr, because the # wrapper hides it unless there is a problem. + case "x$TLS_ARGS" in + *--atimes*) + ( cd "$2" && rsync_ls_lR . ) > "$tmpdir/ls-from" + ;; + *) + ;; + esac + echo "Running: \"$1\"" eval "$1" status=$? @@ -226,10 +234,17 @@ checkit() { failed="$failed status=$status" fi + case "x$TLS_ARGS" in + *--atimes*) + ;; + *) + ( cd "$2" && rsync_ls_lR . ) > "$tmpdir/ls-from" + ;; + esac + echo "-------------" echo "check how the directory listings compare with diff:" echo "" - ( cd "$2" && rsync_ls_lR . ) > "$tmpdir/ls-from" ( cd "$3" && rsync_ls_lR . ) > "$tmpdir/ls-to" diff $diffopt "$tmpdir/ls-from" "$tmpdir/ls-to" || failed="$failed dir-diff" diff --git a/testsuite/ssh-basic.test b/testsuite/ssh-basic.test index c91505b8..4bcbdfe9 100644 --- a/testsuite/ssh-basic.test +++ b/testsuite/ssh-basic.test @@ -19,7 +19,7 @@ if test x"$rsync_enable_ssh_tests" = xyes; then fi if [ "`$SSH -o'BatchMode yes' localhost echo yes`" != "yes" ]; then - test_skipped "Skipping SSH tests because ssh conection to localhost not authorised" + test_skipped "Skipping SSH tests because ssh connection to localhost not authorised" fi echo "Using remote shell: $SSH" diff --git a/testsuite/trimslash.test b/testsuite/trimslash.test index d98ff87b..ec551e65 100644 --- a/testsuite/trimslash.test +++ b/testsuite/trimslash.test @@ -11,7 +11,7 @@ "$TOOLDIR/trimslash" "/usr/local/bin" "/usr/local/bin/" "/usr/local/bin///" \ "//a//" "////" \ - "/Users/Wierd Macintosh Name/// Ooh, translucent plastic/" \ + "/Users/Weird Macintosh Name/// Ooh, translucent plastic/" \ > "$scratchdir/slash.out" diff $diffopt "$scratchdir/slash.out" - <<EOF /usr/local/bin @@ -19,7 +19,7 @@ diff $diffopt "$scratchdir/slash.out" - <<EOF /usr/local/bin //a / -/Users/Wierd Macintosh Name/// Ooh, translucent plastic +/Users/Weird Macintosh Name/// Ooh, translucent plastic EOF # The script would have aborted on error, so getting here means we've won. diff --git a/testsuite/unsafe-byname.test b/testsuite/unsafe-byname.test index 69b338f6..55df0865 100644 --- a/testsuite/unsafe-byname.test +++ b/testsuite/unsafe-byname.test @@ -47,7 +47,7 @@ test_unsafe dir/..//.. from unsafe test_unsafe '' from unsafe -# Based on tests from unsafe-links by Vladimír Michl +# Based on tests from unsafe-links by VladimÃr Michl test_unsafe ../../unsafe/unsafefile from/safe unsafe test_unsafe ..//../unsafe/unsafefile from/safe unsafe test_unsafe ../files/file1 from/safe safe diff --git a/testsuite/unsafe-links.test b/testsuite/unsafe-links.test index 058269ef..479d606c 100644 --- a/testsuite/unsafe-links.test +++ b/testsuite/unsafe-links.test @@ -1,6 +1,6 @@ #! /bin/sh -# Originally by Vladimír Michl <Vladimir.Michl@hlubocky.del.cz> +# Originally by VladimÃr Michl <Vladimir.Michl@hlubocky.del.cz> . "$suitedir/rsync.fns" diff --git a/testsuite/wildmatch.test b/testsuite/wildmatch.test index 9d501897..12b872c8 100644 --- a/testsuite/wildmatch.test +++ b/testsuite/wildmatch.test @@ -1,6 +1,6 @@ #! /bin/sh -# Copyright (C) 2003 by Wayne Davison <wayned@samba.org> +# Copyright (C) 2003-2020 Wayne Davison # This program is distributable under the terms of the GNU GPL (see # COPYING). diff --git a/testsuite/xattrs.test b/testsuite/xattrs.test index f7d9a6df..5931ffcd 100644 --- a/testsuite/xattrs.test +++ b/testsuite/xattrs.test @@ -8,7 +8,7 @@ . $suitedir/rsync.fns lnkdir="$tmpdir/lnk" -$RSYNC --version | grep ", xattrs" >/dev/null || test_skipped "Rsync is configured without xattr support" +$RSYNC --version | grep "[, ] xattrs" >/dev/null || test_skipped "Rsync is configured without xattr support" case "$HOST_OS" in darwin*) @@ -163,7 +163,7 @@ checkit "$RSYNC -aiX $XFILT $dashH --fake-super --link-dest=../chk . ../to" "$ch cd "$todir" xls $dirs $files | diff $diffopt "$scratchdir/xattrs.txt" - -sed -n -e '/^[^ ][^ ]* *[^ ][^ ]* *[^ ][^ ]* *1 /p' "$scratchdir/ls-to" >"$scratchdir/ls-diff-all" +sed -n -e '/^[^d ][^ ]* *[^ ][^ ]* *[^ ][^ ]* *1 /p' "$scratchdir/ls-to" >"$scratchdir/ls-diff-all" fgrep -v './file1' "$scratchdir/ls-diff-all" >"$scratchdir/ls-diff" || : if [ -s "$scratchdir/ls-diff" ]; then echo "Missing hard links on:" @@ -2,7 +2,7 @@ * Trivial ls for comparing two directories after running an rsync. * * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -49,10 +49,6 @@ int list_only = 0; int link_times = 0; int link_owner = 0; int nsec_times = 0; -int preserve_perms = 0; -int preserve_executability = 0; -int preallocate_files = 0; -int inplace = 0; #ifdef SUPPORT_XATTRS @@ -111,6 +107,8 @@ static int stat_xattr(const char *fname, STRUCT_STAT *fst) #endif +static int display_atimes = 0; + static void failed(char const *what, char const *where) { fprintf(stderr, PROGRAM ": %s %s: %s\n", @@ -118,13 +116,38 @@ static void failed(char const *what, char const *where) exit(1); } +static void storetime(char *dest, size_t destsize, time_t t, int nsecs) +{ + if (t) { + int len; + struct tm *mt = gmtime(&t); + + len = snprintf(dest, destsize, + " %04d-%02d-%02d %02d:%02d:%02d", + (int)mt->tm_year + 1900, + (int)mt->tm_mon + 1, + (int)mt->tm_mday, + (int)mt->tm_hour, + (int)mt->tm_min, + (int)mt->tm_sec); + if (nsecs >= 0 && len >= 0) + snprintf(dest + len, destsize - len, ".%09d", nsecs); + } else { + int has_nsecs = nsecs >= 0 ? 1 : 0; + int len = MIN(20 + 10*has_nsecs, (int)destsize - 1); + memset(dest, ' ', len); + dest[len] = '\0'; + } +} + static void list_file(const char *fname) { STRUCT_STAT buf; char permbuf[PERMSTRING_SIZE]; - struct tm *mt; - char datebuf[50]; + char mtimebuf[50]; + char atimebuf[50]; char linkbuf[4096]; + int nsecs; if (do_lstat(fname, &buf) < 0) failed("stat", fname); @@ -150,58 +173,45 @@ static void list_file(const char *fname) buf.st_uid = buf.st_gid = 0; strlcpy(linkbuf, " -> ", sizeof linkbuf); /* const-cast required for silly UNICOS headers */ - len = do_readlink((char *) fname, linkbuf+4, sizeof(linkbuf) - 4); + len = do_readlink((char*)fname, linkbuf+4, sizeof linkbuf - 4); if (len == -1) failed("do_readlink", fname); else /* it's not nul-terminated */ linkbuf[4+len] = 0; } else { - linkbuf[0] = 0; + linkbuf[0] = '\0'; } permstring(permbuf, buf.st_mode); - - if (buf.st_mtime) { - int len; - mt = gmtime(&buf.st_mtime); - - len = snprintf(datebuf, sizeof datebuf, - "%04d-%02d-%02d %02d:%02d:%02d", - (int)mt->tm_year + 1900, - (int)mt->tm_mon + 1, - (int)mt->tm_mday, - (int)mt->tm_hour, - (int)mt->tm_min, - (int)mt->tm_sec); #ifdef ST_MTIME_NSEC - if (nsec_times) { - snprintf(datebuf + len, sizeof datebuf - len, - ".%09d", (int)buf.ST_MTIME_NSEC); - } + if (nsec_times) + nsecs = (int)buf.ST_MTIME_NSEC; + else #endif - } else { - int len = MIN(19 + 9*nsec_times, (int)sizeof datebuf - 1); - memset(datebuf, ' ', len); - datebuf[len] = '\0'; - } + nsecs = -1; + storetime(mtimebuf, sizeof mtimebuf, buf.st_mtime, nsecs); + if (display_atimes) + storetime(atimebuf, sizeof atimebuf, S_ISDIR(buf.st_mode) ? 0 : buf.st_atime, -1); + else + atimebuf[0] = '\0'; /* TODO: Perhaps escape special characters in fname? */ - printf("%s ", permbuf); + if (S_ISCHR(buf.st_mode) || S_ISBLK(buf.st_mode)) { - printf("%5ld,%6ld", - (long)major(buf.st_rdev), - (long)minor(buf.st_rdev)); + printf("%5ld,%6ld", (long)major(buf.st_rdev), (long)minor(buf.st_rdev)); } else printf("%15s", do_big_num(buf.st_size, 1, NULL)); - printf(" %6ld.%-6ld %6ld %s %s%s\n", + + printf(" %6ld.%-6ld %6ld%s%s %s%s\n", (long)buf.st_uid, (long)buf.st_gid, (long)buf.st_nlink, - datebuf, fname, linkbuf); + mtimebuf, atimebuf, fname, linkbuf); } static struct poptOption long_options[] = { /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */ + {"atimes", 'U', POPT_ARG_NONE, &display_atimes, 0, 0, 0}, {"link-times", 'l', POPT_ARG_NONE, &link_times, 0, 0, 0 }, {"link-owner", 'L', POPT_ARG_NONE, &link_owner, 0, 0, 0 }, #ifdef SUPPORT_XATTRS @@ -214,12 +224,13 @@ static struct poptOption long_options[] = { {0,0,0,0,0,0,0} }; -static void tls_usage(int ret) +static void NORETURN tls_usage(int ret) { FILE *F = ret ? stderr : stdout; fprintf(F,"usage: " PROGRAM " [OPTIONS] FILE ...\n"); fprintf(F,"Trivial file listing program for portably checking rsync\n"); fprintf(F,"\nOptions:\n"); + fprintf(F," -U, --atimes display access (last-used) times\n"); fprintf(F," -l, --link-times display the time on a symlink\n"); fprintf(F," -L, --link-owner display the owner+group on a symlink\n"); #ifdef SUPPORT_XATTRS @@ -236,15 +247,13 @@ main(int argc, char *argv[]) const char **extra_args; int opt; - pc = poptGetContext(PROGRAM, argc, (const char **)argv, - long_options, 0); + pc = poptGetContext(PROGRAM, argc, (const char **)argv, long_options, 0); while ((opt = poptGetNextOpt(pc)) != -1) { switch (opt) { case 'h': tls_usage(0); default: - fprintf(stderr, - "%s: %s\n", + fprintf(stderr, "%s: %s\n", poptBadOption(pc, POPT_BADOPTION_NOALIAS), poptStrerror(opt)); tls_usage(1); @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,14 +22,26 @@ #include "rsync.h" #include "itypes.h" #include <zlib.h> +#ifdef SUPPORT_ZSTD +#include <zstd.h> +#endif +#ifdef SUPPORT_LZ4 +#include <lz4.h> +#endif extern int do_compression; extern int protocol_version; extern int module_id; -extern int def_compress_level; +extern int do_compression_level; extern char *skip_compress; -static int compression_level, per_file_default_level; +#ifndef Z_INSERT_ONLY +#define Z_INSERT_ONLY Z_SYNC_FLUSH +#endif + +static int compression_level; /* The compression level for the current file. */ +static int skip_compression_level; /* The least possible compressing for handling skip-compress files. */ +static int per_file_default_level; /* The default level that each new file gets prior to checking its suffix. */ struct suffix_tree { struct suffix_tree *sibling; @@ -40,6 +52,58 @@ struct suffix_tree { static char *match_list; static struct suffix_tree *suftree; +void init_compression_level(void) +{ + int min_level, max_level, def_level, off_level; + + switch (do_compression) { + case CPRES_NONE: + return; + case CPRES_ZLIB: + case CPRES_ZLIBX: + min_level = 1; + max_level = Z_BEST_COMPRESSION; + def_level = 6; /* Z_DEFAULT_COMPRESSION is -1, so set it to the real default */ + off_level = skip_compression_level = Z_NO_COMPRESSION; + if (do_compression_level == Z_DEFAULT_COMPRESSION) + do_compression_level = def_level; + break; +#ifdef SUPPORT_ZSTD + case CPRES_ZSTD: + min_level = skip_compression_level = ZSTD_minCLevel(); + max_level = ZSTD_maxCLevel(); + def_level = ZSTD_CLEVEL_DEFAULT; + off_level = CLVL_NOT_SPECIFIED; + if (do_compression_level == 0) + do_compression_level = def_level; + break; +#endif +#ifdef SUPPORT_LZ4 + case CPRES_LZ4: + min_level = skip_compression_level = 0; + max_level = 0; + def_level = 0; + off_level = CLVL_NOT_SPECIFIED; + break; +#endif + default: /* paranoia to prevent missing case values */ + assert(0); + } + + if (do_compression_level == CLVL_NOT_SPECIFIED) + do_compression_level = def_level; + else if (do_compression_level == off_level) { + do_compression = CPRES_NONE; + return; + } + + /* We don't bother with any errors or warnings -- just make sure that the values are valid. */ + if (do_compression_level < min_level) + do_compression_level = min_level; + else if (do_compression_level > max_level) + do_compression_level = max_level; +} + static void add_suffix(struct suffix_tree **prior, char ltr, const char *str) { struct suffix_tree *node, *newnode; @@ -125,7 +189,7 @@ static void init_set_compression(void) if (!(match_list = t = new_array(char, strlen(f) + 2))) out_of_memory("set_compression"); - per_file_default_level = def_compress_level; + per_file_default_level = do_compression_level; while (*f) { if (*f == ' ') { @@ -146,7 +210,7 @@ static void init_set_compression(void) /* Optimize a match-string of "*". */ *match_list = '\0'; suftree = NULL; - per_file_default_level = 0; + per_file_default_level = skip_compression_level; break; } @@ -183,7 +247,7 @@ void set_compression(const char *fname) for (s = match_list; *s; s += strlen(s) + 1) { if (iwildmatch(s, fname)) { - compression_level = 0; + compression_level = skip_compression_level; return; } } @@ -203,7 +267,7 @@ void set_compression(const char *fname) } if ((ltr = *++s) == '\0') { if (node->word_end) - compression_level = 0; + compression_level = skip_compression_level; return; } if (!(node = node->child)) @@ -239,8 +303,7 @@ static int32 simple_recv_token(int f, char **data) } /* non-compressing send token */ -static void simple_send_token(int f, int32 token, struct map_struct *buf, - OFF_T offset, int32 n) +static void simple_send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 n) { if (n > 0) { int32 len = 0; @@ -293,11 +356,10 @@ static char *obuf; /* Send a deflated token */ static void -send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, - int32 nb, int32 toklen) +send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 nb, int32 toklen) { - int32 n, r; static int init_done, flush_pending; + int32 n, r; if (last_token == -1) { /* initialization */ @@ -366,8 +428,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, * buffer, in case they are the * last 4. Move them to the front. */ - memcpy(tx_strm.next_out, - obuf+MAX_DATA_COUNT-2, 4); + memcpy(tx_strm.next_out, obuf+MAX_DATA_COUNT-2, 4); tx_strm.next_out += 4; tx_strm.avail_out -= 4; } @@ -402,10 +463,9 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, if (token == -1) { /* end of file - clean up */ write_byte(f, END_FLAG); - } else if (token != -2 && do_compression == 1) { + } else if (token != -2 && do_compression == CPRES_ZLIB) { /* Add the data in the current block to the compressor's * history and hash table. */ -#ifndef EXTERNAL_ZLIB do { /* Break up long sections in the same way that * see_deflate_token() does. */ @@ -424,11 +484,6 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, exit_cleanup(RERR_STREAMIO); } } while (toklen > 0); -#else - toklen++; - rprintf(FERROR, "Impossible error in external-zlib code (1).\n"); - exit_cleanup(RERR_STREAMIO); -#endif } } @@ -464,7 +519,7 @@ static int32 recv_deflated_token(int f, char **data) exit_cleanup(RERR_PROTOCOL); } if (!(cbuf = new_array(char, MAX_DATA_COUNT)) - || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE)))) + || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE)))) out_of_memory("recv_deflated_token"); init_done = 1; } else { @@ -579,7 +634,6 @@ static int32 recv_deflated_token(int f, char **data) */ static void see_deflate_token(char *buf, int32 len) { -#ifndef EXTERNAL_ZLIB int r; int32 blklen; unsigned char hdr[5]; @@ -617,13 +671,431 @@ static void see_deflate_token(char *buf, int32 len) exit_cleanup(RERR_STREAMIO); } } while (len || rx_strm.avail_out == 0); -#else - buf++; len++; - rprintf(FERROR, "Impossible error in external-zlib code (2).\n"); - exit_cleanup(RERR_STREAMIO); -#endif } +#ifdef SUPPORT_ZSTD + +static ZSTD_inBuffer zstd_in_buff; +static ZSTD_outBuffer zstd_out_buff; +static ZSTD_CCtx *zstd_cctx; + +static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 nb) +{ + static int comp_init_done, flush_pending; + ZSTD_EndDirective flush = ZSTD_e_continue; + int32 n, r; + + /* initialization */ + if (!comp_init_done) { + + zstd_cctx = ZSTD_createCCtx(); + if (!zstd_cctx) { + rprintf(FERROR, "compression init failed\n"); + exit_cleanup(RERR_PROTOCOL); + } + + obuf = new_array(char, OBUF_SIZE); + if (!obuf) + out_of_memory("send_deflated_token"); + + ZSTD_CCtx_setParameter(zstd_cctx, ZSTD_c_compressionLevel, do_compression_level); + zstd_out_buff.dst = obuf + 2; + + comp_init_done = 1; + } + + if (last_token == -1) { + last_run_end = 0; + run_start = token; + flush_pending = 0; + } else if (last_token == -2) { + run_start = token; + + } else if (nb != 0 || token != last_token + 1 + || token >= run_start + 65536) { + + /* output previous run */ + r = run_start - last_run_end; + n = last_token - run_start; + + if (r >= 0 && r <= 63) { + write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r); + } else { + write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG)); + write_int(f, run_start); + } + if (n != 0) { + write_byte(f, n); + write_byte(f, n >> 8); + } + last_run_end = last_token; + run_start = token; + } + + last_token = token; + + if (nb || flush_pending) { + + zstd_in_buff.src = map_ptr(buf, offset, nb); + zstd_in_buff.size = nb; + zstd_in_buff.pos = 0; + + do { + if (zstd_out_buff.size == 0) { + zstd_out_buff.size = MAX_DATA_COUNT; + zstd_out_buff.pos = 0; + } + + /* File ended, flush */ + if (token != -2) + flush = ZSTD_e_flush; + + r = ZSTD_compressStream2(zstd_cctx, &zstd_out_buff, &zstd_in_buff, flush); + if (ZSTD_isError(r)) { + rprintf(FERROR, "ZSTD_compressStream returned %d\n", r); + exit_cleanup(RERR_STREAMIO); + } + + /* + * Nothing is sent if the buffer isn't full so avoid smaller + * transfers. If a file is finished then we flush the internal + * state and send a smaller buffer so that the remote side can + * finish the file. + */ + if (zstd_out_buff.pos == zstd_out_buff.size || flush == ZSTD_e_flush) { + n = zstd_out_buff.pos; + + obuf[0] = DEFLATED_DATA + (n >> 8); + obuf[1] = n; + write_buf(f, obuf, n+2); + + zstd_out_buff.size = 0; + } + /* + * Loop while the input buffer isn't full consumed or the + * internal state isn't fully flushed. + */ + } while (zstd_in_buff.pos < zstd_in_buff.size || r > 0); + flush_pending = token == -2; + } + + if (token == -1) { + /* end of file - clean up */ + write_byte(f, END_FLAG); + } +} + +static ZSTD_DCtx *zstd_dctx; + +static int32 recv_zstd_token(int f, char **data) +{ + static int decomp_init_done; + static int out_buffer_size; + int32 n, flag; + int r; + + if (!decomp_init_done) { + + zstd_dctx = ZSTD_createDCtx(); + if (!zstd_dctx) { + rprintf(FERROR, "ZSTD_createDStream failed\n"); + exit_cleanup(RERR_PROTOCOL); + } + + /* Output buffer fits two decompressed blocks */ + out_buffer_size = ZSTD_DStreamOutSize() * 2; + cbuf = new_array(char, MAX_DATA_COUNT); + dbuf = new_array(char, out_buffer_size); + if (!cbuf || !dbuf) + out_of_memory("recv_zstd_token"); + + zstd_in_buff.src = cbuf; + zstd_out_buff.dst = dbuf; + + decomp_init_done = 1; + } + + do { + switch (recv_state) { + case r_init: + recv_state = r_idle; + rx_token = 0; + break; + + case r_idle: + flag = read_byte(f); + if ((flag & 0xC0) == DEFLATED_DATA) { + n = ((flag & 0x3f) << 8) + read_byte(f); + read_buf(f, cbuf, n); + + zstd_in_buff.size = n; + zstd_in_buff.pos = 0; + + recv_state = r_inflating; + + } else if (flag == END_FLAG) { + /* that's all folks */ + recv_state = r_init; + return 0; + + } else { + /* here we have a token of some kind */ + if (flag & TOKEN_REL) { + rx_token += flag & 0x3f; + flag >>= 6; + } else + rx_token = read_int(f); + if (flag & 1) { + rx_run = read_byte(f); + rx_run += read_byte(f) << 8; + recv_state = r_running; + } + return -1 - rx_token; + } + break; + + case r_inflating: + zstd_out_buff.size = out_buffer_size; + zstd_out_buff.pos = 0; + + r = ZSTD_decompressStream(zstd_dctx, &zstd_out_buff, &zstd_in_buff); + n = zstd_out_buff.pos; + if (ZSTD_isError(r)) { + rprintf(FERROR, "ZSTD decomp returned %d (%d bytes)\n", r, n); + exit_cleanup(RERR_STREAMIO); + } + + /* + * If the input buffer is fully consumed and the output + * buffer is not full then next step is to read more + * data. + */ + if (zstd_in_buff.size == zstd_in_buff.pos && n < out_buffer_size) + recv_state = r_idle; + + if (n != 0) { + *data = dbuf; + return n; + } + break; + + case r_running: + ++rx_token; + if (--rx_run == 0) + recv_state = r_idle; + return -1 - rx_token; + break; + + case r_inflated: + break; + } + } while (1); +} +#endif /* SUPPORT_ZSTD */ + +#ifdef SUPPORT_LZ4 +static void +send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 nb) +{ + static int init_done, flush_pending; + int size = MAX(LZ4_compressBound(CHUNK_SIZE), MAX_DATA_COUNT+2); + int32 n, r; + + if (last_token == -1) { + if (!init_done) { + if ((obuf = new_array(char, size)) == NULL) + out_of_memory("send_compressed_token"); + init_done = 1; + } + last_run_end = 0; + run_start = token; + flush_pending = 0; + } else if (last_token == -2) { + run_start = token; + } else if (nb != 0 || token != last_token + 1 + || token >= run_start + 65536) { + /* output previous run */ + r = run_start - last_run_end; + n = last_token - run_start; + if (r >= 0 && r <= 63) { + write_byte(f, (n==0? TOKEN_REL: TOKENRUN_REL) + r); + } else { + write_byte(f, (n==0? TOKEN_LONG: TOKENRUN_LONG)); + write_int(f, run_start); + } + if (n != 0) { + write_byte(f, n); + write_byte(f, n >> 8); + } + last_run_end = last_token; + run_start = token; + } + + last_token = token; + + if (nb != 0 || flush_pending) { + int available_in, available_out = 0; + const char *next_in; + + do { + char *ptr = obuf; + char *next_out = obuf + 2; + + if (available_out == 0) { + available_in = MIN(nb, MAX_DATA_COUNT); + next_in = map_ptr(buf, offset, available_in); + } else + available_in /= 2; + + available_out = LZ4_compress_default(next_in, next_out, available_in, size - 2); + if (!available_out) { + rprintf(FERROR, "compress returned %d\n", available_out); + exit_cleanup(RERR_STREAMIO); + } + if (available_out <= MAX_DATA_COUNT) { + ptr[0] = DEFLATED_DATA + (available_out >> 8); + ptr[1] = available_out; + + write_buf(f, ptr, available_out + 2); + + available_out = 0; + nb -= available_in; + offset += available_in; + } + } while (nb != 0); + flush_pending = token == -2; + } + if (token == -1) + /* end of file - clean up */ + write_byte(f, END_FLAG); +} + +static int32 recv_compressed_token(int f, char **data) +{ + static int32 saved_flag; + static int init_done; + int32 n, flag; + int size = MAX(LZ4_compressBound(CHUNK_SIZE), MAX_DATA_COUNT+2); + static const char *next_in; + static int avail_in; + int avail_out; + + for (;;) { + switch (recv_state) { + case r_init: + if (!init_done) { + if (!(cbuf = new_array(char, MAX_DATA_COUNT)) + || !(dbuf = new_array(char, size))) + out_of_memory("recv_compressed_token"); + init_done = 1; + } + recv_state = r_idle; + rx_token = 0; + break; + case r_idle: + case r_inflated: + if (saved_flag) { + flag = saved_flag & 0xff; + saved_flag = 0; + } else + flag = read_byte(f); + if ((flag & 0xC0) == DEFLATED_DATA) { + n = ((flag & 0x3f) << 8) + read_byte(f); + read_buf(f, cbuf, n); + next_in = (char *)cbuf; + avail_in = n; + recv_state = r_inflating; + break; + } + + if (recv_state == r_inflated) + recv_state = r_idle; + + if (flag == END_FLAG) { + /* that's all folks */ + recv_state = r_init; + return 0; + } + + /* here we have a token of some kind */ + if (flag & TOKEN_REL) { + rx_token += flag & 0x3f; + flag >>= 6; + } else + rx_token = read_int(f); + if (flag & 1) { + rx_run = read_byte(f); + rx_run += read_byte(f) << 8; + recv_state = r_running; + } + return -1 - rx_token; + + case r_inflating: + avail_out = LZ4_decompress_safe(next_in, dbuf, avail_in, size); + if (avail_out < 0) { + rprintf(FERROR, "uncompress failed: %d\n", avail_out); + exit_cleanup(RERR_STREAMIO); + } + recv_state = r_inflated; + *data = dbuf; + return avail_out; + + case r_running: + ++rx_token; + if (--rx_run == 0) + recv_state = r_idle; + return -1 - rx_token; + } + } + +} + +# if 0 +static void see_uncompressed_token(char *buf, int32 len) +{ + static const char *next_in; + static int avail_in; + int avail_out; + + int32 blklen; + char hdr[5]; + + avail_in = 0; + blklen = 0; + hdr[0] = 0; + do { + if (avail_in == 0 && len != 0) { + if (blklen == 0) { + /* Give it a fake stored-block header. */ + next_in = hdr; + avail_in = 5; + blklen = len; + if (blklen > 0xffff) + blklen = 0xffff; + hdr[1] = blklen; + hdr[2] = blklen >> 8; + hdr[3] = ~hdr[1]; + hdr[4] = ~hdr[2]; + } else { + next_in = (char *)buf; + avail_in = blklen; + if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */ + buf += blklen; + len -= blklen; + blklen = 0; + } + } + avail_out = LZ4_decompress_safe(next_in, dbuf, avail_in, LZ4_compressBound(CHUNK_SIZE)); + if (avail_out < 0) { + rprintf(FERROR, "uncompress failed: %d\n", avail_out); + exit_cleanup(RERR_STREAMIO); + } + + } while (len); +} +# endif /* 0 */ +#endif /* SUPPORT_LZ4 */ + /** * Transmit a verbatim buffer of length @p n followed by a token. * If token == -1 then we have reached EOF @@ -632,28 +1104,54 @@ static void see_deflate_token(char *buf, int32 len) void send_token(int f, int32 token, struct map_struct *buf, OFF_T offset, int32 n, int32 toklen) { - if (!do_compression) + switch (do_compression) { + case CPRES_NONE: simple_send_token(f, token, buf, offset, n); - else + break; + case CPRES_ZLIB: + case CPRES_ZLIBX: send_deflated_token(f, token, buf, offset, n, toklen); + break; +#ifdef SUPPORT_ZSTD + case CPRES_ZSTD: + send_zstd_token(f, token, buf, offset, n); + break; +#endif +#ifdef SUPPORT_LZ4 + case CPRES_LZ4: + send_compressed_token(f, token, buf, offset, n); + break; +#endif + default: + assert(0); + } } /* - * receive a token or buffer from the other end. If the reurn value is >0 then + * receive a token or buffer from the other end. If the return value is >0 then * it is a data buffer of that length, and *data will point at the data. * if the return value is -i then it represents token i-1 * if the return value is 0 then the end has been reached */ int32 recv_token(int f, char **data) { - int tok; - - if (!do_compression) { - tok = simple_recv_token(f,data); - } else { - tok = recv_deflated_token(f, data); + switch (do_compression) { + case CPRES_NONE: + return simple_recv_token(f,data); + case CPRES_ZLIB: + case CPRES_ZLIBX: + return recv_deflated_token(f, data); +#ifdef SUPPORT_ZSTD + case CPRES_ZSTD: + return recv_zstd_token(f, data); +#endif +#ifdef SUPPORT_LZ4 + case CPRES_LZ4: + return recv_compressed_token(f, data); +#endif + default: + assert(0); } - return tok; } /* @@ -661,6 +1159,24 @@ int32 recv_token(int f, char **data) */ void see_token(char *data, int32 toklen) { - if (do_compression == 1) + switch (do_compression) { + case CPRES_NONE: + break; + case CPRES_ZLIB: see_deflate_token(data, toklen); + break; + case CPRES_ZLIBX: + break; +#ifdef SUPPORT_LZ4 + case CPRES_LZ4: + /*see_uncompressed_token(data, toklen);*/ + break; +#endif +#ifdef SUPPORT_LZ4 + case CPRES_ZSTD: + break; +#endif + default: + assert(0); + } } diff --git a/trimslash.c b/trimslash.c index 7c65205a..1ec928ca 100644 --- a/trimslash.c +++ b/trimslash.c @@ -2,7 +2,7 @@ * Simple utility used only by the test harness. * * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,10 +26,6 @@ int am_root = 0; int am_sender = 1; int read_only = 1; int list_only = 0; -int preserve_perms = 0; -int preserve_executability = 0; -int preallocate_files = 0; -int inplace = 0; int main(int argc, char **argv) diff --git a/tweak_manpage b/tweak_manpage deleted file mode 100755 index 4c55c514..00000000 --- a/tweak_manpage +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/perl -i -p - -use strict; -use warnings; - -# We only need to use "\&'" or "\&." at the start of a line. -s/(?<=.)\\\&(['.])/$1/g; - -# Some quotes turn into open/close quotes. -s/'(.)'/\\(oq$1\\(cq/g; -s/(^|[ (])"(?!( |$))/$1\\(lq/gm; -s/(?<! )"([.,:;! )]|$)/\\(rq$1/gm; -s/(\\\(lq[^(]*) "( |$)/$1 \\(rq$2/gm; -s/(^| )" ([^(]*\\\(rq)/$1\\(lq $2/gm; - -# And some don't. -s/^([. ])(.*)/ $1 . realquotes($2) /egm; -s/(\\f(B|\(CW).*?\\fP)/ realquotes($1) /egs; - -s/^\\\&(\\\(oq)/$1/gm; - -sub realquotes -{ - my($txt) = @_; - $txt =~ s/\\\([lr]q/"/g; - $txt =~ s/\\\([oc]q/'/g; - $txt; -} @@ -3,7 +3,7 @@ * * Copyright (C) 1996 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras - * Copyright (C) 2004-2018 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -47,8 +47,8 @@ extern char *groupmap; #define NFLAGS_NAME_MATCH (1<<1) union name_or_id { - const char *name; - id_t max_id; + const char *name; + id_t max_id; }; struct idlist { @@ -176,8 +176,7 @@ static int is_in_group(gid_t gid) char *gidbuf = new_array(char, ngroups*21+32); if (!gidbuf) out_of_memory("is_in_group"); - pos = snprintf(gidbuf, 32, "process has %d gid%s: ", - ngroups, ngroups == 1? "" : "s"); + pos = snprintf(gidbuf, 32, "process has %d gid%s: ", ngroups, ngroups == 1? "" : "s"); for (n = 0; n < ngroups; n++) { pos += snprintf(gidbuf+pos, 21, " %d", (int)gidset[n]); } @@ -252,7 +251,7 @@ static struct idlist *recv_add_id(struct idlist **idlist_ptr, struct idlist *idm return node; } -/* this function is a definate candidate for a faster algorithm */ +/* this function is a definite candidate for a faster algorithm */ uid_t match_uid(uid_t uid) { static struct idlist *last = NULL; @@ -435,8 +434,7 @@ void recv_id_list(int f, struct file_list *flist) } if (preserve_gid && (!am_root || !numeric_ids || groupmap)) { for (i = 0; i < flist->used; i++) { - F_GROUP(flist->files[i]) = match_gid(F_GROUP(flist->files[i]), - &flist->files[i]->flags); + F_GROUP(flist->files[i]) = match_gid(F_GROUP(flist->files[i]), &flist->files[i]->flags); } } } @@ -497,18 +495,14 @@ void parse_name_map(char *map, BOOL usernames) if (user_to_uid(colon+1, &uid, True)) add_to_list(idmap_ptr, id1, noiu, uid, flags); else { - rprintf(FERROR, - "Unknown --usermap name on receiver: %s\n", - colon+1); + rprintf(FERROR, "Unknown --usermap name on receiver: %s\n", colon+1); } } else { gid_t gid; if (group_to_gid(colon+1, &gid, True)) add_to_list(idmap_ptr, id1, noiu, gid, flags); else { - rprintf(FERROR, - "Unknown --groupmap name on receiver: %s\n", - colon+1); + rprintf(FERROR, "Unknown --groupmap name on receiver: %s\n", colon+1); } } @@ -519,8 +513,7 @@ void parse_name_map(char *map, BOOL usernames) } /* The 0 user/group doesn't get its name sent, so add it explicitly. */ - recv_add_id(idlist_ptr, *idmap_ptr, 0, - numeric_ids ? NULL : usernames ? uid_to_user(0) : gid_to_group(0)); + recv_add_id(idlist_ptr, *idmap_ptr, 0, numeric_ids ? NULL : usernames ? uid_to_user(0) : gid_to_group(0)); } #ifdef HAVE_GETGROUPLIST @@ -534,14 +527,14 @@ const char *getallgroups(uid_t uid, item_list *gid_list) return "getpwuid failed"; gid_list->count = 0; /* We're overwriting any items in the list */ - EXPAND_ITEM_LIST(gid_list, gid_t, 32); + (void)EXPAND_ITEM_LIST(gid_list, gid_t, 32); size = gid_list->malloced; /* Get all the process's groups, with the pw_gid group first. */ if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0) { if (size > (int)gid_list->malloced) { gid_list->count = gid_list->malloced; - EXPAND_ITEM_LIST(gid_list, gid_t, size); + (void)EXPAND_ITEM_LIST(gid_list, gid_t, size); if (getgrouplist(pw->pw_name, pw->pw_gid, gid_list->items, &size) < 0) size = -1; } else @@ -560,7 +553,7 @@ const char *getallgroups(uid_t uid, item_list *gid_list) break; } if (j == size) { /* The default group wasn't found! */ - EXPAND_ITEM_LIST(gid_list, gid_t, size+1); + (void)EXPAND_ITEM_LIST(gid_list, gid_t, size+1); gid_array = gid_list->items; } gid_array[j] = gid_array[0]; @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -117,62 +117,59 @@ void print_child_argv(const char *prefix, char **cmd) /* This returns 0 for success, 1 for a symlink if symlink time-setting * is not possible, or -1 for any other error. */ -int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode) +int set_times(const char *fname, STRUCT_STAT *stp) { static int switch_step = 0; if (DEBUG_GTE(TIME, 1)) { - rprintf(FINFO, "set modtime of %s to (%ld) %s", - fname, (long)modtime, - asctime(localtime(&modtime))); + rprintf(FINFO, + "set modtime, atime of %s to (%ld) %s, (%ld) %s\n", + fname, (long)stp->st_mtime, + timestring(stp->st_mtime), (long)stp->st_atime, timestring(stp->st_atime)); } switch (switch_step) { #ifdef HAVE_SETATTRLIST #include "case_N.h" - if (do_setattrlist_times(fname, modtime, mod_nsec) == 0) + if (do_setattrlist_times(fname, stp) == 0) break; if (errno != ENOSYS) return -1; switch_step++; - /* FALLTHROUGH */ #endif #ifdef HAVE_UTIMENSAT #include "case_N.h" - if (do_utimensat(fname, modtime, mod_nsec) == 0) + if (do_utimensat(fname, stp) == 0) break; if (errno != ENOSYS) return -1; switch_step++; - /* FALLTHROUGH */ #endif #ifdef HAVE_LUTIMES #include "case_N.h" - if (do_lutimes(fname, modtime, mod_nsec) == 0) + if (do_lutimes(fname, stp) == 0) break; if (errno != ENOSYS) return -1; switch_step++; - /* FALLTHROUGH */ #endif #include "case_N.h" switch_step++; if (preserve_times & PRESERVE_LINK_TIMES) { preserve_times &= ~PRESERVE_LINK_TIMES; - if (S_ISLNK(mode)) + if (S_ISLNK(stp->st_mode)) return 1; } - /* FALLTHROUGH */ #include "case_N.h" #ifdef HAVE_UTIMES - if (do_utimes(fname, modtime, mod_nsec) == 0) + if (do_utimes(fname, stp) == 0) break; #else - if (do_utime(fname, modtime, mod_nsec) == 0) + if (do_utime(fname, stp) == 0) break; #endif @@ -346,6 +343,7 @@ int copy_file(const char *source, const char *dest, int ofd, mode_t mode) if (robust_unlink(dest) && errno != ENOENT) { int save_errno = errno; rsyserr(FERROR_XFER, errno, "unlink %s", full_fname(dest)); + close(ifd); errno = save_errno; return -1; } @@ -500,6 +498,11 @@ int robust_rename(const char *from, const char *to, const char *partialptr, { int tries = 4; + /* A resumed in-place partial-dir transfer might call us with from and + * to pointing to the same buf if the transfer failed yet again. */ + if (from == to) + return 0; + while (tries--) { if (do_rename(from, to) == 0) return 0; @@ -719,9 +722,7 @@ int glob_expand(const char *arg, char ***argv_p, int *argc_p, int *maxargs_p) s = strdup(arg); if (!s) out_of_memory("glob_expand"); - clean_fname(s, CFN_KEEP_DOT_DIRS - | CFN_KEEP_TRAILING_SLASH - | CFN_COLLAPSE_DOT_DOT_DIRS); + clean_fname(s, CFN_KEEP_DOT_DIRS | CFN_KEEP_TRAILING_SLASH | CFN_COLLAPSE_DOT_DOT_DIRS); } ENSURE_MEMSPACE(glob.arg_buf, char, glob.absize, MAXPATHLEN); @@ -1002,8 +1003,7 @@ int clean_fname(char *name, int flags) * ALWAYS collapses ".." elements (except for those at the start of the * string up to "depth" deep). If the resulting name would be empty, * change it into a ".". */ -char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, - int flags) +char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, int flags) { char *start, *sanp; int rlen = 0, drop_dot_dirs = !relative_paths || !(flags & SP_KEEP_DOT_DIRS); @@ -1113,6 +1113,7 @@ int change_dir(const char *dir, int set_path_only) skipped_chdir = set_path_only; memcpy(curr_dir, dir, len + 1); } else { + unsigned int save_dir_len = curr_dir_len; if (curr_dir_len + 1 + len >= sizeof curr_dir) { errno = ENAMETOOLONG; return 0; @@ -1122,6 +1123,7 @@ int change_dir(const char *dir, int set_path_only) memcpy(curr_dir + curr_dir_len, dir, len + 1); if (!set_path_only && chdir(curr_dir)) { + curr_dir_len = save_dir_len; curr_dir[curr_dir_len] = '\0'; return 0; } @@ -1335,48 +1337,31 @@ int unsafe_symlink(const char *dest, const char *src) /* Return the date and time as a string. Some callers tweak returned buf. */ char *timestring(time_t t) { - static char TimeBuf[200]; + static int ndx = 0; + static char buffers[4][20]; /* We support 4 simultaneous timestring results. */ + char *TimeBuf = buffers[ndx = (ndx + 1) % 4]; struct tm *tm = localtime(&t); - char *p; - -#ifdef HAVE_STRFTIME - strftime(TimeBuf, sizeof TimeBuf - 1, "%Y/%m/%d %H:%M:%S", tm); -#else - strlcpy(TimeBuf, asctime(tm), sizeof TimeBuf); -#endif - - if ((p = strchr(TimeBuf, '\n')) != NULL) - *p = '\0'; + int len = snprintf(TimeBuf, sizeof buffers[0], "%4d/%02d/%02d %02d:%02d:%02d", + (int)tm->tm_year + 1900, (int)tm->tm_mon + 1, (int)tm->tm_mday, + (int)tm->tm_hour, (int)tm->tm_min, (int)tm->tm_sec); + assert(len > 0); /* Silence gcc warning */ return TimeBuf; } /* Determine if two time_t values are equivalent (either exact, or in * the modification timestamp window established by --modify-window). - * - * @retval 0 if the times should be treated as the same - * - * @retval +1 if the first is later - * - * @retval -1 if the 2nd is later - **/ -int cmp_time(time_t f1_sec, unsigned long f1_nsec, time_t f2_sec, unsigned long f2_nsec) + * Returns 1 if the times the "same", or 0 if they are different. */ +int same_time(time_t f1_sec, unsigned long f1_nsec, time_t f2_sec, unsigned long f2_nsec) { - if (f2_sec > f1_sec) { - /* The final comparison makes sure that modify_window doesn't overflow a - * time_t, which would mean that f2_sec must be in the equality window. */ - if (modify_window <= 0 || (f2_sec > f1_sec + modify_window && f1_sec + modify_window > f1_sec)) - return -1; - } else if (f1_sec > f2_sec) { - if (modify_window <= 0 || (f1_sec > f2_sec + modify_window && f2_sec + modify_window > f2_sec)) - return 1; - } else if (modify_window < 0) { - if (f2_nsec > f1_nsec) - return -1; - else if (f1_nsec > f2_nsec) - return 1; - } - return 0; + if (modify_window == 0) + return f1_sec == f2_sec; + if (modify_window < 0) + return f1_sec == f2_sec && f1_nsec == f2_nsec; + /* The nano seconds doesn't figure into these checks -- time windows don't care about that. */ + if (f2_sec > f1_sec) + return f2_sec - f1_sec <= modify_window; + return f1_sec - f2_sec <= modify_window; } #ifdef __INSURE__XX @@ -1450,8 +1435,7 @@ const char *find_filename_suffix(const char *fn, int fn_len, int *len_ptr) } else if (s_len == 5) { if (strcmp(s+1, "orig") == 0) continue; - } else if (s_len > 2 && had_tilde - && s[1] == '~' && isDigit(s + 2)) + } else if (s_len > 2 && had_tilde && s[1] == '~' && isDigit(s + 2)) continue; *len_ptr = s_len; suf = s; @@ -1462,7 +1446,7 @@ const char *find_filename_suffix(const char *fn, int fn_len, int *len_ptr) if (!isDigit(s)) return suf; } - /* An all-digit suffix may not be that signficant. */ + /* An all-digit suffix may not be that significant. */ s = suf; } @@ -1526,8 +1510,8 @@ uint32 fuzzy_distance(const char *s1, unsigned len1, const char *s2, unsigned le #define BB_PER_SLOT_INTS (BB_SLOT_SIZE / 4) /* Number of int32s per slot */ struct bitbag { - uint32 **bits; - int slot_cnt; + uint32 **bits; + int slot_cnt; }; struct bitbag *bitbag_create(int max_ndx) @@ -1654,8 +1638,7 @@ int flist_ndx_pop(flist_ndx_list *lp) * After the size check, the list's count is incremented by 1 and a pointer * to the "new" list item is returned. */ -void *expand_item_list(item_list *lp, size_t item_size, - const char *desc, int incr) +void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int incr) { /* First time through, 0 <= 0, so list is expanded. */ if (lp->malloced <= lp->count) { @@ -1686,3 +1669,11 @@ void *expand_item_list(item_list *lp, size_t item_size, } return (char*)lp->items + (lp->count++ * item_size); } + +/* This zeroing of memory won't be optimized away by the compiler. */ +void force_memzero(void *buf, size_t len) +{ + volatile uchar *z = buf; + while (len-- > 0) + *z++ = '\0'; +} @@ -4,7 +4,7 @@ * Copyright (C) 1996-2000 Andrew Tridgell * Copyright (C) 1996 Paul Mackerras * Copyright (C) 2001, 2002 Martin Pool <mbp@samba.org> - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -28,13 +28,21 @@ /** * Sleep for a specified number of milliseconds. * - * Always returns TRUE. (In the future it might return FALSE if - * interrupted.) + * Always returns True. **/ int msleep(int t) { -#ifdef HAVE_USLEEP +#ifdef HAVE_NANOSLEEP + struct timespec ts; + + ts.tv_sec = t / 1000; + ts.tv_nsec = (t % 1000) * 1000000L; + + while (nanosleep(&ts, &ts) < 0 && errno == EINTR) {} + +#elif defined HAVE_USLEEP usleep(t*1000); + #else int tdiff = 0; struct timeval tval, t1, t2; @@ -81,21 +89,26 @@ const char *sum_as_hex(int csum_type, const char *sum, int flist_csum) { static char buf[MAX_DIGEST_LEN*2+1]; int i, x1, x2; + int canonical = canonical_checksum(csum_type); int sum_len = csum_len_for_type(csum_type, flist_csum); - char *c = buf + sum_len*2; + char *c; - assert(c - buf < (int)sizeof buf); + if (!canonical) + return NULL; - *c = '\0'; + assert(sum_len*2 < (int)sizeof buf); - for (i = sum_len; --i >= 0; ) { - x1 = CVAL(sum, i); - x2 = x1 >> 4; - x1 &= 0xF; - *--c = x1 <= 9 ? x1 + '0' : x1 + 'a' - 10; - *--c = x2 <= 9 ? x2 + '0' : x2 + 'a' - 10; + for (i = sum_len, c = buf; --i >= 0; ) { + int ndx = canonical < 0 ? sum_len - i - 1 : i; + x2 = CVAL(sum, ndx); + x1 = x2 >> 4; + x2 &= 0xF; + *c++ = x1 <= 9 ? x1 + '0' : x1 + 'a' - 10; + *c++ = x2 <= 9 ? x2 + '0' : x2 + 'a' - 10; } + *c = '\0'; + return buf; } @@ -1,7 +1,7 @@ /* * Test suite for the wildmatch code. * - * Copyright (C) 2003-2018 Wayne Davison + * Copyright (C) 2003-2019 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -3,7 +3,7 @@ * Written by Jay Fenlason, vaguely based on the ACLs patch. * * Copyright (C) 2004 Red Hat, Inc. - * Copyright (C) 2006-2018 Wayne Davison + * Copyright (C) 2006-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -44,8 +44,7 @@ extern int saw_xattr_filter; #define MAX_FULL_DATUM 32 -#define HAS_PREFIX(str, prfx) (*(str) == *(prfx) \ - && strncmp(str, prfx, sizeof (prfx) - 1) == 0) +#define HAS_PREFIX(str, prfx) (*(str) == *(prfx) && strncmp(str, prfx, sizeof (prfx) - 1) == 0) #define XATTR_ABBREV(x) ((size_t)((x).name - (x).datum) < (x).datum_len) @@ -185,8 +184,7 @@ static ssize_t get_xattr_names(const char *fname) /* On entry, the *len_ptr parameter contains the size of the extra space we * should allocate when we create a buffer for the data. On exit, it contains * the length of the datum. */ -static char *get_xattr_data(const char *fname, const char *name, size_t *len_ptr, - int no_missing_error) +static char *get_xattr_data(const char *fname, const char *name, size_t *len_ptr, int no_missing_error) { size_t datum_len = sys_lgetxattr(fname, name, NULL, 0); size_t extra_len = *len_ptr; @@ -215,13 +213,13 @@ static char *get_xattr_data(const char *fname, const char *name, size_t *len_ptr if (len != datum_len) { if (len == (size_t)-1) { rsyserr(FERROR_XFER, errno, - "get_xattr_data: lgetxattr(%s,\"%s\",%ld) failed", - full_fname(fname), name, (long)datum_len); + "get_xattr_data: lgetxattr(%s,\"%s\",%ld) failed", + full_fname(fname), name, (long)datum_len); } else { rprintf(FERROR_XFER, - "get_xattr_data: lgetxattr(%s,\"%s\",%ld) returned %ld\n", - full_fname(fname), name, - (long)datum_len, (long)len); + "get_xattr_data: lgetxattr(%s,\"%s\",%ld) returned %ld\n", + full_fname(fname), name, + (long)datum_len, (long)len); } free(ptr); return NULL; @@ -415,7 +413,7 @@ static int find_matching_xattr(const item_list *xalp) key = xattr_lookup_hash(xalp); - node = hashtable_find(rsync_xal_h, key, 0); + node = hashtable_find(rsync_xal_h, key, NULL); if (node == NULL) return -1; @@ -478,21 +476,17 @@ static int rsync_xal_store(item_list *xalp) new_list->key = xattr_lookup_hash(&new_list->xa_items); if (rsync_xal_h == NULL) - rsync_xal_h = hashtable_create(512, 1); + rsync_xal_h = hashtable_create(512, HT_KEY64); if (rsync_xal_h == NULL) out_of_memory("rsync_xal_h hashtable_create()"); - node = hashtable_find(rsync_xal_h, new_list->key, 1); - if (node == NULL) - out_of_memory("rsync_xal_h hashtable_find()"); - new_ref = new0(rsync_xa_list_ref); if (new_ref == NULL) out_of_memory("new0(rsync_xa_list_ref)"); - new_ref->ndx = ndx; - if (node->data != NULL) { + node = hashtable_find(rsync_xal_h, new_list->key, new_ref); + if (node->data != (void*)new_ref) { rsync_xa_list_ref *ref = node->data; while (ref != NULL) { @@ -504,8 +498,7 @@ static int rsync_xal_store(item_list *xalp) ref->next = new_ref; break; } - } else - node->data = new_ref; + } return ndx; } @@ -719,7 +712,7 @@ int recv_xattr_request(struct file_struct *file, int f_in) num += rel_pos; if (am_sender) { /* The sender-related num values are only in order on the sender. - * We use that order here to scan foward or backward as needed. */ + * We use that order here to scan forward or backward as needed. */ if (rel_pos < 0) { while (cnt < (int)lst->count && rxa->num > num) { rxa--; @@ -926,7 +919,7 @@ void uncache_tmp_xattrs(void) if (rsync_xal_h == NULL) continue; - node = hashtable_find(rsync_xal_h, xa_list_item->key, 0); + node = hashtable_find(rsync_xal_h, xa_list_item->key, NULL); if (node == NULL) continue; @@ -1072,8 +1065,7 @@ static int rsync_xal_set(const char *fname, item_list *xalp, } /* Set extended attributes on indicated filename. */ -int set_xattr(const char *fname, const struct file_struct *file, - const char *fnamecmp, stat_x *sxp) +int set_xattr(const char *fname, const struct file_struct *file, const char *fnamecmp, stat_x *sxp) { rsync_xa_list *glst = rsync_xal_l.items; item_list *lst; diff --git a/zlib/crc32.c b/zlib/crc32.c index 979a7190..05733f4e 100644 --- a/zlib/crc32.c +++ b/zlib/crc32.c @@ -278,7 +278,7 @@ local unsigned long crc32_little(crc, buf, len) } /* ========================================================================= */ -#define DOBIG4 c ^= *++buf4; \ +#define DOBIG4 c ^= *buf4++; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 @@ -300,7 +300,6 @@ local unsigned long crc32_big(crc, buf, len) } buf4 = (const z_crc_t FAR *)(const void FAR *)buf; - buf4--; while (len >= 32) { DOBIG32; len -= 32; @@ -309,7 +308,6 @@ local unsigned long crc32_big(crc, buf, len) DOBIG4; len -= 4; } - buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { diff --git a/zlib/inffast.c b/zlib/inffast.c index bda59ceb..f0d163db 100644 --- a/zlib/inffast.c +++ b/zlib/inffast.c @@ -10,25 +10,6 @@ #ifndef ASMINF -/* Allow machine dependent optimization for post-increment or pre-increment. - Based on testing to date, - Pre-increment preferred for: - - PowerPC G3 (Adler) - - MIPS R5000 (Randers-Pehrson) - Post-increment preferred for: - - none - No measurable difference: - - Pentium III (Anderson) - - M68060 (Nikl) - */ -#ifdef POSTINC -# define OFF 0 -# define PUP(a) *(a)++ -#else -# define OFF 1 -# define PUP(a) *++(a) -#endif - /* Decode literal, length, and distance codes and write out the resulting literal and match bytes until either not enough input or output is @@ -96,9 +77,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; - in = strm->next_in - OFF; + in = strm->next_in; last = in + (strm->avail_in - 5); - out = strm->next_out - OFF; + out = strm->next_out; beg = out - (start - strm->avail_out); end = out + (strm->avail_out - 257); #ifdef INFLATE_STRICT @@ -119,9 +100,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = lcode[hold & lmask]; @@ -134,14 +115,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : "inflate: literal 0x%02x\n", here.val)); - PUP(out) = (unsigned char)(here.val); + *out++ = (unsigned char)(here.val); } else if (op & 16) { /* length base */ len = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -150,9 +131,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = dcode[hold & dmask]; @@ -165,10 +146,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } } @@ -196,30 +177,30 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (len <= op - whave) { do { - PUP(out) = 0; + *out++ = 0; } while (--len); continue; } len -= op - whave; do { - PUP(out) = 0; + *out++ = 0; } while (--op > whave); if (op == 0) { from = out - dist; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--len); continue; } #endif } - from = window - OFF; + from = window; if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -230,14 +211,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from end of window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); - from = window - OFF; + from = window; if (wnext < len) { /* some from start of window */ op = wnext; len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -248,35 +229,35 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else { from = out - dist; /* copy direct from output */ do { /* minimum length is three */ - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } while (len > 2); if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } } @@ -313,8 +294,8 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ hold &= (1U << bits) - 1; /* update state and return */ - strm->next_in = in + OFF; - strm->next_out = out + OFF; + strm->next_in = in; + strm->next_out = out; strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); strm->avail_out = (unsigned)(out < end ? 257 + (end - out) : 257 - (out - end)); diff --git a/zlib/inflate.c b/zlib/inflate.c index a7555738..e43abd9e 100644 --- a/zlib/inflate.c +++ b/zlib/inflate.c @@ -885,9 +885,10 @@ int flush; INITBITS(); state->mode = COPY_; if (flush == Z_TREES) goto inf_leave; + /* FALLTHROUGH */ case COPY_: state->mode = COPY; - /* FALL THROUGH */ + /* FALLTHROUGH */ case COPY: copy = state->length; if (copy) { @@ -1525,9 +1526,10 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return -1L << 16; + if (strm == Z_NULL || strm->state == Z_NULL) + return (long)(((unsigned long)0 - 1) << 16); state = (struct inflate_state FAR *)strm->state; - return ((long)(state->back) << 16) + + return (long)(((unsigned long)((long)state->back)) << 16) + (state->mode == COPY ? state->length : (state->mode == MATCH ? state->was - state->length : 0)); } diff --git a/zlib/inftrees.c b/zlib/inftrees.c index 44d89cf2..571e8100 100644 --- a/zlib/inftrees.c +++ b/zlib/inftrees.c @@ -54,7 +54,7 @@ unsigned short FAR *work; code FAR *next; /* next available space in table */ const unsigned short FAR *base; /* base value table to use */ const unsigned short FAR *extra; /* extra bits table to use */ - int end; /* use base and extra for symbol > end */ + unsigned match; /* use base and extra for symbol >= match */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ @@ -181,19 +181,17 @@ unsigned short FAR *work; switch (type) { case CODES: base = extra = work; /* dummy value--not used */ - end = 19; + match = 20; break; case LENS: base = lbase; - base -= 257; extra = lext; - extra -= 257; - end = 256; + match = 257; break; default: /* DISTS */ base = dbase; extra = dext; - end = -1; + match = 0; } /* initialize state for loop */ @@ -216,13 +214,13 @@ unsigned short FAR *work; for (;;) { /* create table entry */ here.bits = (unsigned char)(len - drop); - if ((int)(work[sym]) < end) { + if (work[sym] + 1u < match) { here.op = (unsigned char)0; here.val = work[sym]; } - else if ((int)(work[sym]) > end) { - here.op = (unsigned char)(extra[work[sym]]); - here.val = base[work[sym]]; + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; } else { here.op = (unsigned char)(32 + 64); /* end of block */ |